feat: instance manifest, tool profiles, metrics, and budget enforcement
Land HARNESS-WP-0001 T01/T02/T04/T05: extend ADR-005 schedule.yml with harness fields, named tool-profile registry, ADR-004 metrics writes, and BudgetTracker wiring. CLI gains validate/profiles; task-file path kept.
This commit is contained in:
parent
16f5c29c08
commit
4144eba160
17 changed files with 1387 additions and 60 deletions
|
|
@ -1,21 +1,24 @@
|
|||
"""One-task run orchestration.
|
||||
|
||||
Flow: lock target repo → snapshot HEAD → persona bundle → prompt → agentic
|
||||
session → verify a new commit exists → hub progress event (+ task close).
|
||||
The run *fails* if the session pushed anywhere or left the repo dirty in a
|
||||
way it should not — the worker never pushes; publishing is a separate,
|
||||
explicitly-granted lane (see integrations/executor-worker-secrets.md in
|
||||
binky-control).
|
||||
Flow: lock target repo → resolve tool profile / budget from instance
|
||||
manifest → snapshot HEAD → persona bundle → prompt → agentic session →
|
||||
verify a new commit exists → kaizen metrics + hub progress event
|
||||
(+ task close). The run *fails* if the session pushed anywhere or left
|
||||
the repo dirty in a way it should not — the worker never pushes;
|
||||
publishing is a separate, explicitly-granted lane.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from agent_harness import hub
|
||||
from agent_harness import hub, metrics
|
||||
from agent_harness.manifest import resolve_run_policy
|
||||
from agent_harness.persona import load_persona_bundle
|
||||
from agent_harness.profiles import UnknownToolProfileError, get_profile
|
||||
from agent_harness.taskspec import TaskSpec
|
||||
|
||||
PROMPT_TEMPLATE = """\
|
||||
|
|
@ -24,6 +27,7 @@ Operating rules, non-negotiable:
|
|||
- Work ONLY inside the current repository working directory.
|
||||
- Green/Blue lane: file edits and local git add/commit only. Never push,
|
||||
never touch the network, never run destructive commands.
|
||||
- Tool profile for this run: {tool_profile} (lane={lane}).
|
||||
- Bounded effort: complete the single task below, commit with a clear
|
||||
message, then stop. If the task cannot be completed, commit nothing and
|
||||
say why in your final output.
|
||||
|
|
@ -45,6 +49,10 @@ class RunResult:
|
|||
persona_source: str
|
||||
session_output: str
|
||||
reason: str = ""
|
||||
tool_profile: str = ""
|
||||
budget_tokens: int | None = None
|
||||
tokens_spent: int | None = None
|
||||
execution_time_s: float = 0.0
|
||||
|
||||
|
||||
def _git(repo: Path, *args: str) -> str:
|
||||
|
|
@ -59,11 +67,49 @@ def _git(repo: Path, *args: str) -> str:
|
|||
return result.stdout.strip()
|
||||
|
||||
|
||||
def run_task(spec: TaskSpec, adapter=None, report_to_hub: bool = True) -> RunResult:
|
||||
def run_task(
|
||||
spec: TaskSpec,
|
||||
adapter=None,
|
||||
report_to_hub: bool = True,
|
||||
write_metrics: bool = True,
|
||||
) -> RunResult:
|
||||
try:
|
||||
profile_name, budget_tokens, lane = resolve_run_policy(
|
||||
spec.target_repo, spec.agent
|
||||
)
|
||||
profile = get_profile(profile_name)
|
||||
except UnknownToolProfileError as exc:
|
||||
return RunResult(
|
||||
ok=False,
|
||||
committed=False,
|
||||
head_before="",
|
||||
head_after="",
|
||||
persona_source="none",
|
||||
session_output="",
|
||||
reason=f"refused: {exc}",
|
||||
tool_profile="",
|
||||
budget_tokens=None,
|
||||
)
|
||||
except Exception as exc:
|
||||
return RunResult(
|
||||
ok=False,
|
||||
committed=False,
|
||||
head_before="",
|
||||
head_after="",
|
||||
persona_source="none",
|
||||
session_output="",
|
||||
reason=f"manifest resolution failed: {exc}",
|
||||
tool_profile="",
|
||||
budget_tokens=None,
|
||||
)
|
||||
|
||||
if adapter is None:
|
||||
from agent_harness.adapter import AgenticClaudeCodeAdapter
|
||||
|
||||
adapter = AgenticClaudeCodeAdapter(workdir=spec.target_repo)
|
||||
adapter = AgenticClaudeCodeAdapter(
|
||||
workdir=spec.target_repo,
|
||||
tool_profile=profile,
|
||||
)
|
||||
|
||||
head_before = _git(spec.target_repo, "rev-parse", "HEAD")
|
||||
persona, persona_source = load_persona_bundle(spec.agent, spec.target_repo)
|
||||
|
|
@ -71,20 +117,30 @@ def run_task(spec: TaskSpec, adapter=None, report_to_hub: bool = True) -> RunRes
|
|||
persona=persona or "(no persona bundle available for this run)",
|
||||
title=spec.title,
|
||||
description=spec.description,
|
||||
tool_profile=profile.name,
|
||||
lane=lane or profile.lane,
|
||||
)
|
||||
|
||||
from llm_connect.models import RunConfig
|
||||
from llm_connect.models import BudgetTracker, RunConfig
|
||||
|
||||
config = RunConfig(timeout_seconds=spec.timeout_seconds, skip_if_exists=False)
|
||||
budget_tracker = BudgetTracker(total=budget_tokens) if budget_tokens else None
|
||||
config = RunConfig(
|
||||
timeout_seconds=spec.timeout_seconds,
|
||||
skip_if_exists=False,
|
||||
budget_tracker=budget_tracker,
|
||||
)
|
||||
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = adapter.execute_prompt(prompt, config)
|
||||
session_output = response.content
|
||||
session_ok = True
|
||||
reason = ""
|
||||
except Exception as exc: # adapter failures must still be reported
|
||||
except Exception as exc: # adapter / budget failures must still be reported
|
||||
session_output = ""
|
||||
session_ok = False
|
||||
reason = f"session failed: {exc}"
|
||||
execution_time_s = time.monotonic() - started
|
||||
|
||||
head_after = _git(spec.target_repo, "rev-parse", "HEAD")
|
||||
committed = head_after != head_before
|
||||
|
|
@ -92,6 +148,8 @@ def run_task(spec: TaskSpec, adapter=None, report_to_hub: bool = True) -> RunRes
|
|||
if session_ok and not committed:
|
||||
reason = "session completed without committing"
|
||||
|
||||
tokens_spent = budget_tracker.spent if budget_tracker is not None else None
|
||||
|
||||
result = RunResult(
|
||||
ok=ok,
|
||||
committed=committed,
|
||||
|
|
@ -100,8 +158,33 @@ def run_task(spec: TaskSpec, adapter=None, report_to_hub: bool = True) -> RunRes
|
|||
persona_source=persona_source,
|
||||
session_output=session_output,
|
||||
reason=reason,
|
||||
tool_profile=profile.name,
|
||||
budget_tokens=budget_tokens,
|
||||
tokens_spent=tokens_spent,
|
||||
execution_time_s=execution_time_s,
|
||||
)
|
||||
|
||||
if write_metrics:
|
||||
try:
|
||||
metrics.record_execution(
|
||||
spec.target_repo,
|
||||
spec.agent,
|
||||
success=ok,
|
||||
execution_time_s=execution_time_s,
|
||||
tokens=tokens_spent,
|
||||
committed=committed,
|
||||
head_after=head_after,
|
||||
reason=reason or None,
|
||||
metadata={
|
||||
"task_title": spec.title,
|
||||
"tool_profile": profile.name,
|
||||
"labels": list(spec.labels),
|
||||
"completion_event_type": spec.completion_event_type,
|
||||
},
|
||||
)
|
||||
except OSError:
|
||||
pass # metrics must not block run completion reporting
|
||||
|
||||
if report_to_hub:
|
||||
detail = {
|
||||
"repo": spec.target_repo.name,
|
||||
|
|
@ -113,6 +196,10 @@ def run_task(spec: TaskSpec, adapter=None, report_to_hub: bool = True) -> RunRes
|
|||
"head_after": head_after,
|
||||
"ok": ok,
|
||||
"reason": reason,
|
||||
"tool_profile": profile.name,
|
||||
"budget_tokens": budget_tokens,
|
||||
"tokens_spent": tokens_spent,
|
||||
"execution_time_s": round(execution_time_s, 3),
|
||||
}
|
||||
hub.post_progress_event(
|
||||
summary=f"executor run: {spec.title} ({'ok' if ok else 'failed'})",
|
||||
|
|
@ -120,6 +207,15 @@ def run_task(spec: TaskSpec, adapter=None, report_to_hub: bool = True) -> RunRes
|
|||
detail=detail,
|
||||
task_id=spec.hub_task_id,
|
||||
)
|
||||
if tokens_spent is not None or budget_tokens is not None:
|
||||
hub.post_token_event(
|
||||
repo=spec.target_repo.name,
|
||||
tokens=tokens_spent or 0,
|
||||
budget=budget_tokens,
|
||||
agent=spec.agent,
|
||||
ok=ok,
|
||||
detail={"task_title": spec.title, "tool_profile": profile.name},
|
||||
)
|
||||
if ok and spec.hub_task_id:
|
||||
hub.close_task(spec.hub_task_id)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue