feat: instance manifest, tool profiles, metrics, and budget enforcement
Land HARNESS-WP-0001 T01/T02/T04/T05: extend ADR-005 schedule.yml with harness fields, named tool-profile registry, ADR-004 metrics writes, and BudgetTracker wiring. CLI gains validate/profiles; task-file path kept.
This commit is contained in:
parent
16f5c29c08
commit
4144eba160
17 changed files with 1387 additions and 60 deletions
|
|
@ -1,10 +1,13 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from agent_harness.manifest import HARNESS_MAJOR
|
||||
from agent_harness.runner import RunResult, run_task
|
||||
from agent_harness.taskspec import TaskSpec, TaskSpecError
|
||||
|
||||
|
|
@ -23,15 +26,26 @@ def _make_repo(tmp_path: Path) -> Path:
|
|||
return repo
|
||||
|
||||
|
||||
def _write_manifest(repo: Path, agents: dict) -> None:
|
||||
kaizen = repo / ".kaizen"
|
||||
kaizen.mkdir(exist_ok=True)
|
||||
data = {"version": "1", "harness": HARNESS_MAJOR, "agents": agents}
|
||||
(kaizen / "schedule.yml").write_text(
|
||||
yaml.safe_dump(data, sort_keys=False), encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
class CommittingAdapter:
|
||||
"""Fake adapter that simulates a session which commits."""
|
||||
|
||||
def __init__(self, repo: Path):
|
||||
self.repo = repo
|
||||
self.prompts: list[str] = []
|
||||
self.configs: list = []
|
||||
|
||||
def execute_prompt(self, prompt, config):
|
||||
self.prompts.append(prompt)
|
||||
self.configs.append(config)
|
||||
(self.repo / "HELLO.md").write_text("hello\n")
|
||||
subprocess.run(["git", "add", "."], cwd=self.repo, check=True)
|
||||
subprocess.run(
|
||||
|
|
@ -51,34 +65,140 @@ class IdleAdapter:
|
|||
return LLMResponse(content="nothing to do", model="fake", usage={}, finish_reason="stop")
|
||||
|
||||
|
||||
def _spec(repo: Path) -> TaskSpec:
|
||||
return TaskSpec(title="write hello", description="create HELLO.md", target_repo=repo)
|
||||
class BudgetBlowingAdapter:
|
||||
def execute_prompt(self, prompt, config):
|
||||
from llm_connect.exceptions import LLMBudgetExceededError
|
||||
|
||||
if config.budget_tracker is not None:
|
||||
# Simulate preflight/exhaustion the way adapters do.
|
||||
config.budget_tracker.consume(config.budget_tracker.total)
|
||||
config.budget_tracker.consume(1)
|
||||
raise LLMBudgetExceededError(
|
||||
"Token budget exceeded",
|
||||
total=1,
|
||||
spent=1,
|
||||
requested=1,
|
||||
)
|
||||
|
||||
|
||||
def _spec(repo: Path, agent: str = "coach") -> TaskSpec:
|
||||
return TaskSpec(
|
||||
title="write hello",
|
||||
description="create HELLO.md",
|
||||
target_repo=repo,
|
||||
agent=agent,
|
||||
)
|
||||
|
||||
|
||||
def test_run_task_success_when_session_commits(tmp_path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
adapter = CommittingAdapter(repo)
|
||||
|
||||
result = run_task(_spec(repo), adapter=adapter, report_to_hub=False)
|
||||
result = run_task(
|
||||
_spec(repo), adapter=adapter, report_to_hub=False, write_metrics=True
|
||||
)
|
||||
|
||||
assert isinstance(result, RunResult)
|
||||
assert result.ok is True
|
||||
assert result.committed is True
|
||||
assert result.head_before != result.head_after
|
||||
assert result.tool_profile == "green-commit-only"
|
||||
assert "write hello" in adapter.prompts[0]
|
||||
assert "Never push" in adapter.prompts[0]
|
||||
assert "Tool profile for this run: green-commit-only" in adapter.prompts[0]
|
||||
|
||||
metrics_path = repo / ".kaizen" / "metrics" / "coach" / "executions.jsonl"
|
||||
assert metrics_path.is_file()
|
||||
rec = json.loads(metrics_path.read_text().strip().splitlines()[-1])
|
||||
assert rec["success"] is True
|
||||
assert rec["committed"] is True
|
||||
|
||||
|
||||
def test_run_task_fails_without_commit(tmp_path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
|
||||
result = run_task(_spec(repo), adapter=IdleAdapter(), report_to_hub=False)
|
||||
result = run_task(
|
||||
_spec(repo), adapter=IdleAdapter(), report_to_hub=False, write_metrics=False
|
||||
)
|
||||
|
||||
assert result.ok is False
|
||||
assert result.committed is False
|
||||
assert result.reason == "session completed without committing"
|
||||
|
||||
|
||||
def test_run_task_refuses_unknown_tool_profile(tmp_path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
_write_manifest(
|
||||
repo,
|
||||
{
|
||||
"coach": {
|
||||
"cadence": "daily",
|
||||
"tool_profile": "not-a-real-profile",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
result = run_task(
|
||||
_spec(repo), adapter=IdleAdapter(), report_to_hub=False, write_metrics=False
|
||||
)
|
||||
|
||||
assert result.ok is False
|
||||
assert result.reason.startswith("refused:")
|
||||
assert "not-a-real-profile" in result.reason
|
||||
|
||||
|
||||
def test_run_task_resolves_manifest_profile_and_budget(tmp_path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
_write_manifest(
|
||||
repo,
|
||||
{
|
||||
"coach": {
|
||||
"cadence": "daily",
|
||||
"lane": "blue",
|
||||
"tool_profile": "blue-mail-triage",
|
||||
"budget": 25000,
|
||||
}
|
||||
},
|
||||
)
|
||||
adapter = CommittingAdapter(repo)
|
||||
|
||||
result = run_task(
|
||||
_spec(repo), adapter=adapter, report_to_hub=False, write_metrics=False
|
||||
)
|
||||
|
||||
assert result.ok is True
|
||||
assert result.tool_profile == "blue-mail-triage"
|
||||
assert result.budget_tokens == 25000
|
||||
assert adapter.configs[0].budget_tracker is not None
|
||||
assert adapter.configs[0].budget_tracker.total == 25000
|
||||
assert "blue-mail-triage" in adapter.prompts[0]
|
||||
|
||||
|
||||
def test_run_task_budget_exhaustion_fails(tmp_path) -> None:
|
||||
repo = _make_repo(tmp_path)
|
||||
_write_manifest(
|
||||
repo,
|
||||
{
|
||||
"coach": {
|
||||
"cadence": "daily",
|
||||
"tool_profile": "green-commit-only",
|
||||
"budget": 10,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
result = run_task(
|
||||
_spec(repo),
|
||||
adapter=BudgetBlowingAdapter(),
|
||||
report_to_hub=False,
|
||||
write_metrics=False,
|
||||
)
|
||||
|
||||
assert result.ok is False
|
||||
assert "session failed" in result.reason
|
||||
assert result.budget_tokens == 10
|
||||
|
||||
|
||||
def test_taskspec_rejects_non_repo(tmp_path) -> None:
|
||||
spec_file = tmp_path / "task.json"
|
||||
spec_file.write_text(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue