feat: instance manifest, tool profiles, metrics, and budget enforcement
Land HARNESS-WP-0001 T01/T02/T04/T05: extend ADR-005 schedule.yml with harness fields, named tool-profile registry, ADR-004 metrics writes, and BudgetTracker wiring. CLI gains validate/profiles; task-file path kept.
This commit is contained in:
parent
16f5c29c08
commit
4144eba160
17 changed files with 1387 additions and 60 deletions
215
tests/test_manifest.py
Normal file
215
tests/test_manifest.py
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from agent_harness.cli import main
|
||||
from agent_harness.manifest import (
|
||||
HARNESS_MAJOR,
|
||||
ManifestError,
|
||||
load_manifest,
|
||||
parse_manifest,
|
||||
resolve_run_policy,
|
||||
validate_manifest,
|
||||
)
|
||||
from agent_harness.profiles import UnknownToolProfileError, get_profile, list_profiles
|
||||
|
||||
|
||||
def _write_manifest(tmp_path: Path, data: dict) -> Path:
|
||||
kaizen = tmp_path / ".kaizen"
|
||||
kaizen.mkdir()
|
||||
path = kaizen / "schedule.yml"
|
||||
path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def test_parse_and_validate_adr005_only() -> None:
|
||||
manifest = parse_manifest(
|
||||
{
|
||||
"version": "1",
|
||||
"timezone": "Europe/Berlin",
|
||||
"agents": {
|
||||
"coach": {"cadence": "weekly", "enabled": True},
|
||||
},
|
||||
}
|
||||
)
|
||||
assert validate_manifest(manifest) == []
|
||||
assert validate_manifest(manifest, require_harness_fields=True)
|
||||
|
||||
|
||||
def test_validate_harness_extensions() -> None:
|
||||
manifest = parse_manifest(
|
||||
{
|
||||
"version": "1",
|
||||
"harness": HARNESS_MAJOR,
|
||||
"agents": {
|
||||
"coach": {
|
||||
"cadence": "daily",
|
||||
"enabled": True,
|
||||
"lane": "green",
|
||||
"tool_profile": "green-commit-only",
|
||||
"budget": 50000,
|
||||
},
|
||||
"mail": {
|
||||
"cadence": "weekly",
|
||||
"lane": "blue",
|
||||
"tool_profile": "blue-mail-triage",
|
||||
"budget": 10000,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
assert validate_manifest(manifest) == []
|
||||
assert validate_manifest(manifest, require_harness_fields=True) == []
|
||||
|
||||
|
||||
def test_unknown_tool_profile_errors() -> None:
|
||||
manifest = parse_manifest(
|
||||
{
|
||||
"version": "1",
|
||||
"agents": {
|
||||
"coach": {
|
||||
"cadence": "daily",
|
||||
"tool_profile": "does-not-exist",
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
errors = validate_manifest(manifest)
|
||||
assert any("unknown tool_profile" in e for e in errors)
|
||||
|
||||
|
||||
def test_lane_profile_mismatch() -> None:
|
||||
manifest = parse_manifest(
|
||||
{
|
||||
"version": "1",
|
||||
"agents": {
|
||||
"coach": {
|
||||
"cadence": "daily",
|
||||
"lane": "blue",
|
||||
"tool_profile": "green-commit-only",
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
errors = validate_manifest(manifest)
|
||||
assert any("does not match tool_profile" in e for e in errors)
|
||||
|
||||
|
||||
def test_budget_must_be_positive() -> None:
|
||||
with pytest.raises(ManifestError, match="budget"):
|
||||
parse_manifest(
|
||||
{
|
||||
"version": "1",
|
||||
"agents": {"coach": {"cadence": "daily", "budget": 0}},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_harness_major_mismatch() -> None:
|
||||
manifest = parse_manifest(
|
||||
{
|
||||
"version": "1",
|
||||
"harness": HARNESS_MAJOR + 1,
|
||||
"agents": {"coach": {"cadence": "daily"}},
|
||||
}
|
||||
)
|
||||
errors = validate_manifest(manifest)
|
||||
assert any("does not match this runtime" in e for e in errors)
|
||||
|
||||
|
||||
def test_resolve_run_policy_defaults(tmp_path: Path) -> None:
|
||||
profile, budget, lane = resolve_run_policy(tmp_path, "coach")
|
||||
assert profile == "green-commit-only"
|
||||
assert budget is None
|
||||
assert lane is None
|
||||
|
||||
|
||||
def test_resolve_run_policy_from_manifest(tmp_path: Path) -> None:
|
||||
_write_manifest(
|
||||
tmp_path,
|
||||
{
|
||||
"version": "1",
|
||||
"harness": HARNESS_MAJOR,
|
||||
"agents": {
|
||||
"coach": {
|
||||
"cadence": "daily",
|
||||
"tool_profile": "blue-mail-triage",
|
||||
"lane": "blue",
|
||||
"budget": 12000,
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
profile, budget, lane = resolve_run_policy(tmp_path, "coach")
|
||||
assert profile == "blue-mail-triage"
|
||||
assert budget == 12000
|
||||
assert lane == "blue"
|
||||
|
||||
|
||||
def test_resolve_unknown_profile_refuses(tmp_path: Path) -> None:
|
||||
_write_manifest(
|
||||
tmp_path,
|
||||
{
|
||||
"version": "1",
|
||||
"agents": {
|
||||
"coach": {
|
||||
"cadence": "daily",
|
||||
"tool_profile": "nope",
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
with pytest.raises(UnknownToolProfileError):
|
||||
resolve_run_policy(tmp_path, "coach")
|
||||
|
||||
|
||||
def test_load_manifest_missing(tmp_path: Path) -> None:
|
||||
with pytest.raises(ManifestError, match="not found"):
|
||||
load_manifest(tmp_path / ".kaizen" / "schedule.yml")
|
||||
|
||||
|
||||
def test_profiles_registry() -> None:
|
||||
names = {p.name for p in list_profiles()}
|
||||
assert names == {"green-commit-only", "blue-mail-triage"}
|
||||
green = get_profile("green-commit-only")
|
||||
assert "git commit" in green.allowed_tools
|
||||
assert "git push" not in green.allowed_tools
|
||||
|
||||
|
||||
def test_cli_validate_ok(tmp_path: Path) -> None:
|
||||
_write_manifest(
|
||||
tmp_path,
|
||||
{
|
||||
"version": "1",
|
||||
"harness": HARNESS_MAJOR,
|
||||
"agents": {
|
||||
"coach": {
|
||||
"cadence": "daily",
|
||||
"lane": "green",
|
||||
"tool_profile": "green-commit-only",
|
||||
"budget": 1000,
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
assert main(["validate", "--target", str(tmp_path), "--strict"]) == 0
|
||||
|
||||
|
||||
def test_cli_validate_fails_unknown_profile(tmp_path: Path) -> None:
|
||||
_write_manifest(
|
||||
tmp_path,
|
||||
{
|
||||
"version": "1",
|
||||
"agents": {
|
||||
"coach": {"cadence": "daily", "tool_profile": "missing"}
|
||||
},
|
||||
},
|
||||
)
|
||||
assert main(["validate", "--target", str(tmp_path)]) == 1
|
||||
|
||||
|
||||
def test_cli_profiles() -> None:
|
||||
assert main(["profiles"]) == 0
|
||||
Loading…
Add table
Add a link
Reference in a new issue