Normalize Glas execution evidence
Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a028de-e2c8-7732-8521-46a7fc5db82f
This commit is contained in:
parent
b933cf52c8
commit
7e81545b24
13 changed files with 528 additions and 33 deletions
115
tests/test_glas_evidence.py
Normal file
115
tests/test_glas_evidence.py
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
"""Normalized Glas evidence boundary (ACTIVITY-WP-0032-T04)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from activity_core.glas_evidence import (
|
||||
normalise_execution_evidence,
|
||||
normalise_ops_result,
|
||||
)
|
||||
|
||||
|
||||
def _gateway_result() -> dict:
|
||||
return {
|
||||
"ok": True,
|
||||
"evidence": {
|
||||
"request_id": "request-1",
|
||||
"correlation_id": "corr-1",
|
||||
"actor": "agt",
|
||||
"project": "activity-core",
|
||||
"target_repo": "activity-core",
|
||||
"contract_version": "1.0",
|
||||
"profile_ref": "harness.agent-dev@1.0.0",
|
||||
"rein_id": "rein-aharness",
|
||||
"rein_version": "0.1.0",
|
||||
"model_route": "claude-code-cli",
|
||||
"resolved_model": "claude-sonnet-4-6",
|
||||
"sandbox_profile": "profile.bwrap-local",
|
||||
"sandbox_id": "sandbox-1",
|
||||
"tool_profile": "green-commit-only",
|
||||
"outcome": "succeeded",
|
||||
"started_at": "2026-08-22T20:00:00Z",
|
||||
"finished_at": "2026-08-22T20:00:05Z",
|
||||
"duration_s": 5.0,
|
||||
"tokens_spent": 0,
|
||||
"token_budget": 60000,
|
||||
"commit_sha": "deadbeef",
|
||||
"artifacts": ["README.md"],
|
||||
"tool_events_count": 0,
|
||||
"tool_events_completeness": "unavailable",
|
||||
"refs": {
|
||||
"assignment_ref": "assignment:42",
|
||||
"goal_refs": ["goal:42@1", {"credential": "must-drop"}],
|
||||
"api_key": "must-drop",
|
||||
},
|
||||
"raw_prompt": "must-drop",
|
||||
"provider_response": {"must": "drop"},
|
||||
},
|
||||
"tool_output": "sensitive direct model output",
|
||||
"tool_error": "sensitive provider exception",
|
||||
"provider_blob": {"credential": "must-drop"},
|
||||
}
|
||||
|
||||
|
||||
def test_normalises_gateway_result_and_drops_direct_output() -> None:
|
||||
result = normalise_ops_result(_gateway_result())
|
||||
|
||||
assert result["ok"] is True
|
||||
evidence = result["execution_evidence"]
|
||||
assert evidence["profile_ref"] == "harness.agent-dev@1.0.0"
|
||||
assert evidence["rein_id"] == "rein-aharness"
|
||||
assert evidence["tokens_spent"] == 0
|
||||
assert evidence["tool_events_count"] == 0
|
||||
assert evidence["tool_events_completeness"] == "unavailable"
|
||||
assert evidence["refs"] == {
|
||||
"assignment_ref": "assignment:42",
|
||||
"goal_refs": ["goal:42@1"],
|
||||
}
|
||||
|
||||
serialized = json.dumps(result)
|
||||
for forbidden in (
|
||||
"tool_output",
|
||||
"tool_error",
|
||||
"provider_blob",
|
||||
"raw_prompt",
|
||||
"provider_response",
|
||||
"credential",
|
||||
):
|
||||
assert forbidden not in serialized
|
||||
|
||||
|
||||
def test_rejects_invalid_enums_and_non_finite_measurements() -> None:
|
||||
evidence = normalise_execution_evidence(
|
||||
{
|
||||
"outcome": "maybe",
|
||||
"failure_stage": "provider-internal",
|
||||
"duration_s": float("nan"),
|
||||
"tokens_spent": -1,
|
||||
"tool_events_completeness": "unknown-ish",
|
||||
}
|
||||
)
|
||||
|
||||
assert evidence == {}
|
||||
|
||||
|
||||
def test_keeps_legacy_artifact_fields_but_not_unknown_blobs() -> None:
|
||||
result = normalise_ops_result(
|
||||
{
|
||||
"ok": True,
|
||||
"path": "briefs/report.md",
|
||||
"head_after": "deadbeef",
|
||||
"target_repo": "activity-core",
|
||||
"artifact_urls": [
|
||||
{"kind": "report", "label": "Report", "url": "https://example.test/a"},
|
||||
{"kind": "unsafe", "label": "Unsafe", "url": "file:///tmp/key"},
|
||||
],
|
||||
"messages": [{"role": "user", "content": "secret"}],
|
||||
}
|
||||
)
|
||||
|
||||
assert result["path"] == "briefs/report.md"
|
||||
assert result["artifact_urls"] == [
|
||||
{"kind": "report", "label": "Report", "url": "https://example.test/a"}
|
||||
]
|
||||
assert "messages" not in result
|
||||
|
|
@ -148,6 +148,13 @@ class TestExecutionRefs:
|
|||
|
||||
assert refs == {"resource_envelope_refs": ["ok"]}
|
||||
|
||||
def test_drops_non_string_items_in_reference_lists(self) -> None:
|
||||
refs = normalise_execution_refs(
|
||||
{"goal_refs": ["goal:1@1", {"credential": "must-drop"}, 17]}
|
||||
)
|
||||
|
||||
assert refs == {"goal_refs": ["goal:1@1"]}
|
||||
|
||||
@pytest.mark.parametrize("bad", [None, "refs", ["a"], 5])
|
||||
def test_non_dict_becomes_empty(self, bad) -> None:
|
||||
assert normalise_execution_refs(bad) == {}
|
||||
|
|
|
|||
|
|
@ -225,6 +225,47 @@ async def test_claim_and_complete_roundtrip(monkeypatch: pytest.MonkeyPatch) ->
|
|||
assert open_run.result["path"] == "briefs/x.md"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_complete_persists_only_normalized_glas_evidence() -> None:
|
||||
from activity_core import ops_run_queue as oq
|
||||
|
||||
row = MagicMock()
|
||||
row.state = "claimed"
|
||||
row.claim_owner = "worker-1"
|
||||
row.result = {}
|
||||
session = AsyncMock()
|
||||
session.get = AsyncMock(return_value=row)
|
||||
|
||||
done = await oq.complete_ops_run(
|
||||
session,
|
||||
uuid.uuid4(),
|
||||
worker_id="worker-1",
|
||||
result={
|
||||
"ok": True,
|
||||
"evidence": {
|
||||
"request_id": "request-1",
|
||||
"profile_ref": "harness.agent-dev@1.0.0",
|
||||
"rein_id": "rein-aharness",
|
||||
"outcome": "succeeded",
|
||||
"duration_s": 0,
|
||||
},
|
||||
"tool_output": "must not persist",
|
||||
},
|
||||
)
|
||||
|
||||
assert done is row
|
||||
assert row.result == {
|
||||
"ok": True,
|
||||
"execution_evidence": {
|
||||
"request_id": "request-1",
|
||||
"profile_ref": "harness.agent-dev@1.0.0",
|
||||
"rein_id": "rein-aharness",
|
||||
"outcome": "succeeded",
|
||||
"duration_s": 0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fail_reopen_under_max_attempts(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from activity_core import ops_run_queue as oq
|
||||
|
|
@ -268,6 +309,44 @@ async def test_fail_permanent_at_max_attempts(monkeypatch: pytest.MonkeyPatch) -
|
|||
assert row.state == "failed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fail_persists_redacted_failure_evidence() -> None:
|
||||
from activity_core import ops_run_queue as oq
|
||||
|
||||
row = MagicMock()
|
||||
row.state = "claimed"
|
||||
row.claim_owner = "worker-1"
|
||||
row.attempt = 1
|
||||
row.result = {}
|
||||
session = AsyncMock()
|
||||
session.get = AsyncMock(return_value=row)
|
||||
|
||||
failed = await oq.fail_ops_run(
|
||||
session,
|
||||
uuid.uuid4(),
|
||||
worker_id="worker-1",
|
||||
error="execution refused",
|
||||
result={
|
||||
"ok": False,
|
||||
"evidence": {
|
||||
"request_id": "request-failed",
|
||||
"profile_ref": "harness.agent-dev@1.0.0",
|
||||
"rein_id": "rein-aharness",
|
||||
"outcome": "failed",
|
||||
"failure_stage": "execution",
|
||||
"error": "execution failed; inspect direct caller error",
|
||||
"duration_s": 3.5,
|
||||
},
|
||||
"tool_error": "raw provider failure must not persist",
|
||||
},
|
||||
)
|
||||
|
||||
assert failed is row
|
||||
assert row.result["error"] == "execution refused"
|
||||
assert row.result["execution_evidence"]["failure_stage"] == "execution"
|
||||
assert "tool_error" not in row.result
|
||||
|
||||
|
||||
def test_label_filter_any_vs_all() -> None:
|
||||
"""Document labels_mode semantics used by claim_ops_runs."""
|
||||
row_labels = {"automated", "research-brief"}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from datetime import datetime, timedelta, timezone
|
|||
from unittest.mock import MagicMock
|
||||
|
||||
from activity_core.run_artifacts import (
|
||||
_ops_summary,
|
||||
artifacts_from_ops_result,
|
||||
build_forgejo_blob_url,
|
||||
match_ops_runs_to_activity_run,
|
||||
|
|
@ -88,3 +89,42 @@ def test_match_ops_runs_window_fallback() -> None:
|
|||
matched = match_ops_runs_to_activity_run(ar, [far, near])
|
||||
assert near in matched
|
||||
assert far not in matched
|
||||
|
||||
|
||||
def test_ops_summary_exposes_compact_execution_constellation() -> None:
|
||||
now = datetime(2026, 8, 22, 20, 0, tzinfo=timezone.utc)
|
||||
row = MagicMock()
|
||||
row.id = uuid.uuid4()
|
||||
row.state = "succeeded"
|
||||
row.title = "Profiled run"
|
||||
row.target_repo = "activity-core"
|
||||
row.claim_owner = "glas-worker@railiance01"
|
||||
row.attempt = 1
|
||||
row.triggering_event_id = "event-1"
|
||||
row.source_id = "profiled"
|
||||
row.approach_hint = "legacy-only"
|
||||
row.harness_profile_ref = "harness.agent-dev@1.0.0"
|
||||
row.execution_refs = {"goal_refs": ["goal:42@1"]}
|
||||
row.created_at = now
|
||||
row.updated_at = now
|
||||
row.result = {
|
||||
"ok": True,
|
||||
"execution_evidence": {
|
||||
"profile_ref": "harness.agent-dev@1.0.0",
|
||||
"rein_id": "rein-aharness",
|
||||
"rein_version": "0.1.0",
|
||||
"resolved_model": "claude-sonnet-4-6",
|
||||
"sandbox_profile": "profile.bwrap-local",
|
||||
"outcome": "succeeded",
|
||||
"duration_s": 5.2,
|
||||
},
|
||||
"raw_output": "must-drop-even-for-historic-rows",
|
||||
}
|
||||
|
||||
summary = _ops_summary(row)
|
||||
|
||||
assert summary["execution_evidence"]["rein_id"] == "rein-aharness"
|
||||
assert summary["result"]["execution_evidence"]["profile_ref"] == (
|
||||
"harness.agent-dev@1.0.0"
|
||||
)
|
||||
assert "raw_output" not in str(summary)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue