Normalize Glas execution evidence
All checks were successful
CI Smoke / host-smoke (push) Successful in 0s
CI Smoke / container-smoke (push) Successful in 1s
Build and Publish Container Image / build-and-push (push) Successful in 20s

Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a028de-e2c8-7732-8521-46a7fc5db82f
This commit is contained in:
tegwick 2026-08-22 23:11:52 +02:00
parent b933cf52c8
commit 7e81545b24
13 changed files with 528 additions and 33 deletions

115
tests/test_glas_evidence.py Normal file
View file

@ -0,0 +1,115 @@
"""Normalized Glas evidence boundary (ACTIVITY-WP-0032-T04)."""
from __future__ import annotations
import json
from activity_core.glas_evidence import (
normalise_execution_evidence,
normalise_ops_result,
)
def _gateway_result() -> dict:
return {
"ok": True,
"evidence": {
"request_id": "request-1",
"correlation_id": "corr-1",
"actor": "agt",
"project": "activity-core",
"target_repo": "activity-core",
"contract_version": "1.0",
"profile_ref": "harness.agent-dev@1.0.0",
"rein_id": "rein-aharness",
"rein_version": "0.1.0",
"model_route": "claude-code-cli",
"resolved_model": "claude-sonnet-4-6",
"sandbox_profile": "profile.bwrap-local",
"sandbox_id": "sandbox-1",
"tool_profile": "green-commit-only",
"outcome": "succeeded",
"started_at": "2026-08-22T20:00:00Z",
"finished_at": "2026-08-22T20:00:05Z",
"duration_s": 5.0,
"tokens_spent": 0,
"token_budget": 60000,
"commit_sha": "deadbeef",
"artifacts": ["README.md"],
"tool_events_count": 0,
"tool_events_completeness": "unavailable",
"refs": {
"assignment_ref": "assignment:42",
"goal_refs": ["goal:42@1", {"credential": "must-drop"}],
"api_key": "must-drop",
},
"raw_prompt": "must-drop",
"provider_response": {"must": "drop"},
},
"tool_output": "sensitive direct model output",
"tool_error": "sensitive provider exception",
"provider_blob": {"credential": "must-drop"},
}
def test_normalises_gateway_result_and_drops_direct_output() -> None:
result = normalise_ops_result(_gateway_result())
assert result["ok"] is True
evidence = result["execution_evidence"]
assert evidence["profile_ref"] == "harness.agent-dev@1.0.0"
assert evidence["rein_id"] == "rein-aharness"
assert evidence["tokens_spent"] == 0
assert evidence["tool_events_count"] == 0
assert evidence["tool_events_completeness"] == "unavailable"
assert evidence["refs"] == {
"assignment_ref": "assignment:42",
"goal_refs": ["goal:42@1"],
}
serialized = json.dumps(result)
for forbidden in (
"tool_output",
"tool_error",
"provider_blob",
"raw_prompt",
"provider_response",
"credential",
):
assert forbidden not in serialized
def test_rejects_invalid_enums_and_non_finite_measurements() -> None:
evidence = normalise_execution_evidence(
{
"outcome": "maybe",
"failure_stage": "provider-internal",
"duration_s": float("nan"),
"tokens_spent": -1,
"tool_events_completeness": "unknown-ish",
}
)
assert evidence == {}
def test_keeps_legacy_artifact_fields_but_not_unknown_blobs() -> None:
result = normalise_ops_result(
{
"ok": True,
"path": "briefs/report.md",
"head_after": "deadbeef",
"target_repo": "activity-core",
"artifact_urls": [
{"kind": "report", "label": "Report", "url": "https://example.test/a"},
{"kind": "unsafe", "label": "Unsafe", "url": "file:///tmp/key"},
],
"messages": [{"role": "user", "content": "secret"}],
}
)
assert result["path"] == "briefs/report.md"
assert result["artifact_urls"] == [
{"kind": "report", "label": "Report", "url": "https://example.test/a"}
]
assert "messages" not in result

View file

@ -148,6 +148,13 @@ class TestExecutionRefs:
assert refs == {"resource_envelope_refs": ["ok"]}
def test_drops_non_string_items_in_reference_lists(self) -> None:
refs = normalise_execution_refs(
{"goal_refs": ["goal:1@1", {"credential": "must-drop"}, 17]}
)
assert refs == {"goal_refs": ["goal:1@1"]}
@pytest.mark.parametrize("bad", [None, "refs", ["a"], 5])
def test_non_dict_becomes_empty(self, bad) -> None:
assert normalise_execution_refs(bad) == {}

View file

@ -225,6 +225,47 @@ async def test_claim_and_complete_roundtrip(monkeypatch: pytest.MonkeyPatch) ->
assert open_run.result["path"] == "briefs/x.md"
@pytest.mark.asyncio
async def test_complete_persists_only_normalized_glas_evidence() -> None:
from activity_core import ops_run_queue as oq
row = MagicMock()
row.state = "claimed"
row.claim_owner = "worker-1"
row.result = {}
session = AsyncMock()
session.get = AsyncMock(return_value=row)
done = await oq.complete_ops_run(
session,
uuid.uuid4(),
worker_id="worker-1",
result={
"ok": True,
"evidence": {
"request_id": "request-1",
"profile_ref": "harness.agent-dev@1.0.0",
"rein_id": "rein-aharness",
"outcome": "succeeded",
"duration_s": 0,
},
"tool_output": "must not persist",
},
)
assert done is row
assert row.result == {
"ok": True,
"execution_evidence": {
"request_id": "request-1",
"profile_ref": "harness.agent-dev@1.0.0",
"rein_id": "rein-aharness",
"outcome": "succeeded",
"duration_s": 0,
},
}
@pytest.mark.asyncio
async def test_fail_reopen_under_max_attempts(monkeypatch: pytest.MonkeyPatch) -> None:
from activity_core import ops_run_queue as oq
@ -268,6 +309,44 @@ async def test_fail_permanent_at_max_attempts(monkeypatch: pytest.MonkeyPatch) -
assert row.state == "failed"
@pytest.mark.asyncio
async def test_fail_persists_redacted_failure_evidence() -> None:
from activity_core import ops_run_queue as oq
row = MagicMock()
row.state = "claimed"
row.claim_owner = "worker-1"
row.attempt = 1
row.result = {}
session = AsyncMock()
session.get = AsyncMock(return_value=row)
failed = await oq.fail_ops_run(
session,
uuid.uuid4(),
worker_id="worker-1",
error="execution refused",
result={
"ok": False,
"evidence": {
"request_id": "request-failed",
"profile_ref": "harness.agent-dev@1.0.0",
"rein_id": "rein-aharness",
"outcome": "failed",
"failure_stage": "execution",
"error": "execution failed; inspect direct caller error",
"duration_s": 3.5,
},
"tool_error": "raw provider failure must not persist",
},
)
assert failed is row
assert row.result["error"] == "execution refused"
assert row.result["execution_evidence"]["failure_stage"] == "execution"
assert "tool_error" not in row.result
def test_label_filter_any_vs_all() -> None:
"""Document labels_mode semantics used by claim_ops_runs."""
row_labels = {"automated", "research-brief"}

View file

@ -7,6 +7,7 @@ from datetime import datetime, timedelta, timezone
from unittest.mock import MagicMock
from activity_core.run_artifacts import (
_ops_summary,
artifacts_from_ops_result,
build_forgejo_blob_url,
match_ops_runs_to_activity_run,
@ -88,3 +89,42 @@ def test_match_ops_runs_window_fallback() -> None:
matched = match_ops_runs_to_activity_run(ar, [far, near])
assert near in matched
assert far not in matched
def test_ops_summary_exposes_compact_execution_constellation() -> None:
now = datetime(2026, 8, 22, 20, 0, tzinfo=timezone.utc)
row = MagicMock()
row.id = uuid.uuid4()
row.state = "succeeded"
row.title = "Profiled run"
row.target_repo = "activity-core"
row.claim_owner = "glas-worker@railiance01"
row.attempt = 1
row.triggering_event_id = "event-1"
row.source_id = "profiled"
row.approach_hint = "legacy-only"
row.harness_profile_ref = "harness.agent-dev@1.0.0"
row.execution_refs = {"goal_refs": ["goal:42@1"]}
row.created_at = now
row.updated_at = now
row.result = {
"ok": True,
"execution_evidence": {
"profile_ref": "harness.agent-dev@1.0.0",
"rein_id": "rein-aharness",
"rein_version": "0.1.0",
"resolved_model": "claude-sonnet-4-6",
"sandbox_profile": "profile.bwrap-local",
"outcome": "succeeded",
"duration_s": 5.2,
},
"raw_output": "must-drop-even-for-historic-rows",
}
summary = _ops_summary(row)
assert summary["execution_evidence"]["rein_id"] == "rein-aharness"
assert summary["result"]["execution_evidence"]["profile_ref"] == (
"harness.agent-dev@1.0.0"
)
assert "raw_output" not in str(summary)