Run outcomes, score, explicit exercise recording, and step captures (RBT-WP-0004-T13)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: sonnet
Assistant-Process: 295952@bnt-lap001
Assistant-Session: e93f64ad-516c-46eb-9666-aad8d300c477

rtut/run.py

@@ -63,6 +63,12 @@ class Run:
         return all(v["state"] in ("done", "skipped") for v in self.data["steps"].values())
 
 
+
+def _writable(run: Run) -> None:
+    if run.data.get("outcome"):
+        raise RunError(f"run is finished (outcome: {run.data['outcome']}); start a new run")
+
+
 def create(pack: Pack, operator: str, params: dict[str, str] | None = None) -> Run:
     if not pack.ok or pack.data.get("engine") != "native":
         raise RunError("only valid native packs can be run")
@@ -125,6 +131,7 @@ def command_for(run: Run, step_id: str) -> str | None:
 
 
 def start(run: Run, step_id: str) -> None:
+    _writable(run)
     run.step_def(step_id)
     _order(run, step_id)
     st = run.state(step_id)
@@ -140,6 +147,7 @@ def verify(run: Run, step_id: str, confirm: bool = False, manual_ok: bool | None
     Stored: exit code, duration, output size, and whether 'contains' matched.
     Output text is never stored.
     """
+    _writable(run)
     v = run.step_def(step_id)["verify"]
     st = run.state(step_id)
     if st["state"] != "running":
@@ -174,6 +182,7 @@ def verify(run: Run, step_id: str, confirm: bool = False, manual_ok: bool | None
 
 
 def complete(run: Run, step_id: str, evidence: dict[str, str] | None = None, note: str = "") -> None:
+    _writable(run)
     sd = run.step_def(step_id)
     st = run.state(step_id)
     if st["state"] != "running":
@@ -192,6 +201,7 @@ def complete(run: Run, step_id: str, evidence: dict[str, str] | None = None, not
 
 
 def fail(run: Run, step_id: str, note: str) -> None:
+    _writable(run)
     st = run.state(step_id)
     if st["state"] != "running":
         raise RunError(f"step {step_id} is not running")
@@ -200,6 +210,7 @@ def fail(run: Run, step_id: str, note: str) -> None:
 
 
 def retry(run: Run, step_id: str) -> None:
+    _writable(run)
     st = run.state(step_id)
     if st["state"] != "failed":
         raise RunError(f"step {step_id} has not failed")
@@ -209,6 +220,7 @@ def retry(run: Run, step_id: str) -> None:
 
 
 def skip(run: Run, step_id: str, reason: str) -> None:
+    _writable(run)
     sd = run.step_def(step_id)
     if sd.get("risk", "read-only") != "read-only":
         raise RunError("only read-only steps may be skipped")
@@ -262,3 +274,109 @@ def all_feedback(pack: Pack) -> list[dict]:
         if fb:
             out.append({"run_id": rid, "operator": r.data["operator"], **fb})
     return sorted(out, key=lambda x: x["at"], reverse=True)
+
+
+# ---- captures: optional documentation of what was run and what came back --------------------
+
+CAPTURE_MAX_OUTPUT = 20000
+CAPTURE_MAX_COMMAND = 2000
+
+
+def add_capture(run: Run, step_id: str, output: str, command: str = "", source: str = "operator") -> dict:
+    """Keep the command actually run and its output as documentation or for inspection.
+
+    The engine cannot detect every secret. It refuses obvious markers and oversized text;
+    the operator remains responsible for what is pasted.
+    """
+    _writable(run)
+    run.step_def(step_id)
+    st = run.state(step_id)
+    if st["state"] == "todo":
+        raise RunError("start the step before capturing its output")
+    if len(output) > CAPTURE_MAX_OUTPUT or len(command) > CAPTURE_MAX_COMMAND:
+        raise RunError(f"capture too large (output max {CAPTURE_MAX_OUTPUT}, command max {CAPTURE_MAX_COMMAND} characters)")
+    if not output.strip() and not command.strip():
+        raise RunError("nothing to capture")
+    entry = {"command": _clean(command.strip(), "captured command"), "output": _clean(output.rstrip(), "captured output"),
+             "source": source, "at": _now()}
+    st.setdefault("captures", []).append(entry)
+    run.save()
+    return entry
+
+
+def verify_pasted(run: Run, step_id: str, output: str, command: str = "") -> dict:
+    """Check an output-contains step against pasted output, for systems the engine cannot reach."""
+    _writable(run)
+    v = run.step_def(step_id)["verify"]
+    st = run.state(step_id)
+    if st["state"] != "running":
+        raise RunError(f"step {step_id} is not running")
+    if v.get("expect") != "output-contains":
+        raise RunError("only output-contains checks can be verified from pasted output")
+    add_capture(run, step_id, output, command, source="pasted-for-verification")
+    needle = render(v["contains"], run.data["params"], run.pack)
+    result = {"kind": "output-contains", "source": "pasted", "passed": needle in output,
+              "output_bytes": len(output.encode()), "at": _now()}
+    st["verification"] = result
+    run.save()
+    return result
+
+
+# ---- outcome and score ------------------------------------------------------------------------
+
+OUTCOMES = ("succeeded", "failed", "abandoned")
+STALE_DAYS = 7
+
+
+def finalize(run: Run, outcome: str, note: str = "") -> None:
+    """Close a run. 'succeeded' needs every step done; 'failed' needs a failed step."""
+    if run.data.get("outcome"):
+        raise RunError(f"run already finished ({run.data['outcome']})")
+    if outcome not in OUTCOMES:
+        raise RunError(f"outcome must be one of {', '.join(OUTCOMES)}")
+    states = [v["state"] for v in run.data["steps"].values()]
+    if outcome == "succeeded" and any(x != "done" for x in states):
+        raise RunError("a run succeeds only when every step is done (skipped steps do not count)")
+    if outcome == "failed" and "failed" not in states:
+        raise RunError("a run can be finished as failed only when a step has failed; otherwise abandon it")
+    run.data["outcome"] = outcome
+    run.data["finished"] = _now()
+    if note.strip():
+        run.data["outcome_note"] = _clean(note.strip(), "note")
+    run.save()
+
+
+def effective_outcome(run: Run, now: datetime | None = None, stale_days: int = STALE_DAYS) -> str:
+    """finished outcome, or 'open', or 'abandoned' for an open run untouched for stale_days."""
+    if run.data.get("outcome"):
+        return run.data["outcome"]
+    now = now or datetime.now(timezone.utc)
+    last = datetime.fromisoformat(run.data.get("updated") or run.data["created"])
+    return "abandoned" if (now - last).days >= stale_days else "open"
+
+
+def scorecard(pack: Pack, now: datetime | None = None, stale_days: int = STALE_DAYS) -> dict:
+    """How often a runbook was tried and how it went, derived from its run records."""
+    card = {"tried": 0, "succeeded": 0, "failed": 0, "abandoned": 0, "open": 0,
+            "feedback": {"good": 0, "neutral": 0, "poor": 0}}
+    for rid in list_runs(pack):
+        r = load(pack, rid)
+        card["tried"] += 1
+        card[effective_outcome(r, now, stale_days)] += 1
+        fb = latest_feedback(r)
+        if fb:
+            card["feedback"][fb["rating"]] += 1
+    return card
+
+
+def apply_exercise(run: Run, today: date | None = None) -> str:
+    """Write 'exercised <date> by <operator>' into pack.yaml for a succeeded run. Returns the new status."""
+    if run.data.get("outcome") != "succeeded":
+        raise RunError("only a run finished as succeeded can exercise the pack")
+    _, new = exercise_update(run, today)
+    run.pack.path.write_text(new)
+    status = STATUS_LINE.search(new).group(0).split(": ", 1)[1]
+    run.pack.data["exercise_status"] = status
+    run.data["exercise_recorded"] = status
+    run.save()
+    return status