Add automation status surface

src/activity_core/rules/executor.py

@@ -41,6 +41,7 @@ class InstructionResult:
     review_required: bool = False
     condition_matched: str | None = None
     validation_error: str | None = None
+    llm_response_metadata: dict[str, Any] | None = None
 
 
 def _resolve_path(obj: Any, path: str) -> Any:
@@ -167,12 +168,14 @@ def _execute(
 
     # Step 3 — call LLM
     raw_output = llm_client.complete(rendered, model=instr.model, config=llm_config)
+    response_metadata = _llm_response_metadata(llm_client)
 
     # Step 4 — validate and optionally retry
     task_specs, report, error = _validate_output(raw_output, instr, allow_list)
     if error:
         retry_prompt = rendered + f"\n\nPrevious output was invalid: {error}\nPlease fix."
         raw_output = llm_client.complete(retry_prompt, model=instr.model, config=llm_config)
+        response_metadata = _llm_response_metadata(llm_client)
         task_specs, report, error = _validate_output(raw_output, instr, allow_list)
         if error:
             # Truncate to keep log volume bounded but long enough to see the
@@ -188,10 +191,13 @@ def _execute(
             # loss. One bad item should cost one item, not the whole report.
             recovered = _resilient_report(
                 instr, raw_output, error, prompt_hash, allow_list,
+                response_metadata=response_metadata,
             )
             if recovered is not None:
                 return recovered
-            failure_report = _invalid_output_report(instr, error, raw_output)
+            failure_report = _invalid_output_report(
+                instr, error, raw_output, response_metadata=response_metadata,
+            )
             if failure_report is not None:
                 return InstructionResult(
                     tasks=[],
@@ -202,6 +208,7 @@ def _execute(
                     review_required=True,
                     condition_matched=instr.condition or None,
                     validation_error=error,
+                    llm_response_metadata=response_metadata,
                 )
             return _empty_result(instr, prompt_hash=prompt_hash, validation_error=error)
 
@@ -213,6 +220,7 @@ def _execute(
         output_validated=True,
         review_required=bool(getattr(instr, "review_required", False)),
         condition_matched=instr.condition or None,
+        llm_response_metadata=response_metadata,
     )
 
 
@@ -252,6 +260,7 @@ def _invalid_output_report(
     instr: Any,
     validation_error: str,
     raw_output: Any,
+    response_metadata: dict[str, Any] | None = None,
 ) -> dict[str, Any] | None:
     """Build a durable diagnostic report for invalid report-sink output.
 
@@ -269,7 +278,7 @@ def _invalid_output_report(
             partial_output = _parse_json_output(raw_output)
         except json.JSONDecodeError:
             partial_output = None
-            raw_preview = raw_output[:4000]
+            raw_preview = raw_output[:_RAW_OUTPUT_PREVIEW_LIMIT]
     else:
         partial_output = raw_output
 
@@ -281,6 +290,8 @@ def _invalid_output_report(
         "status": "validation_failed",
         "validation_error": validation_error,
     }
+    if response_metadata:
+        report["llm_response_metadata"] = response_metadata
     if isinstance(partial_output, dict):
         if isinstance(partial_output.get("summary"), str):
             report["partial_summary"] = partial_output["summary"]
@@ -310,9 +321,43 @@ _SNIPPET_LIMIT = 200
 # fail the whole report or flow unbounded into a downstream consumer.
 _MAX_STRING_LEN = 4000
 _MAX_DEPTH = 8
+_RAW_OUTPUT_PREVIEW_LIMIT = 12000
 _SUMMARY_RE = re.compile(r'"summary"\s*:\s*"((?:[^"\\]|\\.)*)"')
 
 
+_SAFE_RESPONSE_METADATA_KEYS = {
+    "finish_reason",
+    "usage",
+    "model",
+    "model_name",
+    "provider",
+    "request_id",
+    "response_id",
+    "trace_id",
+    "latency_ms",
+    "duration_ms",
+    "elapsed_ms",
+    "created",
+    "created_at",
+}
+
+
+def _llm_response_metadata(llm_client: Any) -> dict[str, Any] | None:
+    metadata = getattr(llm_client, "last_response_metadata", None)
+    if not isinstance(metadata, dict) or not metadata:
+        return None
+    safe: dict[str, Any] = {}
+    for key, value in metadata.items():
+        if key not in _SAFE_RESPONSE_METADATA_KEYS:
+            continue
+        try:
+            json.dumps(value)
+        except (TypeError, ValueError):
+            continue
+        safe[str(key)] = value
+    return safe or None
+
+
 def _snippet(value: Any) -> str:
     text = value if isinstance(value, str) else json.dumps(value, default=str)
     return text[:_SNIPPET_LIMIT]
@@ -561,6 +606,7 @@ def _resilient_report(
     original_error: str,
     prompt_hash: str | None,
     allow_list: set[str] | None = None,
+    response_metadata: dict[str, Any] | None = None,
 ) -> InstructionResult | None:
     """Recover a partial-but-usable report from output that failed validation.
 
@@ -590,6 +636,8 @@ def _resilient_report(
         "quarantined_items": quarantined[:_QUARANTINE_LIMIT],
         "recovery_note": f"original validation error: {original_error}",
     }
+    if response_metadata:
+        report["llm_response_metadata"] = response_metadata
     logger.warning(
         "instruction_output_recovered: instruction=%r, kept=%d, quarantined=%d",
         getattr(instr, "id", None), len(valid), len(quarantined),
@@ -603,6 +651,7 @@ def _resilient_report(
         review_required=True,
         condition_matched=getattr(instr, "condition", "") or None,
         validation_error=None,
+        llm_response_metadata=response_metadata,
     )