Enforce bounded operation guardrails
Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a028de-e2c8-7732-8521-46a7fc5db82f
This commit is contained in:
parent
c384f60530
commit
26934e25b9
51 changed files with 1843 additions and 472 deletions
|
|
@ -38,12 +38,20 @@ class InstructionResult:
|
|||
prompt_hash: str | None = None
|
||||
model: str | None = None
|
||||
output_validated: bool = False
|
||||
review_required: bool = False
|
||||
review_advisory: bool = False
|
||||
condition_matched: str | None = None
|
||||
validation_error: str | None = None
|
||||
llm_response_metadata: dict[str, Any] | None = None
|
||||
|
||||
|
||||
def _review_advisory(instr: Any) -> bool:
|
||||
"""Read the renamed advisory flag while accepting legacy caller objects."""
|
||||
legacy = getattr(instr, "review_required", None)
|
||||
if legacy is not None:
|
||||
return bool(legacy)
|
||||
return bool(getattr(instr, "review_advisory", False))
|
||||
|
||||
|
||||
def _resolve_path(obj: Any, path: str) -> Any:
|
||||
"""Walk a dot-separated path on obj or dict. Returns None if not found."""
|
||||
parts = path.split(".")
|
||||
|
|
@ -134,7 +142,7 @@ def execute_instruction_with_audit(
|
|||
prompt_hash=None,
|
||||
model=getattr(instr, "model", None),
|
||||
output_validated=False,
|
||||
review_required=True,
|
||||
review_advisory=True,
|
||||
condition_matched=getattr(instr, "condition", "") or None,
|
||||
validation_error=str(exc),
|
||||
)
|
||||
|
|
@ -149,7 +157,7 @@ def execute_instruction_with_audit(
|
|||
prompt_hash=None,
|
||||
model=getattr(instr, "model", None),
|
||||
output_validated=False,
|
||||
review_required=True,
|
||||
review_advisory=True,
|
||||
condition_matched=getattr(instr, "condition", "") or None,
|
||||
validation_error=str(exc),
|
||||
)
|
||||
|
|
@ -200,13 +208,10 @@ def _execute(
|
|||
response_metadata = _llm_response_metadata(llm_client)
|
||||
task_specs, report, error = _validate_output(raw_output, instr, allow_list)
|
||||
if error:
|
||||
# Truncate to keep log volume bounded but long enough to see the
|
||||
# actual JSON shape mismatch (typical reports are <2KB).
|
||||
preview = (raw_output or "")[:2000]
|
||||
logger.warning(
|
||||
"instruction_output_error: instruction=%r, prompt_hash=%s, "
|
||||
"error=%s, raw_output_preview=%r",
|
||||
instr.id, prompt_hash, error, preview,
|
||||
"error=%s",
|
||||
instr.id, prompt_hash, error,
|
||||
)
|
||||
# Posture B (WP-0016-T03): try to recover a partial-but-usable
|
||||
# report from individually-parseable items before declaring total
|
||||
|
|
@ -227,7 +232,7 @@ def _execute(
|
|||
prompt_hash=prompt_hash,
|
||||
model=instr.model,
|
||||
output_validated=False,
|
||||
review_required=True,
|
||||
review_advisory=True,
|
||||
condition_matched=instr.condition or None,
|
||||
validation_error=error,
|
||||
llm_response_metadata=response_metadata,
|
||||
|
|
@ -240,7 +245,7 @@ def _execute(
|
|||
prompt_hash=prompt_hash,
|
||||
model=instr.model,
|
||||
output_validated=True,
|
||||
review_required=bool(getattr(instr, "review_required", False)),
|
||||
review_advisory=_review_advisory(instr),
|
||||
condition_matched=instr.condition or None,
|
||||
llm_response_metadata=response_metadata,
|
||||
)
|
||||
|
|
@ -272,7 +277,7 @@ def _empty_result(
|
|||
prompt_hash=prompt_hash,
|
||||
model=getattr(instr, "model", None),
|
||||
output_validated=False,
|
||||
review_required=bool(getattr(instr, "review_required", False)),
|
||||
review_advisory=_review_advisory(instr),
|
||||
condition_matched=getattr(instr, "condition", "") or None,
|
||||
validation_error=validation_error,
|
||||
)
|
||||
|
|
@ -281,47 +286,28 @@ def _empty_result(
|
|||
def _invalid_output_report(
|
||||
instr: Any,
|
||||
validation_error: str,
|
||||
raw_output: Any,
|
||||
_raw_output: Any,
|
||||
response_metadata: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Build a durable diagnostic report for invalid report-sink output.
|
||||
|
||||
Task-only instructions keep the legacy empty-result behavior. Instructions
|
||||
with report sinks should leave operators a bounded artifact that preserves
|
||||
the partial model output without marking it as schema-valid.
|
||||
with report sinks leave a bounded diagnostic without retaining unvalidated
|
||||
provider output.
|
||||
"""
|
||||
if not getattr(instr, "report_sinks", None):
|
||||
return None
|
||||
|
||||
partial_output: Any
|
||||
raw_preview: str | None = None
|
||||
if isinstance(raw_output, str):
|
||||
try:
|
||||
partial_output = _parse_json_output(raw_output)
|
||||
except json.JSONDecodeError:
|
||||
partial_output = None
|
||||
raw_preview = raw_output[:_RAW_OUTPUT_PREVIEW_LIMIT]
|
||||
else:
|
||||
partial_output = raw_output
|
||||
|
||||
report: dict[str, Any] = {
|
||||
"summary": (
|
||||
f"Instruction {instr.id} produced output that failed validation; "
|
||||
"partial output was preserved for operator review."
|
||||
"unvalidated provider output was discarded."
|
||||
),
|
||||
"status": "validation_failed",
|
||||
"validation_error": validation_error,
|
||||
}
|
||||
if response_metadata:
|
||||
report["llm_response_metadata"] = response_metadata
|
||||
if isinstance(partial_output, dict):
|
||||
if isinstance(partial_output.get("summary"), str):
|
||||
report["partial_summary"] = partial_output["summary"]
|
||||
report["partial_report"] = partial_output
|
||||
elif isinstance(partial_output, list):
|
||||
report["partial_report"] = partial_output
|
||||
elif raw_preview is not None:
|
||||
report["raw_output_preview"] = raw_preview
|
||||
return report
|
||||
|
||||
|
||||
|
|
@ -343,7 +329,6 @@ _SNIPPET_LIMIT = 200
|
|||
# fail the whole report or flow unbounded into a downstream consumer.
|
||||
_MAX_STRING_LEN = 4000
|
||||
_MAX_DEPTH = 8
|
||||
_RAW_OUTPUT_PREVIEW_LIMIT = 12000
|
||||
_SUMMARY_RE = re.compile(r'"summary"\s*:\s*"((?:[^"\\]|\\.)*)"')
|
||||
|
||||
|
||||
|
|
@ -670,7 +655,7 @@ def _resilient_report(
|
|||
prompt_hash=prompt_hash,
|
||||
model=getattr(instr, "model", None),
|
||||
output_validated=True,
|
||||
review_required=True,
|
||||
review_advisory=True,
|
||||
condition_matched=getattr(instr, "condition", "") or None,
|
||||
validation_error=None,
|
||||
llm_response_metadata=response_metadata,
|
||||
|
|
@ -745,7 +730,7 @@ def _deterministic_context_report(instr: Any, context: dict) -> InstructionResul
|
|||
prompt_hash=None,
|
||||
model=getattr(instr, "model", None),
|
||||
output_validated=True,
|
||||
review_required=bool(getattr(instr, "review_required", False)),
|
||||
review_advisory=_review_advisory(instr),
|
||||
condition_matched=getattr(instr, "condition", "") or None,
|
||||
)
|
||||
|
||||
|
|
@ -806,7 +791,7 @@ def _sbom_catchup_report(instr: Any, catchup: dict) -> InstructionResult:
|
|||
prompt_hash=None,
|
||||
model=getattr(instr, "model", None),
|
||||
output_validated=True,
|
||||
review_required=bool(getattr(instr, "review_required", False)),
|
||||
review_advisory=_review_advisory(instr),
|
||||
condition_matched=getattr(instr, "condition", "") or None,
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue