tegwick
cf682281d7
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a0e76f-be98-7ae3-965d-e0b31290a4c4
262 lines
13 KiB
Python
262 lines
13 KiB
Python
"""The labelled mutation catalogue — the project's measuring instrument.
|
||
|
||
The structural adaptation claims are measured against this catalogue, so its quality
|
||
caps the credibility of every downstream result. Six mutations, as the milestones
|
||
document originally sketched, cannot support any statement about precision or
|
||
recall; there are 38 here after TD-WP-0003-T03.
|
||
|
||
Each mutation carries a **ground-truth label** recorded before its measurement.
|
||
The original labels came with the use case; M25–M38 are explicitly agent-authored
|
||
synthetic extensions (not newly human-reviewed):
|
||
|
||
MECHANICAL the surface changed; protected semantics are identical.
|
||
test-driver should recover and report an adaptation.
|
||
SEMANTIC intended behaviour genuinely changed. test-driver must escalate
|
||
to a human and must never rewrite a claim by itself.
|
||
DEFECT the system violates unchanged intent. test-driver must report a
|
||
Product Finding and must never adapt to it.
|
||
|
||
The distinction between SEMANTIC and DEFECT is deliberately *not* inferable from
|
||
the code — both change behaviour. It is a statement about intent, which is why
|
||
claims must have independent provenance (D-06) and why ambiguity escalates
|
||
rather than resolving itself.
|
||
|
||
Mutations compose: `build_lab("M01", "M15")` applies both and records both in the
|
||
version string, so a mechanical change shipping alongside a defect is
|
||
reproducible. That combination is decision-table row 3 and the case a naive
|
||
self-healing tool gets wrong.
|
||
|
||
## The test-id axis
|
||
|
||
The UI carries stable `data-td` attributes, as a well-instrumented application
|
||
would. Most mechanical mutations preserve them, and a recorded selector sequence
|
||
keyed on those attributes survives such a mutation untouched — which would make
|
||
H-001 (semantic actions outlast recorded sequences) trivially *false*.
|
||
|
||
That is not a flaw to be rigged away. It is the honest shape of the question:
|
||
**a semantic action earns its keep exactly when stable identifiers are absent or
|
||
not carried forward.** Mutations therefore vary along `preserves_test_ids`, and
|
||
H-001 must be analysed split by that axis rather than as a single rate. A
|
||
catalogue whose mutations all broke naive selectors would flatter the thesis and
|
||
tell us nothing.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from dataclasses import dataclass
|
||
from typing import Callable, Literal
|
||
|
||
from .app import BASE_VERSION, LabApp, ObservationChannel, build_baseline
|
||
|
||
__all__ = [
|
||
"CATALOGUE", "BY_ID", "Mutation", "ObservationChannel", "build_lab",
|
||
"expected_classification",
|
||
]
|
||
|
||
Label = Literal["MECHANICAL", "SEMANTIC", "DEFECT"]
|
||
Layer = Literal["ui", "api", "domain"]
|
||
|
||
|
||
@dataclass(frozen=True, slots=True)
|
||
class Mutation:
|
||
id: str
|
||
title: str
|
||
label: Label
|
||
layer: Layer
|
||
rationale: str
|
||
apply: Callable[[LabApp], None]
|
||
preserves_test_ids: bool = True
|
||
|
||
|
||
def _m(app: LabApp, **flags) -> None:
|
||
for key, value in flags.items():
|
||
setattr(app, key, value)
|
||
|
||
|
||
# --- MECHANICAL -----------------------------------------------------------
|
||
# The surface moves. What the system means does not.
|
||
|
||
def _m01(app): _m(app, ui_share_control="modal")
|
||
def _m02(app): _m(app, ui_dom_style="nested", ui_test_ids="dropped")
|
||
def _m03(app): _m(app, response_id_field="id")
|
||
def _m04(app): _m(app, ui_labels="verbose")
|
||
def _m05(app): _m(app, ui_field_order="reversed")
|
||
def _m06(app): _m(app, api_path_style="short")
|
||
def _m07(app): _m(app, ui_button_element="anchor")
|
||
def _m08(app): _m(app, ui_confirm_revoke=True)
|
||
def _m09(app): _m(app, latency_seconds=0.05)
|
||
def _m10(app): _m(app, denied_status=401)
|
||
|
||
|
||
# --- SEMANTIC -------------------------------------------------------------
|
||
# Intended behaviour changed. A human must decide; test-driver must not.
|
||
|
||
def _m23(app): _m(app, ui_share_control="removed")
|
||
def _m24(app): _m(app, api_grant_path="share")
|
||
|
||
|
||
def _m21(app): _m(app, ui_labels="verbose", ui_test_ids="dropped")
|
||
def _m22(app): _m(app, ui_field_names="renamed", ui_test_ids="dropped")
|
||
|
||
|
||
def _m11(app): _m(app, require_share_acceptance=True)
|
||
def _m12(app): _m(app, revoke_delay_seconds=3600.0)
|
||
def _m13(app): _m(app, grant_permission_for=lambda p: p or "WRITE")
|
||
def _m14(app): _m(app, enforce_tenant_isolation=True, tenant_sharing_announced=True)
|
||
|
||
|
||
# --- DEFECT ---------------------------------------------------------------
|
||
# Unchanged intent, violated.
|
||
|
||
def _revoke_is_cosmetic(app: LabApp, user_id: str, resource_id: str) -> bool:
|
||
"""Record and audit say revoked; enforcement still allows the read."""
|
||
resource = app.resources.get(resource_id)
|
||
if resource is None:
|
||
return False
|
||
if resource["owner"] == user_id:
|
||
return True
|
||
return any(
|
||
r.event == "grant" and r.subject_id == user_id and r.resource_id == resource_id
|
||
for r in app.audit
|
||
)
|
||
|
||
|
||
def _read_permits_everyone(app: LabApp, user_id: str, resource_id: str) -> bool:
|
||
return resource_id in app.resources
|
||
|
||
|
||
def _ignores_tenant(app: LabApp, user_id: str, resource_id: str) -> bool:
|
||
return app.baseline_may_read(user_id, resource_id) or resource_id in app.resources
|
||
|
||
|
||
def _m15(app): _m(app, may_read=_revoke_is_cosmetic)
|
||
def _m16(app): _m(app, grant_permission_for=lambda p: "WRITE")
|
||
def _m17(app): _m(app, may_read=_read_permits_everyone)
|
||
def _m18(app): _m(app, audit_revoke=False)
|
||
def _m19(app): _m(app, revoke_delay_seconds=2.0)
|
||
def _m20(app): _m(app, enforce_tenant_isolation=False, may_read=_ignores_tenant)
|
||
|
||
|
||
CATALOGUE: tuple[Mutation, ...] = (
|
||
Mutation("M01", "Sharing control moves into a modal", "MECHANICAL", "ui",
|
||
"Same action, different place. The canonical mechanical change.", _m01),
|
||
Mutation("M02", "DOM rewritten, test ids not carried forward", "MECHANICAL", "ui",
|
||
"Selectors break; nothing the user can do changes. Unlike M01 this "
|
||
"drops the stable test ids, which is what a real rewrite usually "
|
||
"does.", _m02, preserves_test_ids=False),
|
||
Mutation("M03", "API renames resource_id to id", "MECHANICAL", "api",
|
||
"Compatible representation change of the same field.", _m03),
|
||
Mutation("M04", "UI labels reworded", "MECHANICAL", "ui",
|
||
"'Share' becomes 'Give access'. Wording, not meaning.", _m04),
|
||
Mutation("M05", "Form field order reversed", "MECHANICAL", "ui",
|
||
"Permission precedes subject. Same form, same result.", _m05),
|
||
Mutation("M06", "API paths shortened", "MECHANICAL", "api",
|
||
"/resources/{id} becomes /r/{id}.", _m06),
|
||
Mutation("M07", "Buttons become anchors", "MECHANICAL", "ui",
|
||
"Element type changes; affordance does not.", _m07),
|
||
Mutation("M08", "Revoke gains a confirmation step", "MECHANICAL", "ui",
|
||
"An extra legitimate click. The postcondition is unchanged, so this "
|
||
"is mechanical — but it is the closest mechanical case to a semantic "
|
||
"one, and a good test of where the boundary really sits.", _m08),
|
||
Mutation("M09", "Responses are slower", "MECHANICAL", "api",
|
||
"Latency only. Tests patience, not semantics.", _m09),
|
||
Mutation("M10", "Denials return 401 instead of 403", "MECHANICAL", "api",
|
||
"Both mean refused. A driver keying on the exact code breaks.", _m10),
|
||
|
||
Mutation("M21", "Controls reworded and test ids dropped", "MECHANICAL", "ui",
|
||
"Two signals disturbed at once: the wording changed and the stable "
|
||
"identifiers are gone. Extends the deciding side of the test-id "
|
||
"axis, which was N=1 after T07's first run.", _m21,
|
||
preserves_test_ids=False),
|
||
Mutation("M22", "Form field names changed", "MECHANICAL", "ui",
|
||
"subject_id becomes recipient, test ids dropped. The API still means "
|
||
"the same thing. This is the case that probes whether discovery "
|
||
"generalises or merely pattern-matches known field names.", _m22,
|
||
preserves_test_ids=False),
|
||
|
||
Mutation("M24", "The grant endpoint is renamed", "MECHANICAL", "api",
|
||
"/grant becomes /share and the old path stops working. The agentic "
|
||
"arm recovers by reading the form; a frozen path cannot. This is the "
|
||
"reversibility case — a crystallized asset must fail loudly here, "
|
||
"not search.", _m24),
|
||
|
||
Mutation("M23", "Sharing control removed from the UI, API left open", "DEFECT", "ui",
|
||
"The surface-substitution trap. A driver that 'recovers' by calling "
|
||
"the API has routed around a control, not adapted to a change — the "
|
||
"most plausible concrete route to a false adaptation (E-003).", _m23),
|
||
|
||
Mutation("M11", "A share must be accepted before it takes effect", "SEMANTIC", "domain",
|
||
"Bob genuinely cannot read until he accepts. The old claim 'Bob can "
|
||
"read after the grant' is now wrong, and only a human may say so.", _m11),
|
||
Mutation("M12", "Revocation is deferred, not immediate", "SEMANTIC", "domain",
|
||
"A deliberate product decision that revocation batches. Looks exactly "
|
||
"like M19 from the outside; the difference is intent.", _m12),
|
||
Mutation("M13", "Grants default to WRITE", "SEMANTIC", "domain",
|
||
"Only affects grants that omit a permission. Inert for the reference "
|
||
"scenario, which passes READ explicitly — recorded as such rather "
|
||
"than quietly dropped.", _m13),
|
||
Mutation("M14", "Cross-tenant sharing is announced as prohibited", "SEMANTIC", "domain",
|
||
"Codifies existing enforcement as intended behaviour. A change of "
|
||
"intent with no change of code — the case that shows classification "
|
||
"cannot be a diff.", _m14),
|
||
|
||
Mutation("M15", "Revocation updates the record but not enforcement", "DEFECT", "domain",
|
||
"Audit trail and stored grant both say revoked; the read still "
|
||
"succeeds. The defect that looks correct to an operator.", _m15),
|
||
Mutation("M16", "A READ grant confers WRITE", "DEFECT", "domain",
|
||
"Privilege escalation through the ordinary sharing path.", _m16),
|
||
Mutation("M17", "Any authenticated user can read any resource", "DEFECT", "domain",
|
||
"Authorization effectively removed. Carol reads R.", _m17),
|
||
Mutation("M18", "Revocation is not audited", "DEFECT", "domain",
|
||
"Enforcement is correct; the evidence trail is not. Detectable only "
|
||
"because audit is observed, not assumed.", _m18),
|
||
Mutation("M19", "Revocation propagates after a delay", "DEFECT", "domain",
|
||
"Not a decision — a race. Indistinguishable from M12 by behaviour "
|
||
"alone, which is exactly the point.", _m19),
|
||
Mutation("M20", "Tenant isolation leaks", "DEFECT", "domain",
|
||
"A user of another tenant reads the resource.", _m20),
|
||
)
|
||
|
||
# T03 extension: labels fixed from the synthetic contract before execution.
|
||
# These are agent-authored instrument variants, not newly human-reviewed labels.
|
||
# Paired variants isolate identifier loss from the structural mechanism.
|
||
EXTENDED_LAYOUTS = (
|
||
("M25", "M32", "Reordered fields", {"ui_field_order": "reversed"}),
|
||
("M26", "M33", "Relocated into an open dialog", {"ui_share_control": "modal"}),
|
||
("M27", "M34", "Anchor affordances", {"ui_button_element": "anchor"}),
|
||
("M28", "M35", "Fields grouped in a fieldset", {"ui_form_layout": "fieldset"}),
|
||
("M29", "M36", "Subject input becomes a textarea", {"ui_form_layout": "textarea"}),
|
||
("M30", "M37", "Unrelated form precedes the control", {"ui_form_layout": "decoy"}),
|
||
("M31", "M38", "Control moved after revoke into a sidebar", {"ui_form_layout": "sidebar"}),
|
||
)
|
||
for dropped_id, preserved_id, title, flags in EXTENDED_LAYOUTS:
|
||
for mutation_id, preserved in ((dropped_id, False), (preserved_id, True)):
|
||
CATALOGUE += (Mutation(
|
||
mutation_id, title + ("; ids preserved" if preserved else "; ids dropped"),
|
||
"MECHANICAL", "ui",
|
||
"Agent-authored synthetic variant: the same explicit grant form and "
|
||
"endpoint remain available; stored permissions and enforcement are unchanged.",
|
||
lambda app, flags=flags, preserved=preserved: _m(
|
||
app, **flags, ui_test_ids="stable" if preserved else "dropped"
|
||
),
|
||
preserves_test_ids=preserved,
|
||
),)
|
||
|
||
|
||
BY_ID: dict[str, Mutation] = {m.id: m for m in CATALOGUE}
|
||
|
||
|
||
def expected_classification(mutation_id: str) -> Label:
|
||
"""Pre-recorded instrument labels, never inferred from run outcomes."""
|
||
return BY_ID[mutation_id].label
|
||
|
||
|
||
def build_lab(*mutation_ids: str) -> tuple[LabApp, dict[str, str]]:
|
||
"""Build a lab with the named mutations applied, versioned by what it carries."""
|
||
app, tokens = build_baseline()
|
||
for mutation_id in mutation_ids:
|
||
BY_ID[mutation_id].apply(app)
|
||
app.applied_mutations = tuple(mutation_ids)
|
||
suffix = "+".join(mutation_ids) if mutation_ids else "baseline"
|
||
app.version = f"{BASE_VERSION}-{suffix}"
|
||
return app, tokens
|