test-driver/lab/mutations.py

"""The labelled mutation catalogue — the project's measuring instrument.

Every claim test-driver makes is measured against this catalogue, so its quality
caps the credibility of every downstream result. Six mutations, as the milestones
document originally sketched, cannot support any statement about precision or
recall; there are twenty here.

Each mutation carries a **ground-truth label**, decided by a human from the use
case and recorded before any run:

    MECHANICAL  the surface changed; protected semantics are identical.
                test-driver should recover and report an adaptation.
    SEMANTIC    intended behaviour genuinely changed. test-driver must escalate
                to a human and must never rewrite a claim by itself.
    DEFECT      the system violates unchanged intent. test-driver must report a
                Product Finding and must never adapt to it.

The distinction between SEMANTIC and DEFECT is deliberately *not* inferable from
the code — both change behaviour. It is a statement about intent, which is why
claims must have independent provenance (D-06) and why ambiguity escalates
rather than resolving itself.

Mutations compose: `build_lab("M01", "M15")` applies both and records both in the
version string, so a mechanical change shipping alongside a defect is
reproducible. That combination is decision-table row 3 and the case a naive
self-healing tool gets wrong.

## The test-id axis

The UI carries stable `data-td` attributes, as a well-instrumented application
would. Most mechanical mutations preserve them, and a recorded selector sequence
keyed on those attributes survives such a mutation untouched — which would make
H-001 (semantic actions outlast recorded sequences) trivially *false*.

That is not a flaw to be rigged away. It is the honest shape of the question:
**a semantic action earns its keep exactly when stable identifiers are absent or
not carried forward.** Mutations therefore vary along `preserves_test_ids`, and
H-001 must be analysed split by that axis rather than as a single rate. A
catalogue whose mutations all broke naive selectors would flatter the thesis and
tell us nothing.
"""

from __future__ import annotations

from dataclasses import dataclass
from typing import Callable, Literal

from .app import BASE_VERSION, LabApp, ObservationChannel, build_baseline

__all__ = [
    "CATALOGUE", "BY_ID", "Mutation", "ObservationChannel", "build_lab",
    "expected_classification",
]

Label = Literal["MECHANICAL", "SEMANTIC", "DEFECT"]
Layer = Literal["ui", "api", "domain"]


@dataclass(frozen=True, slots=True)
class Mutation:
    id: str
    title: str
    label: Label
    layer: Layer
    rationale: str
    apply: Callable[[LabApp], None]
    preserves_test_ids: bool = True


def _m(app: LabApp, **flags) -> None:
    for key, value in flags.items():
        setattr(app, key, value)


# --- MECHANICAL -----------------------------------------------------------
# The surface moves. What the system means does not.

def _m01(app): _m(app, ui_share_control="modal")
def _m02(app): _m(app, ui_dom_style="nested", ui_test_ids="dropped")
def _m03(app): _m(app, response_id_field="id")
def _m04(app): _m(app, ui_labels="verbose")
def _m05(app): _m(app, ui_field_order="reversed")
def _m06(app): _m(app, api_path_style="short")
def _m07(app): _m(app, ui_button_element="anchor")
def _m08(app): _m(app, ui_confirm_revoke=True)
def _m09(app): _m(app, latency_seconds=0.05)
def _m10(app): _m(app, denied_status=401)


# --- SEMANTIC -------------------------------------------------------------
# Intended behaviour changed. A human must decide; test-driver must not.

def _m11(app): _m(app, require_share_acceptance=True)
def _m12(app): _m(app, revoke_delay_seconds=3600.0)
def _m13(app): _m(app, grant_permission_for=lambda p: p or "WRITE")
def _m14(app): _m(app, enforce_tenant_isolation=True, tenant_sharing_announced=True)


# --- DEFECT ---------------------------------------------------------------
# Unchanged intent, violated.

def _revoke_is_cosmetic(app: LabApp, user_id: str, resource_id: str) -> bool:
    """Record and audit say revoked; enforcement still allows the read."""
    resource = app.resources.get(resource_id)
    if resource is None:
        return False
    if resource["owner"] == user_id:
        return True
    return any(
        r.event == "grant" and r.subject_id == user_id and r.resource_id == resource_id
        for r in app.audit
    )


def _read_permits_everyone(app: LabApp, user_id: str, resource_id: str) -> bool:
    return resource_id in app.resources


def _ignores_tenant(app: LabApp, user_id: str, resource_id: str) -> bool:
    return app.baseline_may_read(user_id, resource_id) or resource_id in app.resources


def _m15(app): _m(app, may_read=_revoke_is_cosmetic)
def _m16(app): _m(app, grant_permission_for=lambda p: "WRITE")
def _m17(app): _m(app, may_read=_read_permits_everyone)
def _m18(app): _m(app, audit_revoke=False)
def _m19(app): _m(app, revoke_delay_seconds=2.0)
def _m20(app): _m(app, enforce_tenant_isolation=False, may_read=_ignores_tenant)


CATALOGUE: tuple[Mutation, ...] = (
    Mutation("M01", "Sharing control moves into a modal", "MECHANICAL", "ui",
             "Same action, different place. The canonical mechanical change.", _m01),
    Mutation("M02", "DOM rewritten, test ids not carried forward", "MECHANICAL", "ui",
             "Selectors break; nothing the user can do changes. Unlike M01 this "
             "drops the stable test ids, which is what a real rewrite usually "
             "does.", _m02, preserves_test_ids=False),
    Mutation("M03", "API renames resource_id to id", "MECHANICAL", "api",
             "Compatible representation change of the same field.", _m03),
    Mutation("M04", "UI labels reworded", "MECHANICAL", "ui",
             "'Share' becomes 'Give access'. Wording, not meaning.", _m04),
    Mutation("M05", "Form field order reversed", "MECHANICAL", "ui",
             "Permission precedes subject. Same form, same result.", _m05),
    Mutation("M06", "API paths shortened", "MECHANICAL", "api",
             "/resources/{id} becomes /r/{id}.", _m06),
    Mutation("M07", "Buttons become anchors", "MECHANICAL", "ui",
             "Element type changes; affordance does not.", _m07),
    Mutation("M08", "Revoke gains a confirmation step", "MECHANICAL", "ui",
             "An extra legitimate click. The postcondition is unchanged, so this "
             "is mechanical — but it is the closest mechanical case to a semantic "
             "one, and a good test of where the boundary really sits.", _m08),
    Mutation("M09", "Responses are slower", "MECHANICAL", "api",
             "Latency only. Tests patience, not semantics.", _m09),
    Mutation("M10", "Denials return 401 instead of 403", "MECHANICAL", "api",
             "Both mean refused. A driver keying on the exact code breaks.", _m10),

    Mutation("M11", "A share must be accepted before it takes effect", "SEMANTIC", "domain",
             "Bob genuinely cannot read until he accepts. The old claim 'Bob can "
             "read after the grant' is now wrong, and only a human may say so.", _m11),
    Mutation("M12", "Revocation is deferred, not immediate", "SEMANTIC", "domain",
             "A deliberate product decision that revocation batches. Looks exactly "
             "like M19 from the outside; the difference is intent.", _m12),
    Mutation("M13", "Grants default to WRITE", "SEMANTIC", "domain",
             "Only affects grants that omit a permission. Inert for the reference "
             "scenario, which passes READ explicitly — recorded as such rather "
             "than quietly dropped.", _m13),
    Mutation("M14", "Cross-tenant sharing is announced as prohibited", "SEMANTIC", "domain",
             "Codifies existing enforcement as intended behaviour. A change of "
             "intent with no change of code — the case that shows classification "
             "cannot be a diff.", _m14),

    Mutation("M15", "Revocation updates the record but not enforcement", "DEFECT", "domain",
             "Audit trail and stored grant both say revoked; the read still "
             "succeeds. The defect that looks correct to an operator.", _m15),
    Mutation("M16", "A READ grant confers WRITE", "DEFECT", "domain",
             "Privilege escalation through the ordinary sharing path.", _m16),
    Mutation("M17", "Any authenticated user can read any resource", "DEFECT", "domain",
             "Authorization effectively removed. Carol reads R.", _m17),
    Mutation("M18", "Revocation is not audited", "DEFECT", "domain",
             "Enforcement is correct; the evidence trail is not. Detectable only "
             "because audit is observed, not assumed.", _m18),
    Mutation("M19", "Revocation propagates after a delay", "DEFECT", "domain",
             "Not a decision — a race. Indistinguishable from M12 by behaviour "
             "alone, which is exactly the point.", _m19),
    Mutation("M20", "Tenant isolation leaks", "DEFECT", "domain",
             "A user of another tenant reads the resource.", _m20),
)

BY_ID: dict[str, Mutation] = {m.id: m for m in CATALOGUE}


def expected_classification(mutation_id: str) -> Label:
    """Ground truth. Recorded by a human before any run — never inferred."""
    return BY_ID[mutation_id].label


def build_lab(*mutation_ids: str) -> tuple[LabApp, dict[str, str]]:
    """Build a lab with the named mutations applied, versioned by what it carries."""
    app, tokens = build_baseline()
    for mutation_id in mutation_ids:
        BY_ID[mutation_id].apply(app)
    app.applied_mutations = tuple(mutation_ids)
    suffix = "+".join(mutation_ids) if mutation_ids else "baseline"
    app.version = f"{BASE_VERSION}-{suffix}"
    return app, tokens