From 84848e9a0eaeeb36eb1c84f290f431725ff78abc Mon Sep 17 00:00:00 2001 From: tegwick Date: Sun, 23 Aug 2026 00:02:58 +0200 Subject: [PATCH] T08: the classifier, measured and attacked False Adaptation Rate = 0/7 across the labelled catalogue and the three E-003 attacks. 11 of 12 mechanical mutations absorbed without a human, so the safety result is not bought by escalating everything. - classification.py: total function over three signals, rule order chosen so every rule that could excuse a regression sits after the rule that reports one. SAFE_TO_ACCEPT is a two-element closed set, asserted. - CompositeDriver plus scenarios/full_journey.py: one asset crossing both surfaces, so UI mutations are visible as surface differences while the claims they do not touch stay green. - E-003: surface substitution (new M23), concurrent mechanical+defect, evidence starvation, provenance laundering. All held. F-0006 (CONCEPT_DRIFT, resolved): the T02 design listed SEMANTIC_CHANGE as an outcome the table could produce. It cannot - M12 and M19 are behaviourally identical, as the lab has asserted since T05. PRODUCT_DEFECT and SEMANTIC_CHANGE collapse into one escalating outcome, BEHAVIOUR_CHANGED, and the distinction becomes a human adjudication. INTENT_CHANGED survives but is detected by the claim fingerprint moving, not inferred from behaviour. Two classifier defects found and fixed rather than reported: claims downstream of a failed realization now yield INCONCLUSIVE rather than FAIL (a false accusation is the mirror image of a false adaptation), and the browser driver records a page signature so surface change is detectable when the interaction path is unchanged. Co-Authored-By: Claude Opus 5 Assistant: claude-code Assistant-Model: opus Assistant-Process: 1629012@bnt-lap001 Assistant-Session: 78d4fb13-8a1e-474b-87a3-9b9261c49a39 --- WORK-RECORDS.md | 2 +- docs/TestDriverClassificationDesign.md | 12 +- lab/__pycache__/http_api.cpython-312.pyc | Bin 11777 -> 11904 bytes lab/__pycache__/mutations.cpython-312.pyc | Bin 13835 -> 14293 bytes lab/http_api.py | 6 +- lab/mutations.py | 8 + research/concepts/fitness-map.md | 8 +- .../experiments/E-001-mechanical-vs-defect.md | 3 +- .../E-003-surface-substitution-attack.md | 4 +- .../F-0006-classifier-cannot-infer-intent.md | 93 +++++++ .../hypotheses/H-004-independent-judgment.md | 37 ++- .../__pycache__/full_journey.cpython-312.pyc | Bin 0 -> 5134 bytes scenarios/full_journey.py | 98 +++++++ .../__pycache__/browser.cpython-312.pyc | Bin 8516 -> 10497 bytes .../classification.cpython-312.pyc | Bin 0 -> 11620 bytes .../__pycache__/drivers.cpython-312.pyc | Bin 4076 -> 5762 bytes .../__pycache__/runner.cpython-312.pyc | Bin 9536 -> 9974 bytes src/testdriver/browser.py | 30 ++- src/testdriver/classification.py | 245 ++++++++++++++++++ src/testdriver/drivers.py | 30 +++ src/testdriver/runner.py | 25 +- ...lassification.cpython-312-pytest-7.4.4.pyc | Bin 0 -> 22288 bytes ..._ground_truth.cpython-312-pytest-7.4.4.pyc | Bin 23264 -> 25643 bytes tests/test_classification.py | 195 ++++++++++++++ tests/test_lab_ground_truth.py | 52 +++- ...-WP-0002-vertical-spike-crystallization.md | 2 +- 26 files changed, 824 insertions(+), 26 deletions(-) create mode 100644 research/findings/F-0006-classifier-cannot-infer-intent.md create mode 100644 scenarios/__pycache__/full_journey.cpython-312.pyc create mode 100644 scenarios/full_journey.py create mode 100644 src/testdriver/__pycache__/classification.cpython-312.pyc create mode 100644 src/testdriver/classification.py create mode 100644 tests/__pycache__/test_classification.cpython-312-pytest-7.4.4.pyc create mode 100644 tests/test_classification.py diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 65aa4e3..9985b2b 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -19,7 +19,7 @@ | task | TD-WP-0002-T04 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T05 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T06 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | -| task | TD-WP-0002-T07 | todo | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | +| task | TD-WP-0002-T07 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T08 | todo | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T09 | todo | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T10 | todo | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | diff --git a/docs/TestDriverClassificationDesign.md b/docs/TestDriverClassificationDesign.md index d65d619..08bebee 100644 --- a/docs/TestDriverClassificationDesign.md +++ b/docs/TestDriverClassificationDesign.md @@ -66,11 +66,21 @@ to the last accepted run of the same verification asset. | yes | yes | unchanged | `MECHANICAL_ADAPTATION` | | no | yes | changed to FAIL | `PRODUCT_DEFECT` | | yes | yes | changed to FAIL | `PRODUCT_DEFECT` | -| any | yes | unchanged, but the asset's declared claim set differs from the use case | `SEMANTIC_CHANGE` → human | +| any | yes | unchanged, but the asset's declared claim set differs from the use case | ~~`SEMANTIC_CHANGE` → human~~ → **`INTENT_CHANGED`**, see below | | any | no, and no legitimate surface affords it | — | `PRODUCT_DEFECT` | | any | no, but the action is expressible and the actor simply failed | — | `FRAMEWORK_LIMITATION` | | any | any | any oracle `INCONCLUSIVE`, or required evidence missing | `AMBIGUOUS` → escalate | +> **Revised at T08 — see `research/findings/F-0006-classifier-cannot-infer-intent.md`.** +> `SEMANTIC_CHANGE` was specified above as an outcome the table could produce. It +> cannot: a deliberate product decision and a defect are behaviourally identical +> (M12 and M19 in the lab), so no evidence separates them. `PRODUCT_DEFECT` and +> `SEMANTIC_CHANGE` are collapsed into one escalating outcome, +> **`BEHAVIOUR_CHANGED`**, and which of the two it is becomes a human +> adjudication. `INTENT_CHANGED` remains, but is detected by the *claim +> fingerprint* moving — a fact about the recorded use case, not an inference +> about behaviour. The row above described that, filed under the wrong heading. + Two rows carry the whole safety argument: - **Row 3** — a surface change occurring *alongside* a verdict change is classified diff --git a/lab/__pycache__/http_api.cpython-312.pyc b/lab/__pycache__/http_api.cpython-312.pyc index 91d5fde0611d2f7459c914531573a8833fbb0086..153b125d56306b5a73d89a791c6a8cd0cc31825b 100644 GIT binary patch delta 1320 zcmZuwOKclO81}5=^((gHCUH_Hb{q#AZynn-ZAB@eO$$YZN-e>O1fo{hW;UrEd)NNg z4vJZmw&KwuqyTe4v@Yd>5C>j+M5yYS1GEwo{g3${ zGZ)UBF9p8|1bhx$2gly5G$y|Y-V=pgPBZ8}N!(5v?F7^$J0FYW>DXGv;ebIpY=~DJ zh6g|Q6$c$R`VeEBqj4jEIMMZY0!*HaL?BKRJB%j{S0%)5CI;CJ&wxMK^{RyWIzc+c z?zwj#ePD-e%5Y(@5j$|w2%?C1U__54-A} zI-0c;7wNPSMlRQNcEU7oI4aSdG58Mk1%n_yWZUlTx{D5J=1X5tc-#$2O6ir$10r}(U4jl&9p z)CWcUS%OxF1-2ADMBX*uk1i6=>fcKC$Jhz-o;f$ZC6EUDE*>Fu_H%sNZ}14498?yc z7(ehou_4CQEgs0#JH^?8DhzOTj6;)Y6FG9->`qi%Prb+)fkOo0E_u29Ho;8;H|=0e zuC8cMQ)uF>MmOLkBqgEkD}bVFEm&6cwom}Lz`ZZBXHs+IRrXG5@9=h#mb0uj;W@5< zp4~~M7G)GIS5=d#4sK4*aU%3|@${)Dlx2-7#{o0|-b70T>|METS7LV*4szOA$DX~0 zlH{?ggE!cGdK`zKrf10K=7;HkKo;0%ncbURH0pJmz*5h_9>5mos~kiQH4c0zU^`qx zNy4+Y*#$1ymjCMez^-xItL?*F6Gq~crYe%q3q-E56Vjuzi`>1$!M+-NF1-O>;?h+P z1rCE8_#1=yf1u57NRN7)z;e~`9X8<@=K?mjgqKJ`)qe`T>zS=-v{h9O!`Enx*N&xg PPl_iA5r03!Ah-1wbAL_a delta 1150 zcmZvaUu;u#6vumR+uN?C-74GIpUxJ>I_}zT{E_&Nj6cyRLI$Cm5(U}Vy(8P&-g3GX z7#9RJ7&QTsW8%YB;l&UVU(8K-fdn2*OcbLr@#V(w>;n)@cp`y#&aXP1@h10k&+mKw z{C>Z4?zzL~55>Qa#Ueg_zU%m8W;FeE{Hd-z)ZBPz5A?1!+Z=t-=Y;tUE&9wZC&26$ z%}hEmW)EEr_cTAdA}U39!~M~8y{Qal2-pfy+&0^J8`zx(+f>0 z=%a&8@5C}6YpnD;EzAvcr>S?RF}uO%L){#360BwG{xb3{AaLU6P{#f3nN|%3=}GG@ zI7K_!?!y7fw{M2C?s)rljTcutdSQgV>Bz$S?(ZEpVM~b*fXxgqgx2wSt2&2=M3T1R zjKE?T0X)DTPqQmnpnF{}!9};Tdkn%;e>tg^d>uY>ZznHlP@#RPcCcwGm5ZL13TFi+ zs;81$|ED(gh71sl8g=dp+_qIL_a^e~l&B{!L#3GO~3iY91hJciTLmw^`*UM$2* z6U8YTOO}~FYMZC9#-gDuM266++VePRRTs1&#B(g;dHS($2Yf&+nN4lWhxGi@lV z>_fJUu(};ypr0~H9>RmnX86iY_s29S(4f)z-Vz5jx()blBS8(|qUf^%T~HLrP*5e* z*)nv*Wwjudiu}hPKy}CEtc3`lHSMHF?6PHOjY3eTZ;h9?jY_&2llnDeE{y?xB-Wx} zP_SB%5bXaOH5wjx5%$x`fnS&-*)73q?D~N&WXIrB`Zara>+%5__Y61s6qN#M)XJ|W z|F1|Y=hj5Olo-{CCAysJgK@f%t3=h-vI%}coARx1UuV+`OyIGa=Pyni_4GM=)+$%= zxafM-!kUF%u!1t?oub8*c7*>Li(UV@+2ioGl5Nfxtzo=HSMq!H*8ucKqx47q(w}@m BAVL5D diff --git a/lab/__pycache__/mutations.cpython-312.pyc b/lab/__pycache__/mutations.cpython-312.pyc index ef915da8a64b97ebf8b4a728f00f859722f44dfd..50c2c1b00d9a20c2f405d2bb5f99597236e99073 100644 GIT binary patch delta 2266 zcmah}YfM~46ux(trRkI9v0YkT15{yQq5I%j5J71HEm~U8mX^n5&nztNgU-E6TLhO1 z_Ti(COds))*rqXQ(AY3v_=w)T%dJu_Q$ua&sT{pQ^7eCM0l znKSpD@Q<;IZ`QA0qu{Tp;O$9QwW8b?FU!B;RG>auI$Lo;ojZGLyr5)J87zZ0fGlPV zmg888;}#st7BhIeN)k|sf@+IG?cjncs0KIG(46a;DJRL-!dA(9XfAkf_^z)?%lV-W zbM??b^Gr33&?IY`sl&u?gBHoR(v6Z&pqHHF8EBJ6&r+wUpdC77!E-d<#6J(6lHX1@ znfM*hC4IW-W)t57FG&7HT9D2sU?=oSqnBvW6aVgI*d=p)wD^hKZrB6;xLX6XFx?Pd zfmdNK?1MpCV&eD10m%L4)fOJGpU7;+8n%r;v^&%ezAD>TCXQx1)J}deJGx9fwC7suSPw7F`CB8X zde+N7%Q={?YG8f5EjOM{HL?NzU2eKpRukKcQxAygwHV zW8h=sXrS@)8+EeMEM700L=o^xXMAOmk3~^L`RC4CYLL(7dsdeE*>{#n@~g%1-3NupB|tYTSsdSI zHsaAY)V79W0WGYzVW36EKxu|aqZ zPnSAuS1>%DyjJSYO!)jWzC9Bf3q{9CAQm-PEKDBvISH~@gcuW)4D`53$F3d{ra^ z5rSJAPKbHM6vBQ&(+P{k4GKiVF^a?Tn%pEBGl&M-l*k$}k(tn<NQtVg%uzb>6uKM^XZ#>Z=*bV`m0(u1W z3g{CcKULzJ#r6vr79c(*^8U*^EbpkigYtGUahsTIc-dp}j75_((TK9SWb??rU{@4A z=WAJW7gH)e`Y^xkJ8t#=k(uoB>#DQ+PUYSHy$>vw4Lj9)y?u{XS#5jNhij6_x^-%9 iucccpx6H4;ySC(!(ygkN2mSnqdVl8E*1WFDjK2U7yB{(D delta 1857 zcmai#Urbw79LMh|v_@pEqqIPwP|A>9TA}|-83Se!*aXGNoYr++>F=Fa{}X zjZ(0Ik;4 z1R*4Ax~N&h!_Y1HL24=Cc_Z||A=#*x)|4cS&CQqo(1@a6kK*Ljz zmVAb)1DeDvoRf(RZQtROmZ%dLbwRcVH=jBOlxs>-H-+EWUPW0}mXzrz-z;lVJ75l7 zIpKnfa2^j?CoI7{j_wN!Ngct8Wo2Gbmy3q-!Ahd?QKOG^^Ht-UhAymOVVzuY%diu`Pt_f=_J0XD+#Rb@(DK{m!unsTKm#K!rrrjlOfiW7MA%^?(3$A(1QCuD0X7-b2HojFnOm&emy z4~t_M<2Njco~^3;Pv)lLlT$PW@tH)LX8)&wmnE^mDITcF+IPE)$HOMkl;A(q_$yI$p1E)gE=NVT7CO7Y6ab|M^5Ab^}wZ{0sZ!?#tN4ZldWD4>ZSG z&x%gFZeW<*!u(~v(VVC`FP!TrI}afi2;Y-Vhgtt2hK1r0hs#i0QoUt*W-1S}= zxmS$1za?aY8@&6;Pu%JK=&)RQtEkQkk$TM<3hwM#iBxDLd3zBhgTj>dHCe9V&WVzxz}=e<@Ch@~ diff --git a/lab/http_api.py b/lab/http_api.py index 52b7222..dfb3b21 100644 --- a/lab/http_api.py +++ b/lab/http_api.py @@ -73,7 +73,11 @@ def _render(app: LabApp, user_id: str, resource_id: str) -> str: f'<{tag}{href}{role} id="share-submit" data-td="share-submit">' f"{share_label}" ) - if app.ui_share_control == "modal": + if app.ui_share_control == "removed": + # The control is gone from the UI. The API endpoint is still open — the + # trap for a driver that "recovers" by routing around the interface. + share_form = '

Sharing is unavailable.

' + elif app.ui_share_control == "modal": share_form = ( f'<{tag}{href}{role} id="open-share" data-td="open-share">{share_label}…' f"" diff --git a/lab/mutations.py b/lab/mutations.py index 0ea3e83..4016304 100644 --- a/lab/mutations.py +++ b/lab/mutations.py @@ -90,6 +90,9 @@ def _m10(app): _m(app, denied_status=401) # --- SEMANTIC ------------------------------------------------------------- # Intended behaviour changed. A human must decide; test-driver must not. +def _m23(app): _m(app, ui_share_control="removed") + + def _m21(app): _m(app, ui_labels="verbose", ui_test_ids="dropped") def _m22(app): _m(app, ui_field_names="renamed", ui_test_ids="dropped") @@ -169,6 +172,11 @@ CATALOGUE: tuple[Mutation, ...] = ( "generalises or merely pattern-matches known field names.", _m22, preserves_test_ids=False), + Mutation("M23", "Sharing control removed from the UI, API left open", "DEFECT", "ui", + "The surface-substitution trap. A driver that 'recovers' by calling " + "the API has routed around a control, not adapted to a change — the " + "most plausible concrete route to a false adaptation (E-003).", _m23), + Mutation("M11", "A share must be accepted before it takes effect", "SEMANTIC", "domain", "Bob genuinely cannot read until he accepts. The old claim 'Bob can " "read after the grant' is now wrong, and only a human may say so.", _m11), diff --git a/research/concepts/fitness-map.md b/research/concepts/fitness-map.md index 03c9b78..67e13ec 100644 --- a/research/concepts/fitness-map.md +++ b/research/concepts/fitness-map.md @@ -1,6 +1,6 @@ # Concept ↔ Implementation Fitness Map -**Updated:** 2026-08-22 (TD-WP-0002-T07) +**Updated:** 2026-08-22 (TD-WP-0002-T08) Traces each important concept to the implementation, experiment and evidence that support it. **Unsupported entries are the point of this map** — a concept with no @@ -31,11 +31,11 @@ were aspirational, not evidenced. | `C-use-case` | C1 | `intent.py` | — | — | Is a use case expressible without leaking mechanics? | | `C-actor-isolation` | C1 | `world.py` | E-001 | `td://self/actor-isolation` | **F-0003** — only observable when the scenario plants canaries. | | `C-semantic-action` | **C2** | `actions.py`, `agentic.py` | E-001 (partial) | T07 arm comparison | **F-0005** — supported only where stable identifiers are absent. Narrower than the concept model claims. | -| `C-oracle-independence` | C1 | `runner.py`, `oracles.py` | E-001, E-003 | — | Independence of components ≠ independence of belief. (H-004) | +| `C-oracle-independence` | **C2** | `runner.py`, `oracles.py` | E-001, E-003 | — | Independence of components ≠ independence of belief. (H-004) | | `C-evidence-pack` | C1 | `evidence.py` | — | `td://self/evidence-reproducibility` | Verdicts are reproducible from S3 alone, on passing and failing runs. | | `C-observation-channel` | C1 | `lab/app.py` | — | — | **D-07** — required of every system under test. Adoption cost unknown. | -| `C-adaptation` | C1 | — (T08) | E-001 | — | (H-002) | -| `C-classification` | C1 | — (T08) | E-001, E-003 | — | Decision table is total on paper; unexercised. | +| `C-adaptation` | **C2** | `classification.py`, `agentic.py` | E-001 | T08 matrix | 11/12 mechanical absorbed without a human. | +| `C-classification` | **C2** | `classification.py` | E-001, E-003 | T08 matrix, FAR 0/7 | **F-0006** — cannot infer SEMANTIC vs DEFECT; collapses to one escalating outcome. | | `C-crystallization` | C1 | — (T09) | E-002 | — | (H-003) | | `C-intent-provenance` | C1 | `provenance.py` | E-003 | `td://self/intent-independence` | Constrains provenance, not quality. Accepted residual. | | `C-lineage` | C0 | — | — | — | Parent pointer only in the spike. | diff --git a/research/experiments/E-001-mechanical-vs-defect.md b/research/experiments/E-001-mechanical-vs-defect.md index 3360d1c..58619af 100644 --- a/research/experiments/E-001-mechanical-vs-defect.md +++ b/research/experiments/E-001-mechanical-vs-defect.md @@ -35,4 +35,5 @@ tokens · wall time · retries. ## Status -`PLANNED`. Blocked on T05 (lab) and T07 (agentic driver). +`EXECUTED` 2026-08-22 (T08). Arm A run over 23 mutations; arm B over +the 12 mechanical ones. Results in H-001 and H-004. FAR 0/7. diff --git a/research/experiments/E-003-surface-substitution-attack.md b/research/experiments/E-003-surface-substitution-attack.md index c3677d9..1711a19 100644 --- a/research/experiments/E-003-surface-substitution-attack.md +++ b/research/experiments/E-003-surface-substitution-attack.md @@ -37,4 +37,6 @@ what E-001 reports. ## Status -`PLANNED`. Blocked on T05, T07, T08. +`EXECUTED` 2026-08-22 (T08). All four attacks held — see +`tests/test_classification.py::test_attack_*`. M23 was added to the catalogue +to make attack 1 real rather than hypothetical. diff --git a/research/findings/F-0006-classifier-cannot-infer-intent.md b/research/findings/F-0006-classifier-cannot-infer-intent.md new file mode 100644 index 0000000..b270f1a --- /dev/null +++ b/research/findings/F-0006-classifier-cannot-infer-intent.md @@ -0,0 +1,93 @@ +--- +id: F-0006 +type: framework-finding +class: CONCEPT_DRIFT +status: resolved +discovered: "2026-08-22" +resolved: "2026-08-22" +discovered_by: TD-WP-0002-T08 +workplan: TD-WP-0002 +task: TD-WP-0002-T08 +hypotheses: [H-004] +--- + +# F-0006 — The classifier cannot infer a semantic change, only escalate + +## What the design said + +`docs/TestDriverClassificationDesign.md` (T02) specifies a decision table with +four outcomes, one of which is `SEMANTIC_CHANGE`: + +> | any | yes | unchanged, but the asset's declared claim set differs from the use +> case | `SEMANTIC_CHANGE` → human | + +`TestDriverInitialMilestones.md` M6 states the same four-way path: +`IMPLEMENTATION CHANGE` / `INTENT CHANGE` / `PRODUCT DEFECT` / `AMBIGUOUS`. + +Both read as though a classifier could look at a run and determine that intended +behaviour had changed. + +## What building it showed + +It cannot, and the lab already contained the proof before the classifier existed. + +**M12** (revocation deferred by product decision, `SEMANTIC`) and **M19** +(revocation propagates after a delay, `DEFECT`) produce *identical* evidence: +same failing assertion, same step, same snapshot, same audit trail. +`test_deferred_revoke_and_revoke_race_are_behaviourally_identical` has asserted +this since T05. + +No quantity of evidence distinguishes them, because the difference is not in the +system. It is in whether someone decided the new behaviour was wanted. A +classifier that emitted `SEMANTIC_CHANGE` from evidence would be guessing, and +guessing in the one direction the project cannot afford: `SEMANTIC_CHANGE` +sounds benign, and "the team must have meant it" is exactly the reasoning that +normalizes a defect. + +## Resolution + +Path 2 — the concept is deliberately revised. + +The classifier's output space is now: + +| Outcome | Safe to accept | Meaning | +|---|---|---| +| `UNCHANGED` | yes | no observable difference | +| `MECHANICAL_ADAPTATION` | yes | surface moved, every claim still holds | +| `BEHAVIOUR_CHANGED` | **no** | a claim that held no longer holds | +| `INTENT_CHANGED` | **no** | the recorded claim set itself moved | +| `REALIZATION_FAILED` | **no** | the action could not be performed | +| `AMBIGUOUS` | **no** | the evidence does not support a conclusion | + +`BEHAVIOUR_CHANGED` replaces both `PRODUCT_DEFECT` and `SEMANTIC_CHANGE`. +Which of the two it is remains a real and important question — it is simply a +question for a human, recorded as an adjudication, not inferred from a run. + +**`INTENT_CHANGED` is detectable**, but note what makes it so: the *claim +fingerprint* changed, meaning a human already edited what is being asserted. +That is a fact about the recorded use case, not an inference about behaviour. +This is what the T02 table's fourth row was really describing; it was filed under +the wrong heading. + +## Why this makes the framework better, not weaker + +Calling a semantic change a defect is a **false alarm**: a human looks, says "we +meant that", and updates the claim — which then registers as `INTENT_CHANGED` +with full provenance. Cost: one review. + +Calling a defect a semantic change is a **false adaptation**: the thesis dies. + +Collapsing the two into one escalating outcome makes the framework err only in +the direction it can afford. The measured consequence is visible in the T08 +matrix: 2 of 4 `SEMANTIC` mutations are escalated as `BEHAVIOUR_CHANGED`, which +looks like imprecision and is in fact the design working. + +## Consequences + +- `docs/TestDriverClassificationDesign.md` decision table updated in place, with + the original row preserved and marked. +- `TestDriverInitialMilestones.md` M6's four-way path is superseded by this + finding; the milestone's *intent* (do not normalize defects) is unchanged and + is met. +- H-004's falsification condition is unaffected: False Adaptation Rate over + `DEFECT`-labelled mutations. Measured **0/7** at T08. diff --git a/research/hypotheses/H-004-independent-judgment.md b/research/hypotheses/H-004-independent-judgment.md index e4814a1..bac9938 100644 --- a/research/hypotheses/H-004-independent-judgment.md +++ b/research/hypotheses/H-004-independent-judgment.md @@ -1,7 +1,7 @@ --- id: H-004 title: Independent Judgment -status: PROPOSED +status: EXPERIMENTING created: "2026-08-22" experiments: [E-001, E-003] concepts: [C-oracle-independence, C-intent-provenance] @@ -41,6 +41,41 @@ The most plausible route to falsification is not a misclassification but via another surface and scoring as a successful recovery. D-05 exists to close that route; E-003 exists to attack it deliberately. +## Result (TD-WP-0002-T08) + +**False Adaptation Rate = 0/7.** No `DEFECT`-labelled mutation was classified as +safe to accept, including the three deliberate attacks in E-003. + +| Ground truth | Accepted without a human | +|---|---| +| MECHANICAL (12) | 11 | +| SEMANTIC (4) | 2 — both genuinely inert for this scenario | +| DEFECT (7) | **0** | + +E-003 attacks, all held: + +- **surface substitution** (M23, UI control removed, API left open) → `AMBIGUOUS`. + The driver did not route around; discovery failed loudly. +- **concurrent mechanical + defect** (M01+M15, M02+M17, M21+M20) → + `BEHAVIOUR_CHANGED`, with the reason explicitly noting that the coincident + surface change does not excuse it. +- **evidence starvation** → `AMBIGUOUS`, never a pass. +- **provenance laundering** → rejected at authoring, and caught in the record. + +Note what this does and does not establish. FAR = 0 follows largely from +*architecture* — claims are run inputs with no adaptation write path (D-02), and +`SAFE_TO_ACCEPT` is a two-element closed set. The experiment confirms the +architecture behaves as designed over 23 mutations; it does not establish that +the architecture is correct for mutations nobody thought of. That distinction +should survive into any external claim. + +The other side of the trade is asserted too: a classifier that escalated +everything would score a perfect FAR and be useless. +`test_mechanical_changes_are_mostly_absorbed` pins 11 of 12. + ## Status log - 2026-08-22 `PROPOSED`. No evidence. +- 2026-08-22 `EXPERIMENTING`. FAR 0/7 over the labelled set plus E-003 attacks. + Not promoted to SUPPORTED: 23 hand-written mutations is a small, self-chosen + sample. diff --git a/scenarios/__pycache__/full_journey.cpython-312.pyc b/scenarios/__pycache__/full_journey.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..65e17f80c0b44a136c677eb86bab6f08b7588136 GIT binary patch literal 5134 zcmbtYU2GKB6~41GyF2^m-Sr>-!FVtvunt}ajDR>PiNPi&1UuL!G}~00*`2X>%*>8; z?_CVex>~7G18LP*YEp=jDpr+;M0O?mR37@!RDD3|%Q_~)j6g{fDm8DUCQ5|*&~s;Y zyta!PslAB1cka)*=bU@b_nqs%Mj}B3MQ?dyvcCqQFWHA%c&EXwmZ5PODJX>$PVwlR z;YoQo*5-BI5K@BSO?eGp%4hgfenU)&Mj#b1f~lYpN`;JYDr`hj5qrL%*BH@M)NXtA zm=RCK?Y2*^HR@7zcH6Jl8x5%jqcPQJe~bDSqbb#7G^d*F?||N7w5D1)FGLcx1&;@zv79PlbzH@2KC4QT zW&!8b3wxw2HVM)46B3;^B~rlSa#khDU~n`iW7eIMr>4|Ak+i%-K|%(b(?rG6>EnB( z=^O}?WNAvp#LP?6WXKW=xnM1Dl13oGq*Gh2u$S zEGJXR(9TO5u{*IU>l&e2R?5l*q~{fo#|CAcn6RZxgNA%oB~n(GLGf{|BHK`9&|c2y zYA>jC`nY5isI#k4AXLhz(m9Q2Ff7R+OX{qc+8I=|an~u+ISs~{nIK#F(#I?I;@w$Pnx3U!7Nz^Nx@f0R?UNh zP0|hKX4Q1Y%%rn2HudDxg;$c&a9+Pqb;pcj$dpWTlA|t>z9_u~-q1iL$8o{woC(TL znOdGo<~W#0R6}MvN^+KYKNw&4rky_um!>>Ri~`@iz;3_nLa{PYI)C?SXW^4 zSieTHCd|DsQUL7?75XAaoLlAW3LVj-rZHs_jjEL{k5%jJVQYOf2NtNh7047cT}kV5 zCgHKfk%7_S(<4U*Eb;W{K>BFk=m1Cx!eF|h94}94mcMd?6M_|SmRCXnH-L9=9jof0 z8p;Y#yQLNyJVM-RJ=B*Gtpp-M3zZqOg2MHqAMZnG)>{-NVb3s@Tl7x)-LLAe zD@*YdxuUlj1~hJwU#Q4(gP%jc*e@U&aMvk95f#y-4Q_6q^IH@ay$cm9W_^lJ@#_Fw zvwjZIxI0r3DYMNkNkm9mlt(ToV4;~G| z8716a_g+-BaBWxG-}5Na;&xZkY=A!IuA*ITtGISnMR79e_Evw_RtM=`w^!*Xipq|~ zovwFhLq%UPPz)ABw8x#}>NV~iMTtj-61?TL!OY?T%`C4CXO@?mXVpAJVknDM8v+5y zX9Ck*m9T?2^k|C3lLECuP^AGH2tyVDP?chKfnc%sj7+n+nJCG2jTdy?T|6UJq;x;v z9<#&@;IOXcRm?6G-U`)Bfb8vpjRB)@3w&Eq=Oc|(0t}c{XMzl;i3;5mKqTb$r$Hxw z*344??d)2#g9N4!fCRV70uE9Iu87Vdu-`K@K5(^Ubz za7TJaFuP2+lU3~EIu66^=3;hHzt{LfY!L%9Fn&ild2Ffk+3N?EI!~1l?q=%)q^FA| zAZJx8;`~s;L=TyLB531;YqMOHE1 zw1i1WL995Wjk*T+&P~IJL#^zhV#Zw-59y!fKQc0WW^`Z#GjC%btD>xe1RY@(VVE5* zeu`CLRx#R4ghA1%Da&U=3vuE;UBJ5K$4*XxJ7Ah6-ISoSfXnJ zz!u9<2-`qfSDbsu$PBYeg8$@a5Yi=dI~4u-fl~iUEb&joc~5Z{g%vUUrg%wwt6`2_ z8GAqVPU_v4ulfJfcwP8-Yx&^Njrj1V;tTh9Hur8gKKG;f>{arISAYNN_4*s(q0+$L zh0eRtEi0j|x8uz#wJleA-aqutq4}dXcRsPS^NHow-cMQwuT7R)j+bN4-HJEP72eA& z$9H|y@Zt0y#pOg_IdZCh#IdG***?8)0ZI&W<|1+#XopH-374zjL!mtu!)S;JLX_gd?s z!TX`+n6ET={{X6KyIa>j&(FWOT({@;mYplHrYn!m@A{W*^ zuQw397e?{M`%$sRS2}jT7RB0bv~^$Qm)rJV&D}h3Z0W$U<><4e!Q0`s^7eh#epr5C z>_+%>X<#K3n|t)i?)ibm0}F@Bp`NnPV+VK^&Sb5yiwpOITOESN>*#gE*n92`o&y@bh%WLMg>hc-z9ueIgw3*4#^Zhgg!u~3!qD^DSyifL zlv1WZrD=`kO!%F`s>3w)V+u&cr4l<;CwrZj1ce7o(q^?Gh}RmMVa`)X)a3~kGorJj zYJ*uUG65#J)qc2pZ}TS*w;=<)#Qze#46HPZW;qYK1lgdQnaz4Gc^{UE7Cn>9Rhx>! zEqdHc;J;<390~x!7r~t%hlTy7q6iH8SvEM%|JrjI+zs#;RK^qVH1R32Y5)_)bCgc?c6aaHcQE-l{M+06731%O zA2V9Bd}BGtBNQuQpHO<-1U`Wm4}h_Zsa4~kVVVIbqfe3K3RE$t^H#VrlGRP3CVcog z5Qb0K5}gq%P{qj8tUmycWY`Jr5n+f3h*T<-Y%sEfG$7oJ`+8?DpUUH=nisGcXJ8q5 z8>;XAfxauDyN&Jh&E>|&E`{KxzsL}T{Ho=mZ$*sX6q}aBrg`5@Y5$V6|JqntY+4qN z+!P0w#K9ZlbN4(#pkt-BVeTA6-)ZT8$S;E)}$3D$}*_f>D3&8_z3Zgz75lHBz7i0X^j z+ESfV*)DCTjP}df3XEfO2L4i{Qas2EI1Ux_i42Kt+9puj*-q-jk~8~IAkj5UrJ$>a zF@Zi7a3lp4a3#lm?Fn+iH%*A!@vXpfo^O#`eIp|7@jGbG9hCeG9sCR(g2vx`(Yb~> n|FW;C#NP_m&Al{tVmT<4{9gq7O8!+}lG}ej!^OE;`}F<|U}Y}M literal 0 HcmV?d00001 diff --git a/scenarios/full_journey.py b/scenarios/full_journey.py new file mode 100644 index 0000000..e87f53a --- /dev/null +++ b/scenarios/full_journey.py @@ -0,0 +1,98 @@ +"""The reference journey, crossing two surfaces. + +Sharing happens in the browser UI, where a person would do it. Setup and +revocation go through the API. That mix is the realistic case and it is also what +makes classification measurable: a UI mutation must be visible as a *surface* +difference while the claims it does not touch stay green. + +The use case, the claims and the oracles are identical to +`scenarios/alice_bob_carol.py`. Only the realization path differs — which is the +whole point of a semantic action. +""" + +from __future__ import annotations + +from testdriver import ( + Actor, Cast, DirectDriver, Oracle, Scenario, SemanticAction, StateObserver, + Step, VerificationAsset, World, +) +from testdriver.agentic import DiscoveryRuntime +from testdriver.browser import BrowserDriver +from testdriver.drivers import CompositeDriver +from lab.mutations import ObservationChannel, build_lab +from scenarios.alice_bob_carol import RESOURCE, USE_CASE +from scenarios.browser_grant import lab_server # re-exported for callers + +API = frozenset({"api"}) +BROWSER = frozenset({"browser"}) + +__all__ = ["build_journey", "lab_server", "RESOURCE", "USE_CASE"] + + +def build_journey(app, tokens, base_url, runtime=None): + cast = Cast() + for name in ("alice", "bob", "carol"): + cast.add(Actor(name, name.title(), credentials={"token": tokens[name]})) + world = World(id="w-journey", sut=app, sut_version=app.version, cast=cast) + + from testdriver.observers import Watch + + scenario = Scenario( + id="sc-full-journey", + use_case=USE_CASE, + variant=f"journey/{'+'.join(app.applied_mutations) or 'baseline'}", + watches=(Watch("bob", RESOURCE), Watch("carol", RESOURCE)), + steps=( + Step("s1-create", "alice", SemanticAction( + "create_resource", + {"resource_id": RESOURCE, "content": "the secret"}, + permitted_surfaces=API, + postcondition=lambda obs: "audit:R" in obs, + )), + Step("s2-grant", "alice", SemanticAction( + "grant_access", + {"subject_id": "bob", "permission": "READ"}, + permitted_surfaces=BROWSER, + postcondition=lambda obs: obs["state_permission:bob:R"] == "READ", + )), + Step("s3-revoke", "alice", SemanticAction( + "revoke_access", + {"resource_id": RESOURCE, "subject_id": "bob"}, + permitted_surfaces=API, + postcondition=lambda obs: obs["state_permission:bob:R"] is None, + )), + ), + ) + + driver = CompositeDriver( + { + "api": DirectDriver(app, tokens), + "browser": BrowserDriver( + base_url, tokens, runtime or DiscoveryRuntime(), RESOURCE + ), + }, + default="api", + ) + observer = StateObserver(ObservationChannel(app), scenario.watches) + asset = VerificationAsset(id="va-full-journey", scenario=scenario, maturity="T2") + return world, driver, observer, asset, Oracle() + + +def journey_lab_server(*mutations: str): + """Like lab_server but without pre-creating the resource — s1 does that.""" + import threading + from contextlib import contextmanager + from lab.http_api import serve + + @contextmanager + def _run(): + app, tokens = build_lab(*mutations) + server = serve(app) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield app, tokens, f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + + return _run() diff --git a/src/testdriver/__pycache__/browser.cpython-312.pyc b/src/testdriver/__pycache__/browser.cpython-312.pyc index 30939fe22664f600e912cf8f0681da0e51108ded..956759afbb8b271749008080c7031045a9130796 100644 GIT binary patch delta 2617 zcmbVOZ)_aJ6`$F?-P`-?KezYC7u&ZnN$jJ}VUh?(ksB~Lfh4GvnuyTE)LP%J&wJk6 zJI(H4OwXKyQk5VOMFTihQb;8vK%pX4q$+AvKD8iWde8gL z?9k#`4CekNKAX5M;oo=R_pUd)pTzhV*T;IljSo9?nVPU zeI2^S71Q)HIYxgO3;Lx10*!u(g&5Azh9J|8Sh7pH%C{{BKzcVwr$sGX3l#S-LHf8X z;{a9VI3FmEf>Mn4*8=oqIRo@QW;)b%XiMX^e83-E4%PVLxF;Vd=GwC-PtU}oLb&Fu zg=i^$BoxxRYracJ>#2q5uj3~MLRYz#J15aS$Wc0h>_X(^a*+O`_e7W9xu?VR!l@?) z@2&Z@;ANi{ss%5hpK$LBotX06caescgUs zszj$V$w4VdT0-;($(;klU}B=9l7jA#gN*Q(U}fZOGEF~BPV9P$8QaHThCzryfkA{p zhJiu*Qn`$z>2{tNvpTUDlYO5oFfbY9=|U}&(C;P!$5Q-9!lZO8#(x~aKztk%j-}}MjzfDrFQV+54>9Ng$a1Z9 zFI$T(KRt;&eRD@Hz^WI}Uf|8ej8i;mf=hc|ZD9wP(Ak zXCMocO1`WTTOUBA~oOvLsWKPZNifVhLMvSL*qS)%9VmP^AP=Rs8 z)JwYMfSIc0*vjluU2)WcQZcJ79ZN0gHnXRL>Jo@q4k?>C<%m_T7UsdM>L{vbiRg=E zqUoAqsC73Vq&h~~0yp)FQYH$p=T)l!;(FTyX9*m)=j-J~Td7t& zli;_y6@+HY&FMt9AfCUjyrwJ~&V0G*D6_g@6%@Oy%o^2f09h6X{Yrm{G1IJ#Yd zBzOi(>H6w_FM6j_R@sTD_^Vs7SI5wW`^i(UUCEIB&aR*Y;9$`o%L zfW--fp&)DMscwXpaShXRJ#zgGyb6{1Qm+p!p`Y=Kn4E2y;~~NCaW0n|bN%h$ZDvhT z$X=NK9hfNSmi|nI9Da7^_Nyj0VV3i%X&=sQiL8Q#EdzXx{(Oqq#HEw{@m%nw|>EaLGuWBYtfA5%X?fu4y7~FX{DwqWg1duP1@O%h8xmwQ_8OMp!cOM zf}Q6l!;?|$inCS6DO)a9!9&)!S>E+QecGS&F-oKy2 zdAc$FgeC-o32dh_5#!CNoRB#q!4e($}R|NqTA=I(`eCtM#} zTUCy(`^KWF`IlS<+F9!UqIt3Bp5sz@PDuHxZnWKoQq^zn34~r(hdk#{5<6|q^fBN6 ztB9;XbI0aQ1CZ{8d5@$%M-@ros3``V7aQ8!x=Yk-hpoax{gkF+ zx9wQwaJ#L+SN%*x-QXKf8RO!d6>ZLOfFn~C1y=L{==J*BB&)M|QXgl{A04sc|F*;p z)X1^=)PXfyS>{&w7h-eDJb#hHTt%5z#`Yro@Rve4>LGcCaf+IHGkdi#otv`?Ww;fH z$_i-+;7Onp-Ub3_5D%_~axKPfMVy-~0vlDg) zF&SnUSnry%JWH$sCPS={vgK)9%v)rbR;d<}WU@(y9K162NrbSx4Bw52qlKox2aoW( z2!?};upKPHXsAc_s$J@W^2>nP&>b`iKSOD0XMfi^ewKo0I2lp9Pp7%fw35+<&7;TL zGj3r^MYPy*iy0mA;bA)CYPeIY-|A{y{UO|^)n~4cTLGbC2Bl%PBUs&y$9a^4SmLBq z6SI~@-*zoAh)S@YI4CYq_XGHpSkuq4J|2c+3@0d39kAN{9xXtTz6t0C`@S-UN6B$L TgV5k_j`7a~-;Vs`5Iy?~hxNL= diff --git a/src/testdriver/__pycache__/classification.cpython-312.pyc b/src/testdriver/__pycache__/classification.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ea83da061588dc1e2da79d108f94a4eccc651799 GIT binary patch literal 11620 zcmcIqYj70Vb?%<&d5<)DAqgRgmUt{9B=K05$HKA*kU(Y`jFDK-8qc<-ThgF;XuEp= z$zrYYlB`&-160;QTef34NtL14#3X-$6IZCxqat8&pr2?bI&>V{Jf;Z&mmQ0K$Wl0#R7zACv|F2i>fzUBB<%QbQZzBO{KT#0Y3yjrfp zclD?`PUxDr#^#m)mlH<5J=vhG>f1Zh-*UwD>?AtpGly z00>dihA~4+Gb-4Y2Bdf*qhfNl1F1iuyxQC%{>Q`zVq6!8Bn(UYrHnq5R^$4psHd%t z0LI);D@w8$OGxpgCQ6uya1>9CWUw2|0MJuuT~rk*HY9F3+_rUlxZU5a6Fep)=~QAA z;9R6#563Z8dcZ<4+nfa^U@{jXjBC<>jSS77&giL3vY#OUNKdCOD5*G5PbZ-P59`*6 zubUWE#Gb9&MOo3}gFqC3x~Qg8gMR;;ayq7M=>e_}t8t*{kcG9FM94qP`nM#}*}BvF6bimmu^@_&cD)K!-Sh zeq~6y5Km`RDFM_Hk?8T5ln^%^-?p6&%YS^^9-9CYQa{aj0Tii+TZ#NqIN@YELkJuY z3pA!FBN8o7M583z3n=yD;B6A4T@De}bu6n%SX#ru7N8MhAkabVRvMJzDez1@rPDdF z%@c=#PH@V8+7EhKSRsNFhcZbig|qB7N0<|dEiG}ar3E|Jv2((6tJt5RbI0-*hm@2( zkWsM|(=!YoHWb*0--O_0+LKAa;s6dtQAbp?h)JMDYZrTndWc-b0ZbxF{WxegLRDgE zRmO%gVCa~nDXo4YbzmW$f`nk?YxXYSpxyspB)`JE5nz15C`^ll9fnVqbcrF;aDibH zGWrOeplq}`J5r;D=eRU70_b9v0AwFabn*-B_QWmp3X>5;eW+fIy+wL?qmI68vbZB39d*`IT($;g%dJc)ByIQ1}Q6{tdz14Wo5{2a|_71&-QE?0>f>A{M2GY zTR@z8o6IAidf9tPJq$WqVkBq8xSF)@L1#J(qD_p*8%|v%z*z>6V#Tj9BUO4fn zcJqv|YThBZyz@b>dgDYSFC2Tcx@|_N&I_j=ncXAmDtoRToRu50c}z(eE-jJPwIzq; zL_Z{c;Z5X=ht-Sw7k95i5oG5+w6K!EN-os73w54C$y+G-3MD@)mB<0yUarw#xYY2R zf}oQU+Sq{}(6>IMLZoOqxL9h1JSJfT9!rB&K;KI-1Q~B}OG;5>t=$m%)9Hi}(lRP| zK#7`Hpk`D+tS91QjEbXlktp(5fVzs~Vg!=FC?82{kcufePNFoLRCJ>P5>}4KbS`Mx`x{qSPxnk|4&nH~@I#I4^!cGQaR=DBR@6 zx${)JETPs?E{YpaYpF2HYAw}iSuL%!EMW$wfGw>?Em85ZB=g_%oZ`Zc$Zd~Wk2bY| zGSZpUwUmjJ5oM~)l&wd08{$Ay5(64-SX8T#8y<;p6Vt~+)=9Tp72Rmnjv<@i9<5zB zxAw*C+86W98|Rw$WSjTon>Nff?aDUoTCCX+p4;$hcEhXrhNiiOE!l=G`E@VMt=pSj zxA$?#TjrnVygol8^T=&bxT@$GH8oaBOTPhvY84@$&V}7-IaX9FD62$fc%o6 zG+HF>qEYn)>QhA-;~mF%87azl|##aE|u#A8o$7S(s*1kD%}xA>8Xyc&S=kxXvd*LonP)z zH=weZrb7>@kt(x*#PoN>L*b zv>}jw$oehEDkJ!@kc~=kzK6UV} zmlm9yyL6sPPuy;sA!(YGDdJ0LAyF;!t5G`4UE^bpI2UuA!$p6YABUAn_4qpJJK#V` zpwb^Z-mehQ?=pW1InPQLxv(o@cJ?2&BkDo4B=FO=Bb(szo!$3avYoyE8aVwiKY4QU?QBWoG(R2wVQ?nc zcI&OXt8b5IgL`KKr)PxIPw~+N44Ygf3QKT6G|9N)r;zD4xk1`*0T1VXiv2sr9p?UJ z=kP9^2LFB#HEjRqi!VM*L`inVIotBR>>TIMWA^0<6Tk>-QoILNM1rT1_hQ5?xDzftn;simXCyNE%UK z)rd%k3kDvH7v&N(AlRv*9#2B!C+m`IeyDG{a!Dt1B~2CwGpCrj4K0qWZnEuYvWw|V zLKc(KumS@Peh1c=NT-Kk!JEwm0d#BoF125!*Rr#u#Dff+IH&5I+F@5ZA zMXqYcq%ZF+xfZ+{q$b~uNWP|SYUhow-~7gnZ%mKoYPRL8*WOItNKPNkRkzJnAG^Ey zZyWx);ojGC+m1b~uAe&lgL5CAyR|#luw%C2%st=FN`6xE@o;YUSMYD<^qE}s*?j$m z9~}Si_^tL_{hoYP!_DJ2j?Xl2&sDwnxWrTK|260F1?GK#^hqrjEW7sd)t9d;)B1xz z>x|I)?2S!S&t%gW3O6~uAZ=EM3M~E3PX^EzGlXE>>wOqRftn7N6+XFyasK)96Z^;2s7- zgpm&hrh0yG=EE~H8+YGp&NaL;*KjD?a46T%nXB%~1&+)JM@&o=lq(D_NbwYE-}Suf zEymV(K}IcNaGbaI7jwMleb1v8r1Yd`fS36z-mBj81&N6jtaQiv9_H+lJrAU2GtU*x zTHP5a{w~O6&O^7)L>BfeAepwh7WEb5pvU?ahi0Z;Af>+bbOY8T2SVdZtBlj3;^PaI1*sOwDpd{myAimZGB<=;$GOJ2Ac5McEN# zFlnVY9vmWS4F6z)5Kh#y$cnY;FN}s8G>|M3>(reM?nlAUyp!_t1&H*6){wh>oT9QiDWg1#KExSWWJm(6DhY%q!@hc_d?0{01~4NhLYJ}$K8{Y zC$IHh?VYLIm@9eV)=Ljcwq}DJPZwwt-)vaRcl|$YjqyHCnA}w6&eSx)2Fb!%>eClK z=Rc)^MDEt64&3cnvZ);pO15N!2cD{Lp-<0}4Hej!+#iLrf=oB*oG9F)+JMcpuQBD(wt|A{Y{ z3wBNS{^*S#zVYYh9t68`zOH!}=PG^TKL{Rp;5)#2Ju0o3bbb=3cv#zd>rAe8=Yy9c_eXOtpUu{u&97<8 z*EQtVtbf?pgu}a+`PukS#{d592ThS&W8`7=rdzeS>g^A99=*RhxARoC`cz)rFx5R< z-+k-skI&scckh*4TlY-3`$@sH}<-YH<__e)Ae)eP0&-eq@wc;|bT3^AR zqkb9fOxZ9AoPZ_jHgyeC3^f)xP*>0}?qY~Wwxz9M2(|8C1~+Dd4U=c5gsF@9K*jZe z*+Bht^^fX)Sa<8}Jz=Kxm0Z*Qk2lUX9R*1TS50ode)770ec*w&;jt5=Exlv=OFWEC z#^5gI)HQ7s;&aqr!bcsYY>_=laszmiGff8~FwgxJei|uMP=HqC#)pBj52_b9Ed8iu z%a1qT-h8L+i3|TwRY6(MGtUVw&$BiQ(Iq25Kk}vI2UXA_)nY;$hrUp#cY??W1s3n!?gbk0NyLcSw#Sor0RV(i4eV|Al9Rmu%d5$AXB>S8C2(oIhv2vf@5wS;qdB z%|~w<%aXbUaMKgUvZTVVmf~gDIV-|XPaVsW>>d}u1<&rHc3DUKl|32mIXhWtX74jQ zse5+L=bT&pvpr?svU6Jj!ig_V1^#8Tt%S{$XCoD4S(2f$FVfVCxEGfV!Cx8v)WuQZ zlFTqJzn-*?XbC+2Q%nuqbnKZ%)ZisX^u`Ii_^a@(>15$7hNMs+01L8VBk+WRSQ*>f zWx4oR2oPLx@cglmhZytH!PiIO0}@_j*=s0t1PV47NnkaE=`c&G#qum#BmE@l5LpB# zmqt-8?P7;$d97@xU^*7ds0d;h6r}`WTr>oMV2OS;J&dpj<_%#H4c1JBg&AD%O2Uf< z=VKbqO1!kv*o$#}2oo$@h2m7Kpd!18&1iWhm(SJ@O?()$u=P!Eml;*X0zj}Y z9nm48l1gU=hZf^c2pBWcWOQsF-m4RMNMj*B2)?m={D79+o)m(CSgG?FnPN=}QCjxf z@z@!*q8++n;zCmpPs75xw8as&(zO>LvjCJr01JCD$07AA48sfnyr+}7!2+kAL6mtM z0LKdZ(h&Y&du&BH4NosYZIMD;H@LCQ3^0ne!DffxJr(h_A)*)^I#y8JP&y%ZjlDv9 zf0AekE?DQlvPZU^(7on)LDU_uhb)T7B0V@95k z=LZJw;*Grx6^7Ren9~daho)$@>LHqXwHG^}fT&URUG)At{Iu^wrc7{8%D7PD)X*(o zu5|mvk%ytWsom3ObD=F0N9VnqH#FyIp7k_83{`*d_FT=DY|WN@&9VEhFF5(K)BHTg z`%d$ZJf(A<^|PM!OFH%DYZ|A$3ob`l%RJ}swLp=k$?9i4^;W+f*_s`AkE=bv=R1zx z-$Y$|Fr~wXJjgmX&3c+1Ro2Z_w#-(xJa%&BXZZZq?WA*;d}0E$D_gTGzxwdUJs)?? z)EryBWGS0P%zDJ>&a5YFt;@#LoL=79gpZcA`t8ov?9SJ8-`_=>CCGR{#x_ec&UiL0 z+3%O`AEAD|tRHf7y2e>gqt$J1wq|d>=ETg&Gt{q-_3PuGxVd2E2m9wLH)kt1&$RXC zE9=l4tjB7udZr**4+Ni;TX9y4kqzuwb*zqsI~2Xr=*G})L;X1Vi9}1!x$k4M10xO)TV9s$PP>3f}G?jECl?c@k z6oizir2pZeW|ZQ|RI)}?($`di+KgWCnNduTi$#%GFHti=Zn73>xa?)bv{lJ|G`Wum zpIO)pucr6UyUz^t+OPf*TEP&Ub{pBeljr%zj%r@`^;V8w_bV>&3vScDbE|&At$pHN z#h>7>pPE|#grm~q?Oes0smNUYj%@voyQ#UIec7FTGi%<+m49`j^s$?B1g@{2k{|Hv z{>M?p<6RsXo}wM==jrd`_Tu(3YDZ?ZQ>m5bU*{LRm3$i&kD}PbyBCh~Zr<~xb}w&| G$o~Q^aLiW# literal 0 HcmV?d00001 diff --git a/src/testdriver/__pycache__/drivers.cpython-312.pyc b/src/testdriver/__pycache__/drivers.cpython-312.pyc index 6367df1bd014d84b15001e1a25b996d404139e40..c735df095819beeb517a9cb3c5697992c96689f9 100644 GIT binary patch delta 2287 zcmZ`*U1(HC6uy7^lQp|*lihB9FiuP~+0?|JCYFY@)R@*u)K@mZXuoLL!~K z{dXw)wa)q`P#C@$WV_(Y5TCVq-)jvY zCUBZW@+y)BB<)BVk)Rf9MzX^vxW%^t`Ak}iCRVk?Z`$#S+zziTaP9CGz!d6j!Ro|4 zBm{mu0AyY&`zW1v9tZxAceTK7T4k;9DXTestQ>TmDQa`B!VV#XHGBVrPJy#0!~OCh z?`pVn_aWf0H&IV1M;X=445Qr1GlQmOhRn3a`n;dRt@6N~XW@0F7GtrY^b=s$dmGzY zy3nR_)W}j}D#P<^TsJ5~Ls&O5G@VUwb^zD=y}{bH0dzIJP1sn8Ya5Cw&P=%mL^HNN zO*tDv`J>1P2oG2tZXEHxu5Ap8$hP0PS356T2XPx?!CnV47c(ux{g}IKF99;j_P}OY z8+L5UcEdCsVlNAHFY3RGC0TuD2C9Kr zwg5bHY%kGmqUTM36!{Zy`h;$kZ0-t~!zYgQkU`5fEqj7M9ElFtz;eW*YT>Mt&zWea zK?wzq3J`rdBf#7&bMh0}iUW{ZVmjcCZkr@)nIL^$H{4u7u&1e!)osfFSBw(OCcp)HZ03z%cMG`vYqa*7ExMUVo^y zRFsPfwv9>67kK_HN*RVHx#U(T+iIv%9#(7gp5JiYE>y2I)ahnHK#j9OI3yh|*KHP$-(@-b2w3GHcNplf)37L&)d}iW z`j~hyu#u*NTf1$|CKlT+AgWo$pkd|XO94}XGnewy>t`3cjWtJgklFFce`(QFYjEadDuj5G_E$i_C!)^`W8M|OExZzE{;A(?paChxiPnzJh<@Q zT0FTDZ(U36Sh~1)@do|o>}sm_{`THSA<%dlmf~bB*0(%UOa06iie)8TX}$j+eQuOK;#B#s2%0d^dTcm{(1{|vOF(3Y3?B<>+W z1dh=2(qknO2(LG&f&EV-HG#_UAQOWB3+Z~6^8SjA4v5$P7%GcYh=)XA{TF{DJTUE; j`5b+lje#(hGS7Ke;+f=QsZ)OA@3V5V9C;!E5l;RKl6Nhx delta 723 zcmZuvJ8Tm{5WV$%xH$G9Uu?&=z`58c-17fevScBC0!kyLpot>uxNIQAv7_0u6x<-8 zq)BIOLrF`C+(AW+L;)1&Dhf(ypg>B+%vmmCaniosdGmI5@6Eov_;bnmn9Zg&{J!T7 zpIPaVQ)2H%XWtvoUB2-4sG4F!Rn~Wzs}A)pE33cyp2=AoN-8qytfF2S{);m1Zojd> z!x^X+EECuWu>~R;Ko1-YxCslC$P*L@771zuB?2l06@pV!ph(Um9BALN`7cKPhw*Tt zBT0xuQ@6~6`!j`$$j{bXA&$VqAx(sREuK~*J{;u=(}jOTe!QLep;b_l!08tS#}TMf)WQ+=`*ZCY{zuB%`6GHZ=b&izScJmiL{md6>m z%E?sE(SdO`d@922jvT;d5DK7U;XEab0G>#=Oz|c4C{tgf|Ksz?jV3;m#c`^?8_%J% z(+hTlggd0aM+q~zFXV{0O@uD?!o)|s{q&nyT^oPR?6a*aBqdYeD#Fm&*b1XH8Nme) zVI9s;5u@Gj38>?epO|fB;`lY%f0y8YP4y|^s}D}YNjgn&o4Zg{gTi_6;Wfg7%R}lYRqqKM;wI5`@l5TS|tlg+WXa*S=3XHI@$o@guWjb$9qr3yY^t3Tf z!;xu%MTW;t*zk0ay~W$)_#L6V%Rm_p+BhCmuTbmC>@ta;UqO=QtDxX*u~iu^Sn`H3 z3x9dLa1~7P0xegP;v(;deqV@OT=e;EZ02}LjD-U4XY8{yK}_*cSn;)EH)-7lJNa(6 zLm#k&4EH>Ss4oO=zl(4b_+X_+MtN_ExdoT`X+dxiT)8LjE%79(yKPQ_`cU+5qQ(TC zGD|>U{yvSSsq25z$f)c^H~18F9;K+WJ|fE-JWP2)Fc)wHd4oRJ zX|d{~R+Pz>^v1Sw6_!}uvH6pkOC+n=MOyb+0R4&;8I_iaykH3WGs{=ukT{ABI1s3T zAH+C2xz%Aa1i;lDnA;i<^W1>$RX4ghi&d_ewUy=)5rz<%IQI^R(KNpPW;urDV6@gd zWWN2$IJ#^bLgP3^Vek+h;p zny!pV>8zw*NNbXoQnE=kos;CWEM;@Ll*)~cjVQXZQ_^w-q7ZmeSIXejNHRSt$%>Xz z({Cx7G@jI?tnxPDBr>TBirnR-9&j;p>V*cvXlk39*~~x6TyKtTuElHFVu!?p*#(_)&ZyE;pR~S$O?7 z3-oECSRJ0Te`3GcaBD|l$LHr)TYHvTd+zq$KeBkF7_3`II5vdI9z2czVd%!dzlipW zj~qm8{SC@Ax1OQU3wu`JGwB99U_cw>$0d-^yAJ#`a@Ph%9^tG!H~|IPmGgF?kU zYWrZY;{N`|!3gy*R-1@Z%XTrbmtKiD5;0~aexNN8Wr|UTB*nNZv70G&;>2#C>XUc| z#bx0i+&U30ZA1N)HuRB%~-HUQ#9g-LUE&cIKb? zzWL_y=j5G<=siga6Z++q%T_J99$l7%+ae*$!~tyZ+Qz2zD-(E+)CGsZa!5ldX5bT< z=R=NQ4d5oPLI|JAS^Ce!2l&f`)WbLvAA~SoiJL(&L>yx6$4pekZ{r#(I$>;RTKERz zYEp0nCAAw3MZg8&N%8ziw5%3pX$2CJ&DxyDa zo}3(}Glx7v=oz7%5|)_7tCSYZlTVdd<`kJ_PAU|uI0{~rQ)8boiyB9RG{DVd{F$qB zzGuDTd(Uk3UEjCpDb=t)uXP%mQeq!1^pX##p7A{==)>x&qgHJ{ZdEbZb{amQR29$5 zL(%FvKjpBFir=?oaVOq`?iSrEnP{6MlH?bw@GI=@?#x(Bx*!2rg2gxKZK#jGnVm^_ z+Fil&0{M`gCJQjjfH&;|8HiwCVg%n#+z;9vRB|A<@m2C8z=`%(GPX(sTin6%*N$w( z@`Zzi1I3wqp-4?3Z+E+3%^q?ge0hF;u4-3sy*+`C+nY?k0)}|hq!#f0|W3a^kESoKP zPgCCWrS^5Ik!q#0&2)CHaBKKZ@os6e^dLR?NIts5y5Q||;MU&009fim=@dg^hvA>C z=+VFb=)%9cYK0ew`SN^@_VnFYe=!^Qf$1*}2kz#2i#g_gSF#jfntjie+PH_gP>JOq z1;#TcM){vcdHQGt#FEapG$`qKtGf({iJoZz-3@T|$5`vh!soc!vkq@zd2b)g;U{}n apn|=_E#4~m$<^2a{;RJ2~ diff --git a/src/testdriver/browser.py b/src/testdriver/browser.py index 1cf8529..b1bd233 100644 --- a/src/testdriver/browser.py +++ b/src/testdriver/browser.py @@ -99,6 +99,7 @@ class BrowserDriver: ) document = Document.parse(body) + signature = page_signature(document) args = {"resource_id": self._resource_id, **dict(action.args)} try: @@ -110,7 +111,7 @@ class BrowserDriver: "action": action.name, "stage": "discovery", "runtime": getattr(self._runtime, "name", "?"), - "page_bytes": len(body), + "page_signature": signature, }, raised=f"RealizationFailed: {failure}", ) @@ -121,6 +122,7 @@ class BrowserDriver: "runtime": plan.metrics.runtime, "rationale": plan.rationale, "element_path": plan.element_path, + "page_signature": signature, "target": plan.action, "fields": sorted(plan.fields), "metrics": plan.metrics.as_dict(), @@ -136,6 +138,32 @@ class BrowserDriver: return Realization(self.surface.id, mechanics) +def page_signature(document: Document) -> str: + """A coarse, stable shape of the surface as the driver saw it. + + Element counts by tag plus the names of every control. Enough that a + reworded label, an extra confirmation step or a changed element type shows up + as a surface difference — without being so fine that incidental whitespace + makes every run look like an adaptation. + """ + from collections import Counter + + counts = Counter(element.tag for element in document.elements()) + controls = sorted( + element.attrs.get("name", "") + for element in document.elements("input", "select", "textarea") + ) + labels = sorted( + element.full_text.strip().lower() + for element in document.elements("button", "a", "label") + if element.full_text.strip() + ) + return json.dumps( + {"tags": dict(sorted(counts.items())), "controls": controls, "labels": labels}, + sort_keys=True, + ) + + def record_baseline_selectors(html: str) -> dict[str, str]: """Capture the control arm's selectors from a baseline page. diff --git a/src/testdriver/classification.py b/src/testdriver/classification.py new file mode 100644 index 0000000..4623111 --- /dev/null +++ b/src/testdriver/classification.py @@ -0,0 +1,245 @@ +"""Why did this run differ from the last accepted one? + +The centrepiece. Everything else in the framework exists so that this question +can be answered from evidence rather than guessed. + +The classifier is never asked *"is this failure acceptable?"* — it has no +authority to accept anything, and claims are run inputs it cannot reach (D-02). +It is asked only *"what kind of difference is this?"*, and the safe answers +outnumber the convenient ones. + +## What the T02 design got wrong + +`docs/TestDriverClassificationDesign.md` lists `SEMANTIC_CHANGE` as an outcome +the decision table can produce. Building it showed that it cannot — see F-0006. +A deliberate product decision and a defect are behaviourally identical (M12 and +M19 in the lab prove it), so no amount of evidence separates them. What the +classifier can honestly say is *"behaviour changed against intent that did not"*, +and hand that to a human. + +Intent change **is** detectable, but only when a human has actually changed the +intent: the claim fingerprint moves. That is a fact about the recorded use case, +not an inference about behaviour. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Mapping + + +class Classification(str, Enum): + UNCHANGED = "UNCHANGED" + """Same surface, same verdicts. Nothing to report.""" + + MECHANICAL_ADAPTATION = "MECHANICAL_ADAPTATION" + """The surface moved; every protected claim still holds. Safe to accept.""" + + BEHAVIOUR_CHANGED = "BEHAVIOUR_CHANGED" + """A claim that held no longer holds. Never adapted to; always escalated. + + Whether this is a product defect or a deliberate change of intent is a + question about intent, not about evidence, and is resolved by a human. + """ + + INTENT_CHANGED = "INTENT_CHANGED" + """The recorded claim set itself changed. A human has already acted.""" + + REALIZATION_FAILED = "REALIZATION_FAILED" + """The action could not be performed at all. Not a verdict about the system.""" + + AMBIGUOUS = "AMBIGUOUS" + """The evidence does not support a conclusion. Escalate; never default.""" + + +#: Classifications that permit the run to be accepted without a human. +SAFE_TO_ACCEPT = frozenset({Classification.UNCHANGED, Classification.MECHANICAL_ADAPTATION}) + + +@dataclass(frozen=True, slots=True) +class Signals: + """The three strata, reduced to what classification actually needs.""" + + surface_differs: bool + realization_failed: bool + postcondition_met: bool | None + verdicts_regressed: bool + verdicts_inconclusive: bool + claims_differ: bool + evidence_incomplete: bool + + def as_dict(self) -> dict[str, Any]: + return { + "surface_differs": self.surface_differs, + "realization_failed": self.realization_failed, + "postcondition_met": self.postcondition_met, + "verdicts_regressed": self.verdicts_regressed, + "verdicts_inconclusive": self.verdicts_inconclusive, + "claims_differ": self.claims_differ, + "evidence_incomplete": self.evidence_incomplete, + } + + +@dataclass(frozen=True, slots=True) +class Outcome: + classification: Classification + reason: str + signals: Signals + regressions: tuple[str, ...] = () + + @property + def safe_to_accept(self) -> bool: + return self.classification in SAFE_TO_ACCEPT + + def as_dict(self) -> dict[str, Any]: + return { + "classification": self.classification.value, + "reason": self.reason, + "signals": self.signals.as_dict(), + "regressions": list(self.regressions), + } + + +# --- signal extraction ---------------------------------------------------- + + +def _verdict_map(pack: Mapping[str, Any]) -> dict[tuple[str, str], str]: + return {(v["assertion_id"], v["step_id"]): v["verdict"] for v in pack["verdicts"]} + + +def _surface_fingerprint(pack: Mapping[str, Any]) -> str: + """How the run was performed, reduced to a comparable shape. + + Deliberately excludes timings and free text: those change run to run and + would make every run look like a mechanical adaptation. + """ + parts = [] + for obs in pack["observations"]: + if obs["kind"] != "realization": + continue + mechanics = obs["data"].get("mechanics", {}) + parts.append({ + "step": obs["step_id"], + "surface": obs["data"].get("surface"), + "target": mechanics.get("target") or mechanics.get("operation"), + "fields": sorted(mechanics.get("fields") or mechanics.get("arguments") or []), + "element_path": mechanics.get("element_path"), + "page": mechanics.get("page_signature"), + }) + return json.dumps(parts, sort_keys=True) + + +def _claim_fingerprint(pack: Mapping[str, Any]) -> str: + return json.dumps(sorted(pack.get("provenance_index", {}).items())) + + +def extract_signals(baseline: Mapping[str, Any], candidate: Mapping[str, Any]) -> Signals: + before, after = _verdict_map(baseline), _verdict_map(candidate) + + regressed = any( + after.get(key) == "FAIL" and verdict != "FAIL" for key, verdict in before.items() + ) or any(v == "FAIL" for k, v in after.items() if k not in before) + + realization_failed = any( + obs["kind"] == "realization" and obs["data"].get("raised") + for obs in candidate["observations"] + ) + checks = [ + obs["data"] for obs in candidate["observations"] + if obs["kind"] == "realization_check" + ] + met: bool | None + if not checks: + met = None + elif any(c.get("postcondition_met") is False for c in checks): + met = False + elif any(c.get("postcondition_met") is None for c in checks): + met = None + else: + met = True + + return Signals( + surface_differs=_surface_fingerprint(baseline) != _surface_fingerprint(candidate), + realization_failed=realization_failed, + postcondition_met=met, + verdicts_regressed=regressed, + verdicts_inconclusive=any(v == "INCONCLUSIVE" for v in after.values()), + claims_differ=_claim_fingerprint(baseline) != _claim_fingerprint(candidate), + evidence_incomplete=not after or not candidate.get("sut_version"), + ) + + +def regressions(baseline: Mapping[str, Any], candidate: Mapping[str, Any]) -> tuple[str, ...]: + before, after = _verdict_map(baseline), _verdict_map(candidate) + return tuple(sorted( + f"{assertion}@{step}" + for (assertion, step), verdict in after.items() + if verdict == "FAIL" and before.get((assertion, step)) != "FAIL" + )) + + +# --- the decision table --------------------------------------------------- + + +def classify(baseline: Mapping[str, Any], candidate: Mapping[str, Any]) -> Outcome: + """Total over the signal space. Order matters, and the order is the safety argument. + + Every rule that could excuse a regression is placed *after* the rule that + reports one. A surface change occurring alongside a broken claim is a + regression that happens to coincide with a surface change — never a surface + change that happens to break a claim. + """ + signals = extract_signals(baseline, candidate) + failures = regressions(baseline, candidate) + + # 1. Evidence first. A conclusion drawn from incomplete evidence is worse + # than no conclusion, in either direction. + if signals.evidence_incomplete: + return Outcome(Classification.AMBIGUOUS, + "the run did not retain enough evidence to classify", signals) + if signals.verdicts_inconclusive: + return Outcome(Classification.AMBIGUOUS, + "at least one assertion could not be judged from the evidence", + signals, failures) + + # 2. Intent moving is a fact about the recorded use case, not an inference. + if signals.claims_differ: + return Outcome(Classification.INTENT_CHANGED, + "the recorded claim set differs from the baseline; " + "a human changed what is being asserted", signals, failures) + + # 3. A regression is reported before anything is allowed to explain it away. + # This is decision-table row 3: coincidence is not exoneration. + if signals.verdicts_regressed: + note = (" — a surface change occurred in the same run and does not excuse it" + if signals.surface_differs else "") + return Outcome(Classification.BEHAVIOUR_CHANGED, + f"claims that held at baseline no longer hold{note}", + signals, failures) + + # 4. An action the system accepted but did not perform is a behaviour + # change, observed independently of any claim. It is caught here even + # when no claim happens to cover it. + if signals.postcondition_met is False: + return Outcome(Classification.BEHAVIOUR_CHANGED, + "the action was accepted but its declared effect did not occur", + signals, failures) + + # 5. Could not act at all. Says nothing about whether the system is correct. + if signals.realization_failed: + return Outcome(Classification.REALIZATION_FAILED, + "the action could not be realized on this surface", signals) + if signals.postcondition_met is None: + return Outcome(Classification.AMBIGUOUS, + "the action's postcondition could not be evaluated", signals) + + # 6. Only now, with every claim intact and the action verified, may a + # surface difference be called an adaptation. + if signals.surface_differs: + return Outcome(Classification.MECHANICAL_ADAPTATION, + "the surface changed; every protected claim still holds", signals) + + return Outcome(Classification.UNCHANGED, "no observable difference from baseline", + signals) diff --git a/src/testdriver/drivers.py b/src/testdriver/drivers.py index bb8cdc8..eb869c0 100644 --- a/src/testdriver/drivers.py +++ b/src/testdriver/drivers.py @@ -74,3 +74,33 @@ class DirectDriver: return Realization(self.surface.id, mechanics, raised=f"{type(exc).__name__}: {exc}") mechanics["result"] = result return Realization(self.surface.id, mechanics) + + +class CompositeDriver: + """Routes each semantic action to the driver for the surface it permits. + + A real journey crosses surfaces: a share happens in the UI, an audit check + against the API. Binding one asset to one driver would force every step + through the same door and hide exactly the mechanical changes that only + appear on one of them. + + Routing is by the action's *permitted* surfaces (D-05), never by which driver + happens to be able to do it. A driver that could perform an action it was not + permitted to perform is surface substitution, not convenience. + """ + + def __init__(self, drivers: dict[str, Any], default: str) -> None: + self._drivers = drivers + self._default = default + self.surface = drivers[default].surface + + def realize(self, actor: Actor, action: SemanticAction) -> Realization: + permitted = action.permitted_surfaces or {self._default} + for surface_id in sorted(permitted): + driver = self._drivers.get(surface_id) + if driver is not None: + return driver.realize(actor, action) + raise UnsupportedAction( + f"no driver for any permitted surface of {action.name!r}: " + f"{sorted(permitted)}" + ) diff --git a/src/testdriver/runner.py b/src/testdriver/runner.py index 08fdff4..833eab0 100644 --- a/src/testdriver/runner.py +++ b/src/testdriver/runner.py @@ -112,6 +112,7 @@ class Runner: ) judgments: list[Judgment] = [] + scenario_sound = True claims_by_step: dict[str, list] = {} for claim in scenario.use_case.claims: claims_by_step.setdefault(claim.after_step, []).append(claim) @@ -171,13 +172,35 @@ class Runner: step.id, ) + # Only an action that did not *happen* makes the scenario unsound. + # An action that was accepted but did not take effect did happen — + # and that is a statement about the system, judged below, not a + # reason to stop judging. + if refused and not step.expect_refusal: + scenario_sound = False + # --- invariants after every step ----------------------------- + # Invariants are judged regardless: they are supposed to hold at all + # times, however far the scenario got. for invariant in scenario.use_case.invariants: judgments.append(self._oracle.judge(invariant, snapshot, step.id)) # --- claims attached to this step ---------------------------- + # Claims describe the state a *completed* scenario should reach. If a + # step did not happen, a claim about the state after it is not + # failing — it is unevaluable. Reporting FAIL there would accuse the + # system of a defect on the strength of the test's own inability to + # act, which is the mirror image of a false adaptation and just as + # dishonest. for claim in claims_by_step.get(step.id, ()): - judgments.append(self._oracle.judge(claim, snapshot, step.id)) + if scenario_sound: + judgments.append(self._oracle.judge(claim, snapshot, step.id)) + else: + judgments.append(Judgment( + claim.id, claim.text, Verdict.INCONCLUSIVE, step.id, + {"reason": "an earlier step in this scenario did not complete, " + "so the state this claim describes was never reached"}, + )) pack.verdicts = [j.as_dict() for j in judgments] pack.finished_at = datetime.now(timezone.utc).isoformat() diff --git a/tests/__pycache__/test_classification.cpython-312-pytest-7.4.4.pyc b/tests/__pycache__/test_classification.cpython-312-pytest-7.4.4.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e1a42dfefd5accf81c85658c599a174efd5dc3d0 GIT binary patch literal 22288 zcmdU1dvF!kdEec;_n~{Qgm@!iU~PioVi1UzZ2~m1BpYHQ92;U=HN9TlT}fBmhkSM~ zLdv}jPTd5@PK=v0#xYHuHkm?68qf6Q%(QtlnYJ_SbRdc&vf~VyrqfJ1(^M8tnDAGB z-`VGtRzf6>kTVrEVz|ptz<3lgqA_zaC3-?Ia5%KCjLZg;q*vzWXfc*US~1?6aPBnpHt;*i-XzX)AyrKGri+cejr@MJ z&{SN|yP(+I+gx1OyO5{F3X6)1dlwg%^e*Ao@xs#LvfgDpo+vCYuIOFC;|+zDVry?} zab@pHew{4bQe4%$3h{`RIxA{zTKef&?`lcVlSBL5hw;x^#Vh{#GQNnCv;=VGpS9aCPdDi6+!=X6vhQhYpBB7%nbF?!w9vazTZixt?RJElwDkx# zYa0-@X&HoDw2cV2YIh)P*ES*Crfo*JeKeBUTG24!4-e`}uAmx5zCW+CO-fN$jSAB> zMIBJ{CBsxqWGJY8dZB>CoNB6t@<2sb{`2Ig6}5zHrm5zR=~`Pd`Glz(W|ry3aJgjY z*`jK){D~(NLoE&$bVKRWOZkBkFEm&-%zOb2s|IeUh5RwSO`)E2Z{E6fyE4GamEk0s zERQH!899t{Q8x$kr2$3D_xI~eFPXcQys2zN3%G5Tm7;n~=Uwe*YEd64vtvr7WT^f6 z#x|v+l+;fQ>nvYHIb>2zrCchEdL6TSkwd5SlF~QIt3juA1!atKNf{X|Dc;L`|N4inAv9FRZXxX81g_ZPCYmpPl z;!S4_ogLR1O0?t#_1v*+zGUKYankf*9Rb=nFIvszK09N!U(K0iHkxziFOks({Ko}B zyxNSb7X{NHt%6V$KPl`K9tU{Tq*|mVn~2#*RjNj+@&)Jm@hEacYq46qmZ&vUg{Kmd zP!+0icS+RZCUV%vP|S__XOnusk7~3UtH!Gh7i9Z-DCOJ;oT>@8-l23L*@+G<{2Sh1&{VMWy8VJl{qkLe}DO7vktWh<;; zMX>}5nibPn9`DjhxG&3!VyWw<6)Q6}hgC^HuXxAxom&R6)V83oksI7X>v^+X1CJX# z%(~>lo}z8TqgG-VG$7kwW^66$YsJ6uQ3NN2HyT!+dF*1t`bj~Ks^Xay2X zywSezHRBui{_Fnn_9K@PZKtBAj-Nj9%F=VO%c+$YQ|rIKeEqq^`11Bs``=iw`1!GC z#?IbOH(yP>cKdj1$3$!QrPl6m=;N&qOtc=k)Ouv1_2{M6qvNgF@dZy@YoLa%@#dy@ zc-^2k`b@HY(9hv34G#vh7s1eP<>{Co0`erQ4}=JC)wXOK-PQ?wrZy zwVZ<|=!_uHa^MgA9|(gFTov8PU6TUKRH#bsB(FuPks)VU@w%J<%i3F#Rj~!BwP;nY zMk_)sR*jt&jdT9;9OArYu(SX!vKs3LS4n01C3oR!5tEh+KZa=aOgr2cbX+i60`)gf z%Z5wSdw%x`S+K;S`IIPu)79j8_h{+dJ*o;`Pl7o^dgA9q4`eMm?;Zr7FlhV;mve=L|Yq$hq(^dJ@(>9D+#`+PUZ7nx^1d7k;AZayu3 zHZg|vQHyH{t)Uvvi~K5KE_DYq&@-Y!(+(f3+!{Y3YQhPzCgDGVe;NO=T69#%Bzu6= zQr^gjW7TyVhDYzt7|9Pg5oNDJu{+%)M4Y6p8Dj(8YR?#(c%GeZ9z>ix+nhYxGe#y^ z!>_F#7Lqk3Z}7tH$jV4;J^fJq%@ zxpHw>WxAEd?JTiFCiq<|RwyHfVXep-gXIcXB+8l{D3truLbe31&}zsMgGb3$QwQ%B z7`1N3%FJ3!7zn;M3x2bpmN*NSX|SK09@$D!TlTZ;2wv<8UfgAO$<799&pYT`ZMCoL zX4<#cUF|f8=kdVYhNLB7OgVc38N2Xrd=fK9UX5JezjZWMW!@Ytd>Hxb~_^J>W+eBMDrKa$G~1I$ZM@ z%%KoE==PoCnqPkiZ41RUzr+%^Z~l2^2?y8a+oPp{Cl2Y6k5-`JbN0l~>EPOYd(aZ7 zH>3wMQ3_Z;XHRC9AY7ZfjE_J!$mgTaoPDTz=mR`^yGFv}$z0sH;YiFV!SQwOo_l!c z;arU4T+G3_xVhY&e}S=D;ima#U)Xc|>)@Oiz`AHn<~YYXQ4lcq83L}mT)aZW!K+v8R?AJK?-^hWEy?UV}~e}shh zC<3%jLP`*LASC$sv@>{dCwqKa-n-fFQB7F{2GKj2dbd4QD!2?vo*Abi)uE>Wcemj5T^5L0aL}<(fD;eSj1S?$HN0;!tn3D#CoOBfL zz9v@1Ax|K2^NMrcnA<=13__J4M!XgoO^^vB zhfy67G7V$@Od%{s~eN=Tpz7CaDzEOi;JA#I7ZU=-LVSwqNe|J^8j@Or=nbLHP1aN2+tLT*qqDqjKst zg|%5v-H!frPO5e%sG9iP;$MZX)regWQ5=y%Ds9uObAdG*qAUuhfiDZ7Wl|Uu{<82V zKD~&H;o(7FNDs)5p%AJy3w@iDYSq!VFqUXG`WBYg@2^MS%$07B1ET{I@ARYUDJw2G zl-FUsps{~$Qidiw6YF6`6dRLvD`R&h_c@D|Y$3GN!3o(skkW2j7I^yGNYKgwP`4%XCnl zzDzwP1e14eCd9yUt%_g}me~wKRSK{$Sk92DJ!8zTok_VHeTbk%j}h=?CFq^~RBx%& zqM8J~Gt59^7>!ySTIKjL@Uyi&ci<-&L0*u~$rvGZo# z4ad&QiLr~{V4OX2e1m)*BdR6N#yLydpfza8YJDpJp%wzj&7M) z+bI5FoE=U1?5!Wmq{sff!(;0>BMBRQpRQ~KE81TvY}^Dk)`YJC)43hKA8cciXl_Tq z7s0mFdvsRe6xj9x7$ME90^1z}V@qaw&a~c3?4;ti%WHfHG@APypmmrv%|RGF4RSUh zzL{%p!N7tiFC$>!nl(iGd5XzY&1Gkk2fIm>KhW9nz)uJeGvI6Y?S;DGcM0sk%-Udw z$V}Y=)CbDGy$aW;hbShXQs=jzm!O=RZ5=njGGj#wI;_BIpHXIgx@IMdItk8s08b_! zP#qAr%(-KhqnJj4j$o`#YfcKd1L3%_ z81D#-bXr)w{(B4WIU}CC^(*VouYdLN@r@mSbnC^1_ndq`f*&qZejbT5$1X2i^hRp& z>HTL1UwQH`%+H%Im&Q|@CsKR9liD*G5gL-u%g@S_l=0;LNdc#82_d?5Lf-Q3|BcIA zz{3lRI>ld1N$9yjOa7T9`!~v;lM$TTvwurHPw#0grob}P^Yo|1qyNi+9(U{mK<|UT z80LfFk`7lJ!MXN?Ti!p{BEBN98VSIgqZJ_?1b})CTn4280lbl_d`>ttJ?X*iC1BSt z5dw=dOMq}oFiQJ`bnYH44b&eZFVtx{=In`|6FrCoY7ObN57CfU=G)WYNQCq>@P-`e z;D%#2gULb9gYr{Y&jSp@*{H3b320>e5en1Dc!24?U0;5Id=~3}vXnK#IzkbKl^ z!HYBtndo#R>q9C>vayYUzRV&}+7ci1>m+=+z(yp-){=9X@eo~}rhpm@NQWHaLR91I zDdcnXf<6&kBtZ_Un zFh;!kQ5%CC3IoqU!8hG%o@aXdbcY@Uz6&gwS!g zb$N7mC`7!2QCs3pTQqt<540XaakW@A0x>yGB?4mdZ_juq<~=il=3@`n+_?hhGH*=X z_fThM4VfV`OJJrX0^=Q0#@Enk4QFMJjxjB%rK-M}b)nm*KuuxuCJYt$yusXwz>O%H zNmm}h)H=9r`))27<;o?Kl?!Bg7$}!?rJt3H{Kk<3@8#!?2M;Ll6do?ab(nIIw|7B- zr)9~J^T-N9+`9rdnF{w`R!}KS3=r1f{7tUbWkprE-dynw1Z4lP{ZY}3Y9>igSkLqv zT%Vb-o_x2-yNCS5Nu;A*kPU-Z2v2pK$zh#ZqVC%=D6brz?bIHx&lWH||Mq%O)2P6* zU>Fc#A-uZ6SfGRO6(_%TwZyAZE9PLC<;NUPac9&F_@DPNa@e(PCG358PuF)jSa*{O zR2xVM{Cc4 z2-_PLve@JRzjYnwKFm(DLf-c5Wz<4rY5W}mZi!j2{Pc-0tT?ys&)1AMwV%B2io9;( zCV#KX>#oRqC*G0Y_vF1Z0hL|@2UG>(8g^`nUBYSU=qWDDg@P-&=zFkA=8!O0y$Qdl zI_L-?m@v7JR}?^|&MRvE;z|&J!w8%Qs&!Bu0*G~jl^h_OVR-}Qm75T(eE1LPsqbqR z3LPQ6m{|hQaU)+Pk5SNK9(2GLH2@vUF=zM@7=sW7A=E>sfnWpw@XW#731Jl+BON^E zz>+0zLM;7Fzvq<0k|pNt!Q^c#ao;CcGCn@ANu;r@C0rDRzP6e?F4R)hl=0=j8wguF zjnx#^P7{^z*{(pDkU4{`FwVMv*N?rYh1M3Fjc_I>tu<>4t7%7{-8k>nFhdz?6@? z9$$l7_!0X-+seRglvM1%QQ!#Gzj+WI7MwXLgLxXzO@p?Tl?M1$Hbdu57!E8YiLVAr~2BE_~?Xy7!JR+CvuV0~7Da@AbriD~Zg+P5xd_WUeH3PP|RO zi;10NN#8Y-jq>yoXTeOpC`iJ6V5Odhzlg^wiGf{?0bNB^{F28iczNFmwu&}Sff<;_ zR?Xj?_}2rsu^WJsH&%5<(YZD2L5cj=$5zE}&~u-yN_cD)8DD*g{8wSC$oM*cwkmxS zZsWh{_uTlluvL6~psh${n$#%BT4%-fTlBXr!>y5TS$4#6m{Lo6f#WB^wOTSpU;!>x zpp}5F*pok9PfRjD1I~v*rJ*^nudtilvvRj7-OhGhs5?}4+-}9?1s5J0c{Z56@ zEDT~lunrxv%?#u~IP{wBg5u08_G6PRPa*?W#b^1iw-K^y1K-`UDzj&Bs z5jjAG?E}7OQV*lT8QGHg7Og0DZK(w-#y5KzR(wP?v6bDxzIT}zF-xY~1{F~h8$T0S z>~GJT+3eWeH~aK$Zqd)9K4U$Clfvcpov%Lm+Izm9`D*53y6sfgbMHSRpH01T_zRC+ zUb*H^_q?>{RM#gSn29m;OnmGLx53CH$Q*>$L~MWekpy+C+@$zY3o<+KY#ygxBlg!(>q?={le~v#kXHt zeEay~4Hwg$-&llz4`@#&YT<3tJjiCPWENfmwAmZ+bT<2BMJ?EO*q6}+?<=EM#*5lh zA@0m-Mm>22{|5P#JuOU0Qp1L;38|sWcC%xDgJed)#F=UK8H&w=;jPn*dF&^9iHdX0 z*h3WawPj%jDpIJq8w+_ie8t#oJf=a$eUELlA9c+Q<~sXZl%b9p-$HOwxYiVpc8PBa z6uz}cu$5!4-TB>HJI9;4NIUip>A7B)_x=)^xbMm9>?Mb1@eYy%ubUa7GgDV zG&cS7AJ6^N{`^N~LFG-1;Q{_x73{d6PjqO$5oOv3W;jDa(lK^&z)@O({enK|;;9mss#R2_}xq+-7_|%Q98NlX`fkE7&kNJ=gO`HxHOqizR`{@%poRJwQo1B9Hb7OML z2liH=5UeGA_)LfnhR5AT0{=Vk9v~ADL5R{OrJie9?#qVjoPAf9&-KKff?Dj;)O};Nz1>muW4n#>X&ijshBuEs`b>;l=y_u|Do9kbe)f{Mx7|#=9jk(Qzl1a-N`o6cz??DgYB3Ms5wl zNMHo~(PrO7{=cGve~kc^l_nwC{QRnCR(*H*=JAHMHw7vBi15T!z1Fg2xWOIAvCWz6Mgo>C--08ap>g!i;Zh8$;vAT+ONoM6K~(| zlH7K!ky@POUAxY`6F;kTt&e_mYK zrt*(><#Z4=QnTU)sP15;sFvE4M@R*1TbMx!@hK%o-OK4G{F?@#F~%+we3r~qMorrZf);1%pqe1kRxJ7$~ujMPee)kSJFR5^7!e`Uqv4mZV z-$s$IQQ40oz#5N9(b)gVKM~%FI+1G>xrPN5qXp0J`AKipB~8^`{k-mq*FY8fg?Um) z8P4zF35jvCB6tn7N65rC&n%gLa7|+Wj63XK5M*{(@oZKr=WMB=zk+O#8W>SJeAvnP zJ_sk4R!i0~$T}tr$GqX)Dzel<61+RSHN7=1$$*6^>iyx$2sWB^*-t zoSP=Vfeo-lzJ2WbsDLC>V+WY~$%rV5Z%Q#y{#mmiuKh8|bbpZgkK;r45iBubc+ zqI4}Lh~453gthox5xRaLbWX;Etvh}uL`7AcY!;;ElgmD~`l;2Y2fr(>nUX|t+0}?3 zDU%eNOs_~wm*s|2`zBIbE~U0iMsWSZz;zVYs4kkM9BVJ7)=tJL1rFsrg_mid6l~q} z^Q0&x9n8~6DNVtY1(ebpOj$@Ni-Pqmrj#YYl%C*_VysXHcD zQpzpClvR|nI#_BAr6|FaTPfwXV5zl~@*W}GI+3D!FQ;1O7omZh+JeF-6O+3HDeo(Z=c%OC~mo`M8z1VAO8>82<^)N literal 0 HcmV?d00001 diff --git a/tests/__pycache__/test_lab_ground_truth.cpython-312-pytest-7.4.4.pyc b/tests/__pycache__/test_lab_ground_truth.cpython-312-pytest-7.4.4.pyc index e7f430471ab203796b41093d59177f0db8776513..77df5aa7b6948ab7a81256c3f45d78b3193adbc8 100644 GIT binary patch delta 3141 zcmZ`*Z%|a%6@T~bA9mk+kf5-DMZ5(;R^+dLh!tT`Bq6DRjM1dDip#!-ytwYt``)UH z%PdU&P^l5{CXH!Kn(3s89Zi&MI%fKze(48i+G$GiAq^MVeCx*oO5>< zA){~LxA&fV&UxqEdw%EK&wouW-z0gz&&^dNcwD99FML=!l~+uvrZ$90-SkV_WKA#E zEA;gjUA<*`CI6FLPpZy4ineYzEnP+vczP@J>eEv1dfg9Lsc!_V(rW-W=$inm^;$rG z+~KcJbvgz%B!}yRbbl8;7PEu48IA0q_MmA|D`Z51%#6~Aaok`u91ZFgtQep=d}Kl5>eF+#W&FtgW!dlHU;Mx%AVSWhRrk+2l(r^Y}1b6N6?^c2SZ2e z{4SP+En%YobB?`DjR;K$n?oSNhyT4bFZIWba}L*em>H>e>uMaHaTxl^@LoRCT*Ma} zTihH4F5dH4VbMAmva<*;qEKapI6lXXrocK>W7P;75hf81BWyyL;=gO!>iQ`R&D4LJ zK6H|Ce!clC^&GCb5GMH5$F8|vgJFd0+vi;avLwC2uderzLwtHOmDyf?yTn&)yFo)d z#|hhEd$TiKbUDUOM9Bf%Y<9-BQ6{Zm;Hn5^pNe z;vx0EeHXCIjwvDvwhv2efxHI_w31R5H2o#WuwQk`1Ibk`#l(XSn@Jojw=s##T=Xg5R5@K1cE@6V9KkcFv=d z(aw1Zeik*!6(IbFz7{^}^FDfSAs2)e?)jPaijF$|*VZDR=g&4r!7cwE0)Q zO?=@y1+wko4K<}XF~Z2%EeSr|RjB9$_Z#LN;+WoUD{}ZU`+hjboBFaXLHIwnt%lRr zbA$!|DA$$myiC$B(7nSj&=o%3{UYHPy5hN z*ffA8(F{E;8nl4op+Wowsbw4sMr<=gLxaJ{5zzc%^d%aM&{#w_fO$w(GjiOt%>J+; zJf;r=nK#6YkZBp=I30|hph4iDX&W>YJ%;c5b`(lQ^e8>T3?l+{!v^XG)S3^P zueVIM%&2X%TEVQkdZP7A>xCbDtomU(FFU--qFeGPV{9>*gA)m09^sbzAJXEI=q`#t z)2WPY6VinhI${RPL>cpnAr8Kl9N!l&STYcih}U8y^CFUTqzjp`OBvg9 zKSH5@dqq@fkqZ05Z*+XCssBOw{Y$6-k_{{ACn05%bA&+h9^O+RkI0t(A~!uglqw&^>}--h|5}LeFiNWIq)wKH9dHUu`Vl&o^sH=ZFKicL}(s_|@U@ zeReR6eGoHFbSsOps7)<1Y(#AAe_aE>lWcy{ej`nm481mCFUxRj$pdIxHfU029E0Ym zQ*aF3FU_ZlurXjGk%4@6Fru50Bh)w)h~-wso)yGeqThX*iQ4e+LI6B|0FARo$%n8%TO5|ipN(osAc(!Z zz+_Dx;_<<8v@iD{R903wQ80w(m2wfzZ3paH>PAP(v*LC$viC>i?G$;sg^&xpwL3{( zitiXxhh$R~zzFm%SAMDQVCqW;^{ zaxALH!bU5*2~)^8$@+6j{T?Hw<&t)g%&I-JY76|$%d)oRw(5CQ9@Q3n{9Nz$GA{{` Yh2lJ-ER^Q}6y%cck~>r;JB9oI18`Ju4gdfE delta 1509 zcmai!YfKzf6vywKeX-24u&^w2g#s?G1(HH}Y?0Tp@{oX_joNm7EDJlWlt=2FffiWY ztu|3$D+{OEMjJK8#3B?zCw%dn(L@tt4Ac~YCcZU(Xj~R7@r!!zR8!!CPBOpypYxbG zb7tjE$eg#l<3ed}W?E?Mj(8ref=KULP)8x>UhwuX65#J{#QP4mD?UNVs1} zpTjH|p*imE>dU!+=I%=5w)t`|Aa}Mek21%XPnqkpQ|9>!DD#I^_9FSH>SUh5&XE93 zB0&myp)jIKn4;<_dWvHdLG%qQ9Xc!L6wh+_CHUg78;7BwG$V1GI>`iuohQjmPr{R> z&WtdV`3waN7a2MkiWpvj4@%1ufJ$Z)M<{~m7ciA|K|)!gES6QO=|ip`xy?^f7tt_; zAfd4YZdC>}SLmex?gt{+1+7(9+y>8QTOnAbGgL_|R5|?cS79;?IQ8JjNrF*FEZkRa z)(j$IP&{eVT&P`*#_$+Ag~m8P_H(_o+%c73CEkF#@+`_!;c+~K>(jthV}rya`SQgY6V6H?I$8s%W>9+KWNQDb{8_8RuYSrN zj_|Xq{K0ptu06LX%}O_IWox$T6rD-5YFH?=a6fbKT~iW1#X6kq!@Umr9ynzFpEm0Tk%&VL ztkD+u$!SsPB@--J)1XhMmu>Za47ex>R$5Zv2Uk;4>^V$70~f}j(`qyvYv)Irq3*$E zj+8)dLsiNK-`d`BtliVn-gxve(n0GRWPc?LHu$ao(P6%^(_EnRwP!#el1CK!y#rED zH;gr=2Hv7K7^0_WqI)NZel(|VA)Utfa?CoPG*vrZd&RxV=dByeU-Opmxyz%|qc?fm z8eh1=7tWtu^w0WN`Kkz~iZkwLk-<2Act^)HqM(h9GybOi72Pv!GZPJrF`ge%tWgx$ z9Tfe?uQ8r)QT$u`4gW^$y2I`p+}R(mRH8hVRT5E(+f!&JtB7T7GVCpPQ%@Tq|NgKX zvuE$gPrQgjvg24a#;<~OJb)))`}m_Bv$ToieTHic%7*tcrR@6Gn6fi;GhAkFt*0SkB=HYRV4Nt@MU-9 Z0UdVYs9BGV(bPEHhPN^~Tth50?QfODn5F;# diff --git a/tests/test_classification.py b/tests/test_classification.py new file mode 100644 index 0000000..1e9c116 --- /dev/null +++ b/tests/test_classification.py @@ -0,0 +1,195 @@ +"""The classifier, measured against the labelled catalogue — and attacked. + +`test_response_matrix` samples benign and hostile cases alike. The E-003 group +below does something different: it *tries to make the framework unsafe*. An +experiment that only samples cases chosen by the same person who wrote the +implementation cannot establish a safety property. +""" + +from __future__ import annotations + +import json + +import pytest + +from testdriver import Runner, Verdict +from testdriver.classification import ( + Classification, SAFE_TO_ACCEPT, classify, +) +from testdriver.provenance import InadmissibleProvenance, Provenance +from testdriver.intent import Claim +from lab.mutations import CATALOGUE +from scenarios.full_journey import build_journey, journey_lab_server +from tests.selfverification.checks import check_intent_independence + + +def pack_for(*mutations: str, observer_factory=None) -> dict: + with journey_lab_server(*mutations) as (app, tokens, base_url): + world, driver, observer, asset, oracle = build_journey(app, tokens, base_url) + if observer_factory is not None: + observer = observer_factory(observer) + return json.loads(Runner(world, driver, observer, oracle).run(asset).evidence.to_json()) + + +@pytest.fixture(scope="module") +def baseline() -> dict: + return pack_for() + + +# --- the response matrix -------------------------------------------------- + +EXPECTED = { + "M01": Classification.MECHANICAL_ADAPTATION, + "M02": Classification.MECHANICAL_ADAPTATION, + "M03": Classification.UNCHANGED, + "M04": Classification.MECHANICAL_ADAPTATION, + "M05": Classification.UNCHANGED, + "M06": Classification.MECHANICAL_ADAPTATION, + "M07": Classification.MECHANICAL_ADAPTATION, + "M08": Classification.MECHANICAL_ADAPTATION, + "M09": Classification.UNCHANGED, + "M10": Classification.UNCHANGED, + "M21": Classification.MECHANICAL_ADAPTATION, + "M22": Classification.AMBIGUOUS, + "M11": Classification.BEHAVIOUR_CHANGED, + "M12": Classification.BEHAVIOUR_CHANGED, + "M13": Classification.UNCHANGED, + "M14": Classification.UNCHANGED, + "M15": Classification.BEHAVIOUR_CHANGED, + "M16": Classification.BEHAVIOUR_CHANGED, + "M17": Classification.BEHAVIOUR_CHANGED, + "M18": Classification.BEHAVIOUR_CHANGED, + "M19": Classification.BEHAVIOUR_CHANGED, + "M20": Classification.BEHAVIOUR_CHANGED, + "M23": Classification.AMBIGUOUS, +} + + +@pytest.mark.parametrize("mutation_id", sorted(EXPECTED)) +def test_response_matrix(baseline, mutation_id): + assert classify(baseline, pack_for(mutation_id)).classification is EXPECTED[mutation_id] + + +def test_baseline_classifies_as_unchanged(baseline): + assert classify(baseline, pack_for()).classification is Classification.UNCHANGED + + +# --- the metric that matters ---------------------------------------------- + + +def test_false_adaptation_rate_is_zero(baseline): + """The project's existential safety metric. + + Not "low". Zero. A non-zero result is a stop-and-redesign signal, because a + framework that normalizes a real defect once has spent the credibility it + exists to accumulate. + """ + false_adaptations = [ + mutation.id for mutation in CATALOGUE + if mutation.label == "DEFECT" + and classify(baseline, pack_for(mutation.id)).safe_to_accept + ] + assert false_adaptations == [] + + +def test_every_defect_is_escalated(baseline): + for mutation in CATALOGUE: + if mutation.label != "DEFECT": + continue + outcome = classify(baseline, pack_for(mutation.id)) + assert not outcome.safe_to_accept, f"{mutation.id}: {outcome.reason}" + + +def test_mechanical_changes_are_mostly_absorbed(baseline): + """Adaptation has to be *useful*, not merely safe. + + A classifier that escalated everything would score a perfect False Adaptation + Rate and be worthless. This asserts the other side of the trade. + """ + mechanical = [m for m in CATALOGUE if m.label == "MECHANICAL"] + absorbed = [ + m.id for m in mechanical + if classify(baseline, pack_for(m.id)).safe_to_accept + ] + assert len(absorbed) >= len(mechanical) - 1 + + +def test_no_classification_is_outside_the_defined_space(baseline): + for mutation in CATALOGUE: + outcome = classify(baseline, pack_for(mutation.id)) + assert isinstance(outcome.classification, Classification) + assert outcome.reason + + +# --- E-003: attacking the boundary --------------------------------------- + + +def test_attack_surface_substitution(baseline): + """M23: the control is gone from the UI; the API endpoint is still open. + + The most plausible concrete route to a false adaptation. A driver that + 'recovers' by calling the API has routed around a control, not adapted to a + change — and would score a successful mechanical adaptation while an + authorization control sat broken. + """ + outcome = classify(baseline, pack_for("M23")) + assert not outcome.safe_to_accept + assert outcome.classification is Classification.AMBIGUOUS + + +def test_attack_concurrent_mechanical_and_defect(baseline): + """Decision-table row 3. Coincidence is not exoneration. + + This is the case a naive self-healing tool gets wrong: it sees a surface + change, explains the failure with it, and adapts. + """ + for combination in (("M01", "M15"), ("M02", "M17"), ("M21", "M20")): + outcome = classify(baseline, pack_for(*combination)) + assert outcome.classification is Classification.BEHAVIOUR_CHANGED + assert not outcome.safe_to_accept + assert "does not excuse it" in outcome.reason + + +def test_attack_evidence_starvation(baseline): + """Cut the independent observation channel and the answer must be AMBIGUOUS. + + Never a pass, never an adaptation. Evidence starvation is the condition under + which a framework is most tempted to fall back on the actor's own account of + what happened. + """ + class Starved: + def __init__(self, real): + self._real, self._calls, self.name = real, 0, real.name + self.watches = real.watches + + def snapshot(self): + self._calls += 1 + return self._real.snapshot() if self._calls < 2 else {} + + outcome = classify(baseline, pack_for(observer_factory=Starved)) + assert outcome.classification is Classification.AMBIGUOUS + assert not outcome.safe_to_accept + + +def test_attack_provenance_laundering(): + """Intent derived from the implementation must not become a claim. + + Rejected at authoring time, and caught again in the record if it ever got + past — belt and braces, because this one cannot be noticed by looking at + behaviour. + """ + with pytest.raises(InadmissibleProvenance): + Claim("c-laundered", "whatever the system does", + Provenance.AGENT_FROM_IMPLEMENTATION, lambda obs: True, after_step="s1") + + tampered = pack_for("M15") + tampered["provenance_index"]["c-bob-revoked"] = "agent-from-implementation" + assert check_intent_independence(tampered) + + +def test_safe_to_accept_is_a_closed_set(): + """Only two outcomes may proceed without a human. Widening this set is the + single easiest way to destroy the safety property, so it is asserted.""" + assert SAFE_TO_ACCEPT == { + Classification.UNCHANGED, Classification.MECHANICAL_ADAPTATION, + } diff --git a/tests/test_lab_ground_truth.py b/tests/test_lab_ground_truth.py index d2fcbc2..c4c8a6a 100644 --- a/tests/test_lab_ground_truth.py +++ b/tests/test_lab_ground_truth.py @@ -28,7 +28,7 @@ def run_against(*mutations: str): def test_catalogue_is_large_enough_to_support_a_rate(): """Six mutations cannot support precision or recall. Twenty can begin to.""" - assert len(CATALOGUE) >= 20 + assert len(CATALOGUE) >= 20 # 23 as of T08 def test_every_mutation_is_labelled_and_reasoned(): @@ -82,10 +82,19 @@ EXPECTED_VERDICT = { "M18": Verdict.FAIL, "M19": Verdict.FAIL, "M20": Verdict.FAIL, } -# The two SEMANTIC mutations the reference scenario cannot see, and why. -KNOWN_INERT = { +# Mutations the reference scenario cannot see, and why. Coverage is scoped to +# what a scenario asserts (F-0002) *and* to the surfaces it touches: this +# scenario is API-only, so a defect that lives in the UI is outside its reach. +# Declaring them is mandatory — `test_out_of_scope_mutations_are_declared` +# fails on any invisible mutation that is not named here. +OUT_OF_SCOPE = { "M13": "only affects grants that omit a permission; the scenario passes READ explicitly", "M14": "a change of intent with no change of code; nothing observable moved", + "M21": "a UI mutation; this scenario never loads the UI", + "M22": "a UI mutation; this scenario never loads the UI", + "M23": "a UI-surface defect; this scenario is API-only. Covered by the " + "cross-surface journey in tests/test_classification.py, where it is " + "the E-003 surface-substitution attack.", } @@ -105,26 +114,43 @@ def test_no_mechanical_mutation_changes_the_verdict(): assert run_against(mutation.id).verdict is Verdict.PASS, mutation.id -def test_every_defect_is_detected(): +def test_every_in_scope_defect_is_detected(): """The floor of the whole project. A defect the lab cannot surface is a defect no later classifier can be measured against.""" missed = [ m.id for m in CATALOGUE - if m.label == "DEFECT" and run_against(m.id).verdict is Verdict.PASS + if m.label == "DEFECT" + and m.id not in OUT_OF_SCOPE + and run_against(m.id).verdict is Verdict.PASS ] assert missed == [], f"undetected seeded defects: {missed}" -def test_inert_semantic_mutations_are_declared(): - """A mutation the scenario cannot see must be named, not silently ignored.""" +def test_out_of_scope_mutations_are_declared(): + """A mutation the scenario cannot see must be named, not silently ignored. + + Applies to defects as much as to semantic changes — an undeclared invisible + defect is precisely how a suite comes to look greener than it is. + """ for mutation in CATALOGUE: - if mutation.label != "SEMANTIC": + if run_against(mutation.id).verdict is not Verdict.PASS: continue - if run_against(mutation.id).verdict is Verdict.PASS: - assert mutation.id in KNOWN_INERT, ( - f"{mutation.id} is invisible to the reference scenario and " - "undeclared — either cover it or record why not" - ) + if mutation.label == "MECHANICAL": + continue # passing is the correct outcome for these + assert mutation.id in OUT_OF_SCOPE, ( + f"{mutation.id} ({mutation.label}) is invisible to the reference " + "scenario and undeclared — either cover it or record why not" + ) + + +def test_declared_out_of_scope_mutations_really_are_invisible(): + """Stale declarations rot silently. If a mutation becomes visible, the + declaration must be removed rather than left as a standing excuse.""" + for mutation_id in OUT_OF_SCOPE: + assert run_against(mutation_id).verdict is Verdict.PASS, ( + f"{mutation_id} is declared out of scope but the reference scenario " + "now detects it — remove the declaration" + ) def test_deferred_revoke_and_revoke_race_are_behaviourally_identical(): diff --git a/workplans/TD-WP-0002-vertical-spike-crystallization.md b/workplans/TD-WP-0002-vertical-spike-crystallization.md index 69569d2..fce0d16 100644 --- a/workplans/TD-WP-0002-vertical-spike-crystallization.md +++ b/workplans/TD-WP-0002-vertical-spike-crystallization.md @@ -358,7 +358,7 @@ The headline result is a narrowing, not a confirmation: ```task id: TD-WP-0002-T08 -status: todo +status: progress priority: high state_hub_task_id: "e2403d3c-bc35-5dd3-b1c8-2474663e06d0" ```