tegwick
44faf3de8e
Two decisions taken with the operator: stdlib HTML driver instead of Playwright (F-0004), and a deterministic discovery runtime instead of a live model. Both sit behind interfaces so the alternatives drop in later. - html.py: stdlib DOM parse and query - agentic.py: DiscoveryRuntime (agentic arm, ignores data-td by construction) and RecordedSelectorRuntime (control arm, uses the strongest identifier the page offers) - browser.py: per-actor sessions over real HTTP, constructed per call so no actor inherits another's connection state - cost/nondeterminism metrics recorded from the first run F-0005 (CONCEPT_DRIFT): the H-001 result is a narrowing. Where test ids are preserved, discovery 9/9 and recorded selectors 9/9 - the semantic action buys nothing. Where they are dropped, discovery 2/3 and recorded 0/3. The concept model presents semantic actions as generally superior; the evidence says conditionally superior. M21 and M22 added mid-task: the deciding side of the axis was N=1. M22 (field names renamed) defeats the heuristic and is the first concrete evidence that a live model would add capability, not just cost. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 1629012@bnt-lap001 Assistant-Session: 78d4fb13-8a1e-474b-87a3-9b9261c49a39
310 lines
12 KiB
Python
310 lines
12 KiB
Python
"""The test-driver lab — the system under test.
|
|
|
|
Grown from the T04 seed (`lab/minimal.py`, now removed). Users belong to
|
|
tenants, own resources, share them with READ or WRITE permission, and revoke
|
|
that sharing. Every state change is audited.
|
|
|
|
Two access paths exist, and the distinction is the whole point:
|
|
|
|
* the **enforcement path** (`request`) is what an actor uses. It checks
|
|
authorization and can therefore be *wrong* — that is where a seeded
|
|
authorization defect lives.
|
|
* the **observation channel** (`ObservationChannel`) reads stored state directly
|
|
and probes enforcement out-of-band. It is the independent channel required by
|
|
decision D-07.
|
|
|
|
An oracle consulting only stored state would verify test-driver's own
|
|
reimplementation of the rules rather than the system's enforcement of them. An
|
|
oracle consulting only enforcement could not notice that record and enforcement
|
|
disagree. test-driver observes both and treats disagreement as meaningful in
|
|
itself — that disagreement is the precise signature of an authorization defect
|
|
which leaves the audit trail looking correct.
|
|
|
|
Mutations are applied by `lab.mutations.build_lab`, never by editing this file.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Callable, Literal
|
|
|
|
Permission = Literal["READ", "WRITE"]
|
|
|
|
BASE_VERSION = "lab-0.2.0"
|
|
|
|
|
|
class Denied(Exception):
|
|
"""The enforcement path refused the request."""
|
|
|
|
def __init__(self, reason: str, status: int = 403) -> None:
|
|
super().__init__(reason)
|
|
self.status = status
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class AuditRecord:
|
|
sequence: int
|
|
event: str
|
|
actor_id: str
|
|
resource_id: str | None = None
|
|
subject_id: str | None = None
|
|
permission: str | None = None
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class User:
|
|
id: str
|
|
tenant_id: str
|
|
token: str
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class LabApp:
|
|
"""In-process resource-sharing service.
|
|
|
|
Mutation hooks are plain attributes holding callables or flags. A mutation
|
|
replaces one, so that every mutated build differs from baseline in exactly
|
|
one named way and the difference is inspectable at runtime.
|
|
"""
|
|
|
|
version: str = f"{BASE_VERSION}-baseline"
|
|
applied_mutations: tuple[str, ...] = ()
|
|
|
|
users: dict[str, User] = field(default_factory=dict)
|
|
_tokens: dict[str, str] = field(default_factory=dict)
|
|
resources: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
grants: dict[tuple[str, str], Permission] = field(default_factory=dict)
|
|
audit: list[AuditRecord] = field(default_factory=list)
|
|
_seq: int = 0
|
|
|
|
# --- mutation hooks (baseline behaviour) ----------------------------
|
|
may_read: Callable[["LabApp", str, str], bool] | None = None
|
|
grant_permission_for: Callable[[str | None], Permission] | None = None
|
|
audit_revoke: bool = True
|
|
revoke_delay_seconds: float = 0.0
|
|
denied_status: int = 403
|
|
require_share_acceptance: bool = False
|
|
enforce_tenant_isolation: bool = True
|
|
response_id_field: str = "resource_id"
|
|
latency_seconds: float = 0.0
|
|
|
|
# --- presentation hooks (surface only; never affect domain semantics) ---
|
|
ui_share_control: str = "inline"
|
|
ui_dom_style: str = "flat"
|
|
ui_labels: str = "plain"
|
|
ui_field_order: str = "natural"
|
|
ui_button_element: str = "button"
|
|
ui_test_ids: str = "stable"
|
|
ui_field_names: str = "canonical"
|
|
ui_confirm_revoke: bool = False
|
|
api_path_style: str = "long"
|
|
tenant_sharing_announced: bool = False
|
|
|
|
# -- setup -----------------------------------------------------------
|
|
|
|
def add_user(self, user_id: str, tenant_id: str = "t-acme") -> str:
|
|
token = f"tok-{user_id}"
|
|
self.users[user_id] = User(user_id, tenant_id, token)
|
|
self._tokens[token] = user_id
|
|
return token
|
|
|
|
def _audit(self, event: str, actor_id: str, **kw: Any) -> None:
|
|
self._seq += 1
|
|
self.audit.append(AuditRecord(self._seq, event, actor_id, **kw))
|
|
|
|
def _whoami(self, token: str) -> str:
|
|
if token not in self._tokens:
|
|
raise Denied("unknown token", status=401)
|
|
return self._tokens[token]
|
|
|
|
def tenant_of(self, user_id: str) -> str | None:
|
|
user = self.users.get(user_id)
|
|
return user.tenant_id if user else None
|
|
|
|
# -- enforcement path (what actors use) ------------------------------
|
|
|
|
def request(self, token: str, op: str, **args: Any) -> Any:
|
|
"""The single entry point actors go through. Authorization is enforced here."""
|
|
if self.latency_seconds:
|
|
time.sleep(self.latency_seconds)
|
|
user_id = self._whoami(token)
|
|
handler = getattr(self, f"_op_{op}", None)
|
|
if handler is None:
|
|
raise Denied(f"unknown operation {op!r}", status=404)
|
|
return handler(user_id, **args)
|
|
|
|
def _op_create_resource(self, user_id: str, resource_id: str, content: str) -> dict:
|
|
self.resources[resource_id] = {"owner": user_id, "content": content}
|
|
self._audit("create", user_id, resource_id=resource_id)
|
|
return {self.response_id_field: resource_id}
|
|
|
|
def _op_read_resource(self, user_id: str, resource_id: str) -> dict:
|
|
resource = self.resources.get(resource_id)
|
|
if resource is None:
|
|
raise Denied("no such resource", status=404)
|
|
if not self._may_read(user_id, resource_id):
|
|
raise Denied("not authorized to read", status=self.denied_status)
|
|
return {self.response_id_field: resource_id, "content": resource["content"]}
|
|
|
|
def _op_write_resource(self, user_id: str, resource_id: str, content: str) -> dict:
|
|
resource = self.resources.get(resource_id)
|
|
if resource is None:
|
|
raise Denied("no such resource", status=404)
|
|
if not self._may_write(user_id, resource_id):
|
|
raise Denied("not authorized to write", status=self.denied_status)
|
|
resource["content"] = content
|
|
self._audit("write", user_id, resource_id=resource_id)
|
|
return {"written": True}
|
|
|
|
def _op_grant(
|
|
self,
|
|
user_id: str,
|
|
resource_id: str,
|
|
subject_id: str,
|
|
permission: Permission | None = None,
|
|
) -> dict:
|
|
resource = self.resources.get(resource_id)
|
|
if resource is None or resource["owner"] != user_id:
|
|
raise Denied("only the owner may grant")
|
|
if self.enforce_tenant_isolation:
|
|
if self.tenant_of(subject_id) != self.tenant_of(user_id):
|
|
raise Denied("cross-tenant sharing is not permitted")
|
|
effective = (
|
|
self.grant_permission_for(permission)
|
|
if self.grant_permission_for
|
|
else (permission or "READ")
|
|
)
|
|
self.grants[(resource_id, subject_id)] = effective
|
|
if self.require_share_acceptance:
|
|
self.resources[resource_id].setdefault("pending", set()).add(subject_id)
|
|
self._audit(
|
|
"grant", user_id, resource_id=resource_id,
|
|
subject_id=subject_id, permission=effective,
|
|
)
|
|
return {"granted": effective}
|
|
|
|
def _op_accept_share(self, user_id: str, resource_id: str) -> dict:
|
|
pending = self.resources.get(resource_id, {}).get("pending")
|
|
if pending:
|
|
pending.discard(user_id)
|
|
self._audit("accept", user_id, resource_id=resource_id, subject_id=user_id)
|
|
return {"accepted": True}
|
|
|
|
def _op_revoke(self, user_id: str, resource_id: str, subject_id: str) -> dict:
|
|
resource = self.resources.get(resource_id)
|
|
if resource is None or resource["owner"] != user_id:
|
|
raise Denied("only the owner may revoke")
|
|
if self.revoke_delay_seconds:
|
|
self.resources[resource_id].setdefault("revoke_after", {})[subject_id] = (
|
|
time.monotonic() + self.revoke_delay_seconds
|
|
)
|
|
else:
|
|
self.grants.pop((resource_id, subject_id), None)
|
|
if self.audit_revoke:
|
|
self._audit("revoke", user_id, resource_id=resource_id, subject_id=subject_id)
|
|
return {"revoked": True}
|
|
|
|
def _may_read(self, user_id: str, resource_id: str) -> bool:
|
|
"""Authorization as the system actually enforces it."""
|
|
if self.may_read is not None:
|
|
return self.may_read(self, user_id, resource_id)
|
|
return self.baseline_may_read(user_id, resource_id)
|
|
|
|
def _may_write(self, user_id: str, resource_id: str) -> bool:
|
|
resource = self.resources.get(resource_id)
|
|
if resource is None:
|
|
return False
|
|
if resource["owner"] == user_id:
|
|
return True
|
|
return self.grants.get((resource_id, user_id)) == "WRITE"
|
|
|
|
def baseline_may_read(self, user_id: str, resource_id: str) -> bool:
|
|
resource = self.resources.get(resource_id)
|
|
if resource is None:
|
|
return False
|
|
if resource["owner"] == user_id:
|
|
return True
|
|
deadline = resource.get("revoke_after", {}).get(user_id)
|
|
if deadline is not None and time.monotonic() < deadline:
|
|
return True # revocation is scheduled but not yet effective
|
|
if user_id in resource.get("pending", set()):
|
|
return False
|
|
return (resource_id, user_id) in self.grants
|
|
|
|
|
|
class ObservationChannel:
|
|
"""Independent read access to lab state — decision D-07.
|
|
|
|
Bypasses authorization deliberately. This is the channel test-driver requires
|
|
of any system under test, and the main integration burden the framework
|
|
imposes on an adopter.
|
|
"""
|
|
|
|
def __init__(self, app: LabApp) -> None:
|
|
self._app = app
|
|
|
|
@property
|
|
def version(self) -> str:
|
|
return self._app.version
|
|
|
|
def state_permission(self, user_id: str, resource_id: str) -> str | None:
|
|
"""What the stored record says, independent of any enforcement decision."""
|
|
resource = self._app.resources.get(resource_id)
|
|
if resource is None:
|
|
return None
|
|
if resource["owner"] == user_id:
|
|
return "OWNER"
|
|
return self._app.grants.get((resource_id, user_id))
|
|
|
|
def probe_read(self, user_id: str, resource_id: str) -> bool:
|
|
"""Exercise the enforcement path out-of-band and report what it did.
|
|
|
|
This uses the subject's own credentials, which can look like a violation
|
|
of actor isolation but is not: independence means the *actor's report* is
|
|
never the evidence. The observer issues its own request and records the
|
|
raw outcome. No actor is ever asked whether it succeeded.
|
|
"""
|
|
user = self._app.users.get(user_id)
|
|
if user is None:
|
|
return False
|
|
try:
|
|
self._app.request(user.token, "read_resource", resource_id=resource_id)
|
|
except Denied:
|
|
return False
|
|
return True
|
|
|
|
def probe_write(self, user_id: str, resource_id: str) -> bool:
|
|
"""Out-of-band probe of the write path, non-destructive on refusal."""
|
|
user = self._app.users.get(user_id)
|
|
if user is None:
|
|
return False
|
|
original = self._app.resources.get(resource_id, {}).get("content")
|
|
try:
|
|
self._app.request(
|
|
user.token, "write_resource", resource_id=resource_id, content=original
|
|
)
|
|
except Denied:
|
|
return False
|
|
return True
|
|
|
|
def audit_events(self, resource_id: str) -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"sequence": r.sequence,
|
|
"event": r.event,
|
|
"actor_id": r.actor_id,
|
|
"subject_id": r.subject_id,
|
|
"permission": r.permission,
|
|
}
|
|
for r in self._app.audit
|
|
if r.resource_id == resource_id
|
|
]
|
|
|
|
|
|
def build_baseline() -> tuple[LabApp, dict[str, str]]:
|
|
"""Known initial state, so that runs replay from the same starting point."""
|
|
app = LabApp()
|
|
tokens = {u: app.add_user(u) for u in ("alice", "bob", "carol")}
|
|
tokens["mallory"] = app.add_user("mallory", tenant_id="t-other")
|
|
return app, tokens
|