diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 9185eb1..d9e71d0 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -36,7 +36,13 @@ jobs: # Positive control for the cost collector (AC-5..AC-9). Does not # gate on a dollar figure — transcripts are not present in CI — but # proves the collector still detects the failures it claims to. - - run: make cost-test + - run: make self-tests + + # InnerLoop rules made executable (CB-WP-0003 T01). Fails on an + # overlong artifact, a survey missing its tier/chaos declaration, an + # approved tier-L survey with no review trail, `unmeasured` in an + # evidence table, or a reporting tool with no --self-test. + - run: make loop-lint # InnerLoop v1.0 positive control, enforced rather than asserted in # prose: --test runs every benchmark once, so a workload that diff --git a/Makefile b/Makefile index 46e9e79..ce08512 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ CARGO := cargo -.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin loc all +.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin loop-lint self-tests loc all ## fmt + clippy (deny warnings) + HashMap deny-lint check: @@ -29,6 +29,17 @@ cost: cost-test cost-test: python3 tools/cb-cost.py --self-test +# InnerLoop rules that are mechanically checkable (CB-WP-0003 T01). +loop-lint: + python3 tools/loop-lint.py + +# Positive control for every reporting tool, per InnerLoop v1.1 Step 5. +self-tests: + python3 tools/cb-cost.py --self-test + python3 tools/loop-lint.py --self-test + python3 tools/rule-coverage.py --self-test + python3 tools/dep-weight.py --self-test + cost-pin: cost-test python3 tools/cb-cost.py --pin fc76445 --composition --by-task @@ -51,4 +62,4 @@ loc: printf '%-28s %s\n' $$d "$$(find $$d/src -name '*.rs' | xargs cat | grep -vcE '^\s*(//|$$)')"; \ done -all: check test sim coverage dep-weight cost-test bench-test +all: check test sim coverage dep-weight self-tests loop-lint bench-test diff --git a/history/260731-inner-loop-rule-audit.md b/history/260731-inner-loop-rule-audit.md new file mode 100644 index 0000000..96203dd --- /dev/null +++ b/history/260731-inner-loop-rule-audit.md @@ -0,0 +1,148 @@ +# 2026-07-31 — InnerLoop v1.1 rule enforceability audit + +CB-WP-0003 T01. Every rule in `specs/InnerLoop.md` v1.1 classified as +**executable** (a command fails when it is violated), **checkable** (a +human or agent can verify it cheaply and objectively in review), or +**decorative** (neither) — and, per the revised task, tagged with the +**failure class it catches**. + +## Failure classes on record + +Seven error instances across three classes, from +`history/260731-inner-loop-retrospective.md` and +`history/260731-cost-accounting-retrospective.md`: + +| tag | class | instances | +|---|---|---| +| **HDN** | harness-does-nothing — a run succeeds while performing no work | 4 | +| **TA** | trusted arithmetic — a correct-looking sum over data that really exists | 2 | +| **SSB** | same-sample blind spot — a property verified on the large sample, assumed on the small | 1 | + +The reason for tagging: a rule that catches a class no observed error +belongs to is a candidate for deletion even when it is perfectly +executable, and a class with no rule covering it is where the next rule +should go. + +## The audit + +Status after this pass. `loop-lint` = `tools/loop-lint.py`, new here. + +| # | Rule | Class | Status | Enforced by | +|---|---|---|---|---| +| 1 | No implementation code before the ADR is committed | — | **checkable** | git history vs ADR date; not automated (see §Deferred) | +| 2 | Tier declared before work starts | — | **executable** | `loop-lint` tier-declared | +| 3 | Chaos roll recorded every time, even when it changes nothing | — | **executable** | `loop-lint` chaos-recorded | +| 4 | Invariants bind at every tier regardless of roll | — | executable (elsewhere) | `make all` — determinism tests, `disallowed_types` | +| 5 | Survey names a benchmark-to-beat per dimension | — | **checkable** | review; a regex cannot judge whether a number is a benchmark | +| 6 | Cited-only baselines cap the verdict at `parity` | TA | **decorative → checkable** | see §Changes | +| 7 | Tier-L survey gets one round of adversarial review | SSB | **executable** | `loop-lint` review-trail | +| 8 | Review trail: research/challenge/response in `history/` | SSB | **executable** | `loop-lint` review-trail | +| 9 | ADR states expected advantage per dimension | — | **checkable** | review | +| 10 | Spec carries an acceptance-metrics table | — | **checkable** | review | +| 11 | Every metric names its instrument | HDN | **checkable** | review; naming is textual, existence is not | +| 12 | **v1.1** The instrument must exist and emit its own target | TA | **checkable** | review; see §Deferred for why not yet executable | +| 13 | **v1.1** Inherited numbers are re-derived before use as a target | TA | **decorative** | nothing can detect a quoted number | +| 14 | A metric is checked against contracts in its own spec | — | **checkable** | review | +| 15 | Harness asserts it performed the work it reports | HDN | **executable** | `cargo bench -- --test`, cb-sim empty-run, `--self-test` | +| 16 | Fail loudly rather than report when the assertion fails | HDN | **executable** | same | +| 17 | Divisor pinned by a test | HDN | **executable** | `bench_shape` test module | +| 18 | A number that cannot prove its work is void | HDN | **executable** | same as 15 | +| 19 | **v1.1** Every reporting tool exposes `--self-test` | HDN, SSB | **executable** | `loop-lint` self-test | +| 20 | `--self-test` names a failure it detects, not the happy path | HDN | **checkable** | review; see §Deferred | +| 21 | Evidence names its disanalogies | — | **checkable** | review | +| 22 | `unmeasured` illegal in an evidence file | — | **executable** | `loop-lint` evidence-unmeasured | +| 23 | No silently-ignored input | HDN | **checkable** | review | +| 24 | Decisions get commands, not defaults | — | **checkable** | review | +| 25 | Scaffolds are exercised or marked | HDN | **checkable** | review | +| 26 | Coverage gates that count tags say so | — | **checkable** | review | +| 27 | Whole-file loadability (~400 lines) | — | **executable** | `loop-lint` loadability | +| 28 | Structured over prose | — | **decorative** | kept as guidance, marked | +| 29 | One command surface | — | **checkable** | review | +| 30 | Self-contained tasks | — | **checkable** | review | +| 31 | Evidence or it didn't happen | — | **checkable** | review | +| 32 | Token discipline per the global budget policy | — | **decorative — dead** | nothing; CB-WP-0003 T05 replaces it | +| 33–41 | Definition-of-done checklist (9 items) | mixed | **checkable** | review; each maps to an artifact whose existence is testable — see §Deferred | + +Counts: **11 executable**, **22 checkable**, **4 decorative** (one of +which is dead policy). + +## What the audit found by running + +`tools/loop-lint.py` was written to make rules 2, 3, 7, 8, 19, 22 and 27 +executable. On its first run it produced **three findings, all real, none +previously visible**: + +1. **`specs/ArchitectureBlueprint.md` is 543 lines** against a ~400-line + limit the loop has stated since v0.2. Nobody noticed because nothing + measured it. Disposition: split (§Changes). +2. **`tools/dep-weight.py` has no `--self-test`.** It has positive-control + logic — it refuses to report when a crate cannot be located — but + nothing verifies that control still works. +3. **`tools/rule-coverage.py` has no `--self-test`.** Same shape. + +Findings 2 and 3 are exactly the recursion this workplan is about: the +positive-control rule was applied to benchmarks and to the newest tool, +and not to the two older tools that report AM-1 and AM-4 numbers into +evidence files. + +## Changes made in this task + +- **Rule 27 made executable and the violation fixed.** + `ArchitectureBlueprint.md` split at its own section boundaries into + `ArchitectureBlueprint.md` (§1–8, the stack) and + `ArchitectureRuntime.md` (§9–15, runtime/tooling/process), linked both + ways. +- **Rules 2, 3, 7, 8, 19, 22 made executable** via `tools/loop-lint.py`, + wired as `make loop-lint` and into `make all` and CI. +- **Rule 6 (parity cap) demoted from decorative to checkable** by stating + the check explicitly: an evidence row citing a baseline whose provenance + is `cited` may not carry verdict `better`. That is mechanical against the + survey's provenance column and is queued for `loop-lint` once a second + evidence file exists to test it against — writing a matcher with one + sample is the SSB error this pass is trying to stop making. +- **Rule 28 (structured over prose) demoted to guidance**, with that + status stated in the spec. It is a style preference; nothing can judge + it, and leaving it phrased as a requirement is the false assurance this + audit exists to remove. +- **Rule 32 (token discipline) marked dead** in place, pointing at + CB-WP-0003 T05. It is not deleted yet because deleting it is T05's + decision, but it now reads as dead rather than as a live control. + +## Deferred, with reasons + +- **Rule 1 (ADR gate) is checkable but not automated.** A mechanical check + needs a map from capability → source paths, which does not exist. Cheap + version worth doing later: require every `specs/.md` to reference an + ADR, and every ADR to precede the first commit touching its capability's + directory. +- **Rule 12 (instrument emits its target) resists automation for now.** + Detecting "this number was typed rather than emitted" requires the tool's + output to be committed alongside the spec. The tractable form is to + require acceptance targets to appear verbatim in a committed tool output + file; deferred to T02 rather than guessed at here. +- **Rule 20 (self-test names a failure) is genuinely checkable only.** A + self-test that asserts `True` passes any structural check. Reviewing the + assertions is the control, and CB-WP-0002's AC-9 is the model: pin the + exact defect that occurred. +- **Rule 13 (re-derive inherited numbers) has no mechanical form at all** + and is left decorative *deliberately* — with its status stated. It is + the rule that caught the largest error in CB-WP-0002 ($248.46 → $92.21), + which is the counter-example to this workplan's own hypothesis: **an + unenforceable rule was the most valuable one in the pass.** The + hypothesis "a rule that cannot be executed is not a rule" is therefore + wrong as stated. The correct version is narrower: *a rule that cannot be + executed cannot be relied on to fire, so it must not be the only defence + for a class that matters.* + +## Class coverage — where the gaps are + +| class | executable rules covering it | assessment | +|---|---|---| +| HDN | 15, 16, 17, 18, 19 | **well covered.** Five executable rules; the class that started this. | +| TA | none | **uncovered by any executable rule.** Rules 6, 12, 13 are checkable or decorative. This is the largest gap, and it is the class with the second-most instances. | +| SSB | 7, 8, 19 | **partly covered.** Rule 19 caught the one instance, by accident of running over all data rather than by design. Rules 7/8 enforce that a review *happened*, not that it sampled differently. T03 addresses the design gap. | + +The honest read: **the loop is hardened against the class it has already +suffered most from, and has no executable defence against the class that +produced its largest single error.** Trusted arithmetic is caught today +only by re-derivation, which is a discipline, not a gate. diff --git a/specs/ArchitectureBlueprint.md b/specs/ArchitectureBlueprint.md index fbbdd1a..6d0ed3a 100644 --- a/specs/ArchitectureBlueprint.md +++ b/specs/ArchitectureBlueprint.md @@ -331,213 +331,14 @@ capabilities. --- -## 9. Rendering, input, and creation tools - -Rust-first stack: - -- `winit` — windows and platform input -- `wgpu` — GPU rendering (Vulkan / Metal / D3D / browser) -- `egui` — engine tools, inspectors, early editors -- Custom scene renderer for the game table -- **glTF** as the primary imported 3D format, wrapped in Clay-Borg asset - metadata and provenance - -Creator environment (grows over milestones): scene hierarchy, object -inspector, prototype browser, card-sheet importer, deck builder, zone and -snap-point editors, rule-state inspector, event timeline, player-view -switcher, hidden-information debugger, physics debugger, scenario recorder, -replay controls, package validator. - --- -## 10. Networking and sessions +**Continued in [ArchitectureRuntime.md](ArchitectureRuntime.md)** — §9 +rendering/input/creation tools, §10 networking and sessions, §11 the +agentic inner loop, §12 TargetRevenue integration, §13 repository +structure, §14 milestones, §15 governing design decisions. -Authoritative session host: - -```text -Client gesture -→ proposed command -→ session server validation -→ authoritative events -→ state update -→ player-specific projection -→ client animation -``` - -Capabilities: session discovery, auth and seat assignment, lobby/readiness, -command submission, commit/reveal windows, event-stream replication, -snapshot transfer, reconnection, state-hash verification, spectators, -player-specific redaction, host migration (later). - -Transport: **Quinn** (QUIC) for native; browser transport is a separate -adapter (WebTransport or WebSockets). The canonical protocol is defined -independently of any transport: - -```text -cb-session-protocol -├── CommandEnvelope -├── EventEnvelope -├── SnapshotEnvelope -├── CommitmentEnvelope -├── AssetRequest -└── CapabilityNegotiation -``` - ---- - -## 11. Agentic inner loop - -Agentic coding is a first-class product surface. Optimize for small -capability boundaries, executable specifications, controlled work areas, and -replayable failures. - -### Work packet (every agent task) - -```yaml -task_id: CB-PHYS-0042 -capability: tabletop.card-stacking -intent: Keep card stacks stable after drag release. -allowed_crates: - - cb-physics-api - - cb-physics-rapier - - cb-tabletop-physics -forbidden_changes: - - canonical game event schema -invariants: - - semantic card order must not depend on collider order -scenarios: - - scenarios/card-stack-20.yaml -benchmarks: - - benches/card-stack-stability.yaml -acceptance: - - all conformance tests pass - - no state divergence over 10,000 ticks - - benchmark regression below 3% -``` - -### CLI surface (`cb`) - -```bash -cb inspect capability tabletop.card -cb task prepare CB-PHYS-0042 -cb generate contracts -cb check --affected -cb test --affected # supports --format json -cb sim ground scenarios/mutual-attack.yaml -cb play ground --players 4 -cb replay artifacts/failure.cbreplay -cb compare physics-reference physics-rapier -cb bench --affected -cb evidence build CB-PHYS-0042 -cb release assess CB-PHYS-0042 -``` - -### Quality gates - -Formatting/linting, dependency-policy check, unit tests, capability -conformance tests, property tests, golden scenario tests, replay -determinism, snapshot migration, performance and memory budgets, rendering -comparison where relevant, security/sandbox tests, documentation and schema -consistency. - -Tooling: `cargo-nextest` (isolated parallel tests), Criterion -(regression-sensitive benchmarks), `sccache` (compile reuse), `tracing` -(structured diagnostics). - ---- - -## 12. TargetRevenue integration - -TargetRevenue governs **versioned capability improvements**, not the -monorepo as one indivisible target. - -```toml -improvement_id = "CB-GROUND-001" -capability = "game.ground.simultaneous-resolution" -classification = "10x" - -estimated_days = 4 -daily_rate = 1000 -target_revenue = 40000 - -phase = "commercial-recovery" -release_when_target_reached = "MIT" -trust_record = "required" -``` - -Components: improvement registry, workload ledger, cost model, revenue -attribution, dependency graph, phase license generator, revenue meter, -release gate, evidence bundle, trust service. - -Economic rule: - -> Optimized assimilations may be financed as independent improvements, while -> the canonical interface remains stable and reusable. - ---- - -## 13. Repository structure - -```text -clay-borg/ -├── INTENT.md -├── SCOPE.md -├── ARCHITECTURE.md # or specs/ArchitectureBlueprint.md (this file) -├── Cargo.toml -├── rust-toolchain.toml -│ -├── canon/ # entities, events, capabilities, schemas, terminology -├── crates/ -│ ├── cb-kernel/ cb-ids/ cb-time/ cb-rng/ cb-events/ -│ ├── cb-snapshot/ cb-capability/ -│ ├── cb-world/ cb-world-api/ cb-ecs-bevy/ -│ ├── cb-physics-api/ cb-physics-null/ cb-physics-reference/ cb-physics-rapier/ -│ ├── cb-render-api/ cb-render-null/ cb-render-wgpu/ -│ ├── cb-tabletop/ cb-tabletop-physics/ cb-tabletop-view/ -│ ├── cb-game-runtime/ cb-game-protocol/ cb-game-wasm/ -│ ├── cb-session/ cb-network-api/ cb-network-loopback/ cb-network-quic/ -│ └── cb-assets/ cb-ui/ cb-editor/ cb-observe/ cb-evidence/ -│ -├── games/ # ground/, fixture-cards/ -├── tools/ # cb-cli/, cb-agent/, cb-import/, cb-pack/ -├── scenarios/ -├── conformance/ -├── benchmarks/ -├── replays/ -├── examples/ -├── decisions/ -├── assimilation/ # assimilation manifests -└── target-revenue/ -``` - -Stay a monorepo during architectural formation. Extract a repository only -when a capability has a stable contract, an independent lifecycle, and a -genuine external consumer. - ---- - -## 14. Milestones - -| # | Milestone | Proves | -|---|---|---| -| 0 | Headless GROUND | Authoritative rules, commit/reveal, DARVO, replay — no rendering, no physics | -| 1 | Inspectable 2D table | Presentation bindings without 3D complexity | -| 2 | Physical 3D tabletop | wgpu + Rapier projection of semantic events | -| 3 | Networked sessions | Authoritative host, private projections, reconnection | -| 4 | Game creation framework | Editors, importers, Wasm game components | -| 5 | Second fixture game | Generality — abstractions promoted to Canon only after a second concrete use | - ---- - -## 15. Governing design decisions - -1. Build GROUND first, not a general engine first. -2. Keep rules independent from rendering and physics. -3. Use commands and events as the authoritative mutation mechanism. -4. Provide null, reference, and optimized implementations of important capabilities. -5. Never leak assimilated-library types into canonical interfaces. -6. Use server-authoritative physics and deterministic semantic rules. -7. Treat player visibility as a projection, not a UI afterthought. -8. Make every defect reproducible as a scenario and replay. -9. Give coding agents bounded work packets and stable commands. -10. Attach TargetRevenue phases to versioned improvements and evidence bundles. +*(Split 2026-07-31: this file was 543 lines against the loop's ~400-line +whole-file loadability rule. The rule had been stated since v0.2 and +nothing measured it until `tools/loop-lint.py` — see +`history/260731-inner-loop-rule-audit.md`.)* diff --git a/specs/ArchitectureRuntime.md b/specs/ArchitectureRuntime.md new file mode 100644 index 0000000..6d08b53 --- /dev/null +++ b/specs/ArchitectureRuntime.md @@ -0,0 +1,221 @@ +# Clay-Borg Architecture — Runtime, Tooling, and Process + +Second half of the architecture blueprint, split from +[ArchitectureBlueprint.md](ArchitectureBlueprint.md) on 2026-07-31 for +whole-file loadability. §1–8 (the layered stack, Clay Canon, runtime +substrate, simulation kernel, physics, world-building, tabletop domain +framework, game runtime) remain there; §9–15 are here. + +Section numbering is continuous with the first half and deliberately +unchanged, so existing references keep resolving. + +## 9. Rendering, input, and creation tools + +Rust-first stack: + +- `winit` — windows and platform input +- `wgpu` — GPU rendering (Vulkan / Metal / D3D / browser) +- `egui` — engine tools, inspectors, early editors +- Custom scene renderer for the game table +- **glTF** as the primary imported 3D format, wrapped in Clay-Borg asset + metadata and provenance + +Creator environment (grows over milestones): scene hierarchy, object +inspector, prototype browser, card-sheet importer, deck builder, zone and +snap-point editors, rule-state inspector, event timeline, player-view +switcher, hidden-information debugger, physics debugger, scenario recorder, +replay controls, package validator. + +--- + +## 10. Networking and sessions + +Authoritative session host: + +```text +Client gesture +→ proposed command +→ session server validation +→ authoritative events +→ state update +→ player-specific projection +→ client animation +``` + +Capabilities: session discovery, auth and seat assignment, lobby/readiness, +command submission, commit/reveal windows, event-stream replication, +snapshot transfer, reconnection, state-hash verification, spectators, +player-specific redaction, host migration (later). + +Transport: **Quinn** (QUIC) for native; browser transport is a separate +adapter (WebTransport or WebSockets). The canonical protocol is defined +independently of any transport: + +```text +cb-session-protocol +├── CommandEnvelope +├── EventEnvelope +├── SnapshotEnvelope +├── CommitmentEnvelope +├── AssetRequest +└── CapabilityNegotiation +``` + +--- + +## 11. Agentic inner loop + +Agentic coding is a first-class product surface. Optimize for small +capability boundaries, executable specifications, controlled work areas, and +replayable failures. + +### Work packet (every agent task) + +```yaml +task_id: CB-PHYS-0042 +capability: tabletop.card-stacking +intent: Keep card stacks stable after drag release. +allowed_crates: + - cb-physics-api + - cb-physics-rapier + - cb-tabletop-physics +forbidden_changes: + - canonical game event schema +invariants: + - semantic card order must not depend on collider order +scenarios: + - scenarios/card-stack-20.yaml +benchmarks: + - benches/card-stack-stability.yaml +acceptance: + - all conformance tests pass + - no state divergence over 10,000 ticks + - benchmark regression below 3% +``` + +### CLI surface (`cb`) + +```bash +cb inspect capability tabletop.card +cb task prepare CB-PHYS-0042 +cb generate contracts +cb check --affected +cb test --affected # supports --format json +cb sim ground scenarios/mutual-attack.yaml +cb play ground --players 4 +cb replay artifacts/failure.cbreplay +cb compare physics-reference physics-rapier +cb bench --affected +cb evidence build CB-PHYS-0042 +cb release assess CB-PHYS-0042 +``` + +### Quality gates + +Formatting/linting, dependency-policy check, unit tests, capability +conformance tests, property tests, golden scenario tests, replay +determinism, snapshot migration, performance and memory budgets, rendering +comparison where relevant, security/sandbox tests, documentation and schema +consistency. + +Tooling: `cargo-nextest` (isolated parallel tests), Criterion +(regression-sensitive benchmarks), `sccache` (compile reuse), `tracing` +(structured diagnostics). + +--- + +## 12. TargetRevenue integration + +TargetRevenue governs **versioned capability improvements**, not the +monorepo as one indivisible target. + +```toml +improvement_id = "CB-GROUND-001" +capability = "game.ground.simultaneous-resolution" +classification = "10x" + +estimated_days = 4 +daily_rate = 1000 +target_revenue = 40000 + +phase = "commercial-recovery" +release_when_target_reached = "MIT" +trust_record = "required" +``` + +Components: improvement registry, workload ledger, cost model, revenue +attribution, dependency graph, phase license generator, revenue meter, +release gate, evidence bundle, trust service. + +Economic rule: + +> Optimized assimilations may be financed as independent improvements, while +> the canonical interface remains stable and reusable. + +--- + +## 13. Repository structure + +```text +clay-borg/ +├── INTENT.md +├── SCOPE.md +├── ARCHITECTURE.md # or specs/ArchitectureBlueprint.md (this file) +├── Cargo.toml +├── rust-toolchain.toml +│ +├── canon/ # entities, events, capabilities, schemas, terminology +├── crates/ +│ ├── cb-kernel/ cb-ids/ cb-time/ cb-rng/ cb-events/ +│ ├── cb-snapshot/ cb-capability/ +│ ├── cb-world/ cb-world-api/ cb-ecs-bevy/ +│ ├── cb-physics-api/ cb-physics-null/ cb-physics-reference/ cb-physics-rapier/ +│ ├── cb-render-api/ cb-render-null/ cb-render-wgpu/ +│ ├── cb-tabletop/ cb-tabletop-physics/ cb-tabletop-view/ +│ ├── cb-game-runtime/ cb-game-protocol/ cb-game-wasm/ +│ ├── cb-session/ cb-network-api/ cb-network-loopback/ cb-network-quic/ +│ └── cb-assets/ cb-ui/ cb-editor/ cb-observe/ cb-evidence/ +│ +├── games/ # ground/, fixture-cards/ +├── tools/ # cb-cli/, cb-agent/, cb-import/, cb-pack/ +├── scenarios/ +├── conformance/ +├── benchmarks/ +├── replays/ +├── examples/ +├── decisions/ +├── assimilation/ # assimilation manifests +└── target-revenue/ +``` + +Stay a monorepo during architectural formation. Extract a repository only +when a capability has a stable contract, an independent lifecycle, and a +genuine external consumer. + +--- + +## 14. Milestones + +| # | Milestone | Proves | +|---|---|---| +| 0 | Headless GROUND | Authoritative rules, commit/reveal, DARVO, replay — no rendering, no physics | +| 1 | Inspectable 2D table | Presentation bindings without 3D complexity | +| 2 | Physical 3D tabletop | wgpu + Rapier projection of semantic events | +| 3 | Networked sessions | Authoritative host, private projections, reconnection | +| 4 | Game creation framework | Editors, importers, Wasm game components | +| 5 | Second fixture game | Generality — abstractions promoted to Canon only after a second concrete use | + +--- + +## 15. Governing design decisions + +1. Build GROUND first, not a general engine first. +2. Keep rules independent from rendering and physics. +3. Use commands and events as the authoritative mutation mechanism. +4. Provide null, reference, and optimized implementations of important capabilities. +5. Never leak assimilated-library types into canonical interfaces. +6. Use server-authoritative physics and deterministic semantic rules. +7. Treat player visibility as a projection, not a UI afterthought. +8. Make every defect reproducible as a scenario and replay. +9. Give coding agents bounded work packets and stable commands. +10. Attach TargetRevenue phases to versioned improvements and evidence bundles. diff --git a/specs/InnerLoop.md b/specs/InnerLoop.md index 178bd2a..4ae75f5 100644 --- a/specs/InnerLoop.md +++ b/specs/InnerLoop.md @@ -151,11 +151,12 @@ as a target, or it is cited as unverified.** Quoting is not measuring. from a prior pass. Re-derivation put it at $92.21 — the quoted figure double-counted transcript lines and priced a three-model session at one model's rate. Neither error was of the harness-does-nothing class; both -sums ran over real data, and a positive control would have passed them.)* A metric must -also be checked against the contracts in the *same spec*: if a contract -makes a target unreachable, one of the two is wrong and the conflict is -resolved when it is noticed, not at the acceptance run. Re-check the -table whenever a contract is added. +sums ran over real data, and a positive control would have passed them.)* + +**A metric is checked against the contracts in its own spec.** If a +contract makes a target unreachable, one of the two is wrong and the +conflict is resolved when it is noticed, not at the acceptance run. +Re-check the table whenever a contract is added. *(v1.0, from CB-WP-0001: AM-4's ≤20-crate target was made unreachable by the K5 and K7 contracts written after it, and AM-12's cost metric was @@ -297,8 +298,9 @@ The loop exists to be driven by agents. Therefore: 1. **Whole-file loadability** — every loop artifact stays under ~400 lines; split before exceeding, link with relative paths. -2. **Structured over prose** — tables and fenced blocks for anything a - later step must parse (baselines, acceptance metrics, evidence rows). +2. **Structured over prose** *(guidance, not a requirement — nothing can + check it)* — tables and fenced blocks for anything a later step must + parse (baselines, acceptance metrics, evidence rows). 3. **One command surface** — all checks runnable through repo-root commands (eventually `cb *`; until then, `make`/`cargo` aliases declared in one place), each supporting deterministic, greppable output. @@ -307,9 +309,20 @@ The loop exists to be driven by agents. Therefore: task text plus linked files alone. 5. **Evidence or it didn't happen** — claims of "better" live in committed evidence files with numbers, never only in commit messages or chat. -6. **Token discipline** — per the global budget policy, a loop iteration +6. ~~**Token discipline** — per the global budget policy, a loop iteration that exceeds its budget without measurable progress is stopped and - decomposed, not pushed through. + decomposed, not pushed through.~~ **DEAD POLICY.** The 8k/10k per-task + token budget was never referenced or enforced, and CB-WP-0001 T08 + exceeded it by orders of magnitude with no signal. It implies a control + that does not exist. Replacement in USD is CB-WP-0003 T05; until then + this is documentation of a gap, not a rule. + +**Enforcement status.** Rules above that a command can check are enforced +by `make loop-lint`; the full classification of every InnerLoop rule as +executable / checkable / decorative, with the failure class each catches, +is in `history/260731-inner-loop-rule-audit.md`. Rules marked *guidance* +or *dead* say so where they appear, so a reader can tell a requirement +from a preference without consulting the audit. --- @@ -335,4 +348,3 @@ A capability has completed the loop when all of the following are committed: ([CostAccounting.md](CostAccounting.md)) - [ ] retrospective note (may be one paragraph appended to the evidence file): what the loop itself should change -``` diff --git a/tools/dep-weight.py b/tools/dep-weight.py index dbcb597..f5df6b3 100755 --- a/tools/dep-weight.py +++ b/tools/dep-weight.py @@ -16,7 +16,7 @@ that cannot be found is reported and the run exits non-zero rather than silently under-reporting the total — under-reporting is the exact direction this metric could be gamed. -Usage: python3 tools/dep-weight.py [--json] +Usage: python3 tools/dep-weight.py [--json] [--self-test] """ import glob @@ -88,7 +88,50 @@ def source_lines(name, version): return 0 +def self_test(): + """Each assertion pins a failure this tool must detect. + + The controls that matter here are: an unlocatable crate must not be + silently counted as zero lines (that under-reports, the direction this + metric could be gamed), and a target breach must fail rather than + merely print. + """ + results = [] + + def check(name, ok, detail=""): + results.append((name, ok, detail)) + + # A crate that does not exist must measure zero, so the caller's + # `lines == 0` guard fires rather than silently shrinking the total. + check("unlocatable crate measures zero (so the guard fires)", + source_lines("definitely-not-a-real-crate-xyz", "9.9.9") == 0) + + # A crate we do depend on must measure non-zero, or the guard above + # would fire on everything and the tool would never report at all. + real = source_lines("serde", "1") + check("a real vendored crate measures non-zero", real > 0, + f"{real:,} lines") + + # Targets must be present and numeric — a missing target would make + # the breach check vacuous. + check("targets defined for every configuration", + set(TARGETS) == set(CONFIGS) and all( + isinstance(v, int) and v > 0 for v in TARGETS.values()), + f"{TARGETS}") + + print("dep-weight self-test (positive control)") + ok = True + for name, passed, detail in results: + print(f" [{'ok ' if passed else 'FAIL'}] {name}" + + (f" — {detail}" if detail else "")) + ok &= passed + return 0 if ok else 1 + + def main(): + if "--self-test" in sys.argv: + return self_test() + report = {} missing = [] for label, args in CONFIGS.items(): diff --git a/tools/loop-lint.py b/tools/loop-lint.py new file mode 100644 index 0000000..d57b9a0 --- /dev/null +++ b/tools/loop-lint.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +"""Executable checks for specs/InnerLoop.md rules (CB-WP-0003 T01). + +The loop's own rules were prose. A rule nobody can run is a suggestion, +and the audit in history/260731-inner-loop-rule-audit.md found several +that were already being violated with no signal. This makes the +mechanically-checkable ones fail a command. + +Each check names the InnerLoop rule it enforces. Checks that cannot be +made mechanical are recorded in the audit as `checkable` or `decorative` +and are deliberately absent here — see the audit for why. + +Positive control (InnerLoop v1.1 §Step 5): --self-test asserts each check +actually detects its failure, using fixtures with known answers. A linter +that passes everything because its matcher is broken is the same defect +class as a benchmark timing rejected work. + +Usage: + python3 tools/loop-lint.py # lint the repo + python3 tools/loop-lint.py --self-test # positive control +""" + +import os +import re +import sys + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LOADABILITY_LIMIT = 400 + +# Artifact classes the loop produces. history/ is an append-only trail +# (verbatim challenge text is not something to split), so it is exempt. +LOOP_DIRS = ("specs", "research", "decisions", "evidence", "workplans") + + +class Finding: + def __init__(self, rule, path, detail): + self.rule, self.path, self.detail = rule, path, detail + + def __str__(self): + return f" [{self.rule}] {self.path}\n {self.detail}" + + +def _md_files(root=REPO): + """Loop artifacts only. + + Vendored third-party trees (a baseline harness ships its own + node_modules) are not artifacts this loop produces, and linting them + buries the two real findings under eighteen irrelevant ones. + """ + for d in LOOP_DIRS: + base = os.path.join(root, d) + for dirpath, dirnames, files in os.walk(base): + dirnames[:] = [x for x in dirnames if x != "node_modules"] + for f in sorted(files): + if f.endswith(".md"): + yield os.path.relpath(os.path.join(dirpath, f), root) + + +def check_loadability(root=REPO): + """§Agentic-efficiency 1 — every loop artifact stays under ~400 lines.""" + out = [] + for rel in _md_files(root): + with open(os.path.join(root, rel)) as fh: + n = sum(1 for _ in fh) + if n > LOADABILITY_LIMIT: + out.append( + Finding( + "loadability", + rel, + f"{n} lines exceeds the ~{LOADABILITY_LIMIT}-line limit; " + f"split and link with relative paths", + ) + ) + return out + + +def check_evidence_no_unmeasured(root=REPO): + """§Rubric — `unmeasured` is legal in a survey, illegal in an evidence file.""" + out = [] + base = os.path.join(root, "evidence") + if not os.path.isdir(base): + return out + for f in sorted(os.listdir(base)): + if not f.endswith(".md"): + continue + rel = os.path.join("evidence", f) + for i, line in enumerate(open(os.path.join(root, rel)), 1): + # A row asserting the verdict, not prose discussing the word. + if re.search(r"\|\s*unmeasured\s*\|", line): + out.append( + Finding("evidence-unmeasured", f"{rel}:{i}", + "verdict `unmeasured` in an evidence table") + ) + return out + + +def check_survey_tier_and_chaos(root=REPO): + """§Loop tiers — tier declared, and the chaos roll recorded every time.""" + out = [] + base = os.path.join(root, "research") + if not os.path.isdir(base): + return out + for f in sorted(os.listdir(base)): + if not f.endswith(".md"): + continue + rel = os.path.join("research", f) + text = open(os.path.join(root, rel)).read() + if not re.search(r"^tier:\s*[SML]\b", text, re.M): + out.append(Finding("tier-declared", rel, "no `tier:` declaration")) + elif "chaos" not in text.lower(): + out.append( + Finding("chaos-recorded", rel, + "tier declared without the chaos roll; the rule requires " + "recording it even when it changes nothing") + ) + return out + + +def check_review_trail(root=REPO): + """§Step 2 — a tier-L survey carries research/challenge/response history.""" + out = [] + base = os.path.join(root, "research") + hist = os.path.join(root, "history") + if not (os.path.isdir(base) and os.path.isdir(hist)): + return out + files = os.listdir(hist) + for f in sorted(os.listdir(base)): + if not f.endswith(".md"): + continue + rel = os.path.join("research", f) + text = open(os.path.join(root, rel)).read() + if not re.search(r"^tier:\s*L\b", text, re.M): + continue + if not re.search(r"^status:\s*approved", text, re.M): + continue + for kind in ("challenge", "response"): + if not any(x.endswith(f"-{kind}.md") and kind in x for x in files): + out.append( + Finding("review-trail", rel, + f"tier-L approved survey with no history/*-{kind}.md") + ) + return out + + +def check_reporting_tools_self_test(root=REPO): + """§Step 5 v1.1 — every tool that reports a number exposes --self-test.""" + out = [] + base = os.path.join(root, "tools") + if not os.path.isdir(base): + return out + for f in sorted(os.listdir(base)): + if not f.endswith(".py"): + continue + rel = os.path.join("tools", f) + text = open(os.path.join(root, rel)).read() + if "--self-test" not in text: + out.append( + Finding("self-test", rel, + "reporting tool with no --self-test entry point; " + "nothing verifies its positive control still works") + ) + return out + + +CHECKS = ( + check_loadability, + check_evidence_no_unmeasured, + check_survey_tier_and_chaos, + check_review_trail, + check_reporting_tools_self_test, +) + + +def self_test(): + """Each check must DETECT its failure, not merely run.""" + import shutil + import tempfile + + results = [] + + def check(name, ok, detail=""): + results.append((name, ok, detail)) + + tmp = tempfile.mkdtemp() + try: + for d in LOOP_DIRS + ("tools", "history"): + os.makedirs(os.path.join(tmp, d), exist_ok=True) + + # loadability: 401 lines must trip, 400 must not. + with open(os.path.join(tmp, "specs", "Big.md"), "w") as fh: + fh.write("x\n" * (LOADABILITY_LIMIT + 1)) + with open(os.path.join(tmp, "specs", "Ok.md"), "w") as fh: + fh.write("x\n" * LOADABILITY_LIMIT) + f = check_loadability(tmp) + check("loadability detects overlong artifact", + len(f) == 1 and "Big.md" in f[0].path, + f"{len(f)} finding(s)") + + # evidence: a table verdict trips; the word in prose does not. + with open(os.path.join(tmp, "evidence", "E.md"), "w") as fh: + fh.write("| AC-1 | x | unmeasured |\n" + "the word unmeasured appearing in prose is fine\n") + f = check_evidence_no_unmeasured(tmp) + check("evidence-unmeasured detects a table verdict, not prose", + len(f) == 1, f"{len(f)} finding(s), expected exactly 1") + + # tier/chaos: missing tier trips; tier without chaos trips. + with open(os.path.join(tmp, "research", "A.md"), "w") as fh: + fh.write("# survey\nno tier here\n") + with open(os.path.join(tmp, "research", "B.md"), "w") as fh: + fh.write("tier: L (structural L)\n") + f = check_survey_tier_and_chaos(tmp) + rules = sorted(x.rule for x in f) + check("tier/chaos detects both omissions", + rules == ["chaos-recorded", "tier-declared"], f"{rules}") + + # self-test: a tool without the flag trips. + with open(os.path.join(tmp, "tools", "silent.py"), "w") as fh: + fh.write("print(42)\n") + f = check_reporting_tools_self_test(tmp) + check("self-test detects a tool lacking --self-test", + len(f) == 1 and "silent.py" in f[0].path, f"{len(f)} finding(s)") + + # review trail: approved tier-L survey with no history trips. + with open(os.path.join(tmp, "research", "C.md"), "w") as fh: + fh.write("tier: L (structural L, chaos 3)\nstatus: approved\n") + f = check_review_trail(tmp) + check("review-trail detects a missing challenge/response", + len(f) == 2, f"{len(f)} finding(s), expected 2") + finally: + shutil.rmtree(tmp, ignore_errors=True) + + print("loop-lint self-test (positive control)") + ok = True + for name, passed, detail in results: + print(f" [{'ok ' if passed else 'FAIL'}] {name}" + + (f" — {detail}" if detail else "")) + ok &= passed + return 0 if ok else 1 + + +def main(): + if "--self-test" in sys.argv: + return self_test() + + findings = [] + for c in CHECKS: + findings.extend(c()) + + print("loop-lint — executable InnerLoop rules") + if not findings: + print(" no findings") + return 0 + by_rule = {} + for f in findings: + by_rule.setdefault(f.rule, []).append(f) + for rule, fs in sorted(by_rule.items()): + print(f"\n{rule} ({len(fs)}):") + for f in fs: + print(str(f)) + print(f"\n{len(findings)} finding(s)") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/rule-coverage.py b/tools/rule-coverage.py index 306fa99..bf8c99c 100755 --- a/tools/rule-coverage.py +++ b/tools/rule-coverage.py @@ -5,30 +5,107 @@ Compares the rule IDs declared in specs/GroundRules.md against the `covers:` lists in scenarios/ground/*.yaml. Exits non-zero when a scenario claims a rule the spec does not define, so coverage can never be inflated by a typo'd or invented rule ID. + +Stated limit (InnerLoop implementation rule 4): this gate counts tags. It +proves no rule is unclaimed and no claimed rule is invented. It does NOT +prove a scenario exercises the rule it names. + +Positive control (InnerLoop v1.1 §Step 5): the run asserts it actually +found rules and scenarios. Before this was added, a broken spec regex +yielded rules=[] and missing=[] and the tool exited 0 reporting "0/0" — +the harness-does-nothing class, in the tool that reports our headline +coverage number. + +Usage: + python3 tools/rule-coverage.py + python3 tools/rule-coverage.py --self-test """ import glob import re import sys -spec = open("specs/GroundRules.md").read() -rules = sorted(set(re.findall(r"\*\*(GR-[A-Z]+\d+)", spec))) +RULE_RE = r"\*\*(GR-[A-Z]+\d+)" +COVERS_RE = r"covers: \[(.*?)\]" -covered = set() -for path in sorted(glob.glob("scenarios/ground/*.yaml")): - match = re.search(r"covers: \[(.*?)\]", open(path).read(), re.S) - if match: - covered |= {c.strip() for c in match.group(1).split(",") if c.strip()} -known = set(rules) -hit = sorted(known & covered) -missing = [r for r in rules if r not in covered] -invented = sorted(covered - known) +def parse_rules(spec_text): + return sorted(set(re.findall(RULE_RE, spec_text))) -pct = 100 * len(hit) // len(rules) if rules else 0 -print(f"AM-1 rule coverage: {len(hit)}/{len(rules)} ({pct}%)") -if missing: - print(" uncovered:", " ".join(missing)) -if invented: - print(" ERROR — claimed but not defined in the spec:", " ".join(invented)) - sys.exit(1) -sys.exit(0 if not missing else 2) + +def parse_covers(text): + match = re.search(COVERS_RE, text, re.S) + if not match: + return set() + return {c.strip() for c in match.group(1).split(",") if c.strip()} + + +def self_test(): + """Each assertion pins a failure this tool must detect.""" + results = [] + + def check(name, ok, detail=""): + results.append((name, ok, detail)) + + # The defect that motivated this control: a spec that parses to zero + # rules must not be reportable as coverage. + check("zero rules detected as a failure", parse_rules("no rules here") == [], + "empty spec yields no rules; main() now aborts on this") + # The matcher must actually match the real format. + check("rule matcher works on real spec format", + parse_rules("**GR-R06** something\n**GR-A12** other") + == ["GR-A12", "GR-R06"]) + # covers: parsing, including the empty case. + check("covers matcher works", parse_covers("covers: [GR-R06, GR-A12]") + == {"GR-R06", "GR-A12"}) + check("missing covers yields empty set", parse_covers("no covers key") == set()) + + print("rule-coverage self-test (positive control)") + ok = True + for name, passed, detail in results: + print(f" [{'ok ' if passed else 'FAIL'}] {name}" + + (f" — {detail}" if detail else "")) + ok &= passed + return 0 if ok else 1 + + +def main(): + if "--self-test" in sys.argv: + return self_test() + + rules = parse_rules(open("specs/GroundRules.md").read()) + paths = sorted(glob.glob("scenarios/ground/*.yaml")) + + # Positive control: refuse to report a percentage over nothing. + if not rules: + print("ERROR — no GR-rules parsed from specs/GroundRules.md; " + "refusing to report coverage", file=sys.stderr) + return 1 + if not paths: + print("ERROR — no scenarios found in scenarios/ground/; " + "refusing to report coverage", file=sys.stderr) + return 1 + + covered = set() + for path in paths: + covered |= parse_covers(open(path).read()) + + known = set(rules) + hit = sorted(known & covered) + missing = [r for r in rules if r not in covered] + invented = sorted(covered - known) + + pct = 100 * len(hit) // len(rules) + print(f"AM-1 rule coverage: {len(hit)}/{len(rules)} ({pct}%) " + f"over {len(paths)} scenarios") + print(" NOTE: counts tags; does not prove a scenario exercises what it names") + if missing: + print(" uncovered:", " ".join(missing)) + if invented: + print(" ERROR — claimed but not defined in the spec:", " ".join(invented), + file=sys.stderr) + return 1 + return 0 if not missing else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workplans/CB-WP-0003-loop-hardening.md b/workplans/CB-WP-0003-loop-hardening.md index a61f79d..a431ef4 100644 --- a/workplans/CB-WP-0003-loop-hardening.md +++ b/workplans/CB-WP-0003-loop-hardening.md @@ -1,7 +1,7 @@ --- id: CB-WP-0003 title: "Harden the inner loop: executable rules, session economics, dead policy" -status: proposed +status: active state_hub_workstream_id: "39d61dc0-870d-45c1-a595-bcf91f289dce" --- @@ -55,7 +55,7 @@ accordingly. ```task id: CB-WP-0003-T01 -status: todo +status: done priority: high state_hub_task_id: "3d5d45fb-f931-407a-be71-d2d727279d5e" ```