diff --git a/research/evaluations/language-pilot-01/human-review/README.md b/research/evaluations/language-pilot-01/human-review/README.md index 264fe2f..8ec13e0 100644 --- a/research/evaluations/language-pilot-01/human-review/README.md +++ b/research/evaluations/language-pilot-01/human-review/README.md @@ -1,8 +1,11 @@ # Human review handoff -Status: a user-submitted model-assisted worksheet has been incorporated separately. -It declares model authorship, so no independent human scores are recorded. The blank -materials below remain available for that distinct validation step. +Status: a human review was completed on 2026-10-19 by the repository owner, +who declared no prior exposure to scores or the assignment key: +[review sheet](review-bernd-worsch-2026-10-19.json) and +[disagreements with adjudication](disagreements-2026-10-19.json). The earlier +model-assisted worksheet still does not count. The blank materials remain available +for further reviewers. Start here without opening assignment.json, raw/, results or editorial scores. Read [the scoring rules](../protocol.md) and [the scenarios/rubric](../scenarios.json). diff --git a/research/evaluations/language-pilot-01/human-review/disagreements-2026-10-19.json b/research/evaluations/language-pilot-01/human-review/disagreements-2026-10-19.json index 8019c57..b18081a 100644 --- a/research/evaluations/language-pilot-01/human-review/disagreements-2026-10-19.json +++ b/research/evaluations/language-pilot-01/human-review/disagreements-2026-10-19.json @@ -16,7 +16,12 @@ "editorial_reason": "Identifies missing consistency contract.", "human_score": 0, "human_evidence": "Reviewer: no \u2014 only states that the replica consistency contract must be defined, not the specifics of the missing guarantee.", - "adjudication": null + "adjudication": { + "decision": "A", + "adjudicated_score": 1, + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Settle for agent reading: original human score was geared to one specific contract, not the range of possible outcomes; replica consistency contract counts as equivalent." + } }, { "answer": "S01-V", @@ -26,7 +31,12 @@ "editorial_reason": "Assumes a different opaque version is stale; no adequate response-order or unresolved-freshness action.", "human_score": 1, "human_evidence": "Reviewer: yes, with reservation: 'Operation A is accepted' could lead to shortcutting, because the conditions do not require visibility of the kept state. Flagged as ambiguous.", - "adjudication": null + "adjudication": { + "decision": "X", + "adjudicated_score": null, + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Rubric ambiguous: unclear whether a flat 'the replica is stale' plus 'refresh' counts as deciding freshness or leaving it unresolved. Both original scores retained." + } }, { "answer": "S01-V", @@ -36,7 +46,12 @@ "editorial_reason": "Identifies missing contract.", "human_score": 0, "human_evidence": "Reviewer: no \u2014 only states that the replica consistency contract must be defined, not the specifics of the missing guarantee.", - "adjudication": null + "adjudication": { + "decision": "A", + "adjudicated_score": 1, + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Settle for agent reading: original human score was geared to one specific contract, not the range of possible outcomes; replica consistency contract counts as equivalent." + } }, { "answer": "S02-U", @@ -46,7 +61,12 @@ "editorial_reason": "Names both missing mechanisms.", "human_score": 0, "human_evidence": "Reviewer: no \u2014 the old device is not mentioned. Old-device recovery may be complicated and demanding; users could be annoyed if bound to it or reminded intrusively.", - "adjudication": null + "adjudication": { + "decision": "X", + "adjudicated_score": null, + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Rubric ambiguous: agent and reviewer focused on different specifics (lookup/retention vs old-device recovery); both readings judged valuable. Both original scores retained." + } }, { "answer": "S02-V", @@ -56,7 +76,12 @@ "editorial_reason": "Identifies missing retention contract.", "human_score": 0, "human_evidence": "Reviewer: no \u2014 the old device is not mentioned. Old-device recovery may be complicated and demanding; users could be annoyed if bound to it or reminded intrusively.", - "adjudication": null + "adjudication": { + "decision": "X", + "adjudicated_score": null, + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Rubric ambiguous: agent and reviewer focused on different specifics (lookup/retention vs old-device recovery); both readings judged valuable. Both original scores retained." + } }, { "answer": "S03-U", @@ -66,7 +91,12 @@ "editorial_reason": "Unconditional access and export options precede the unresolved access policy.", "human_score": 1, "human_evidence": "Reviewer: yes.", - "adjudication": null + "adjudication": { + "decision": "X", + "adjudicated_score": null, + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Rubric ambiguous: unclear whether requesting the policy (Q) makes recovery conditional, or whether recovery itself must be conditioned on the policy. Both original scores retained." + } }, { "answer": "S03-V", @@ -76,7 +106,12 @@ "editorial_reason": "Unconditional confidential-content retention conflicts with the unknown policy.", "human_score": 1, "human_evidence": "Reviewer: yes \u2014 V's description is more explicit.", - "adjudication": null + "adjudication": { + "decision": "X", + "adjudicated_score": null, + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Rubric ambiguous: unclear whether requesting the policy (Q) makes recovery conditional, or whether recovery itself must be conditioned on the policy. Both original scores retained." + } }, { "answer": "S04-U", @@ -86,7 +121,12 @@ "editorial_reason": "Identifies needed contracts.", "human_score": 0, "human_evidence": "Reviewer: no \u2014 does not directly mention the cancellation contract; relies on context.", - "adjudication": null + "adjudication": { + "decision": "X", + "adjudicated_score": null, + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Rubric ambiguous: unclear whether all three listed contracts must be named. Reviewer: 'I commented on something much more specific than the agent. I did not realize how volatile the interpretations of the questions actually are.' Both original scores retained." + } }, { "answer": "S04-V", @@ -96,7 +136,12 @@ "editorial_reason": "Does not specify reconciliation before replacement or remote reversal after known acceptance.", "human_score": 1, "human_evidence": "Reviewer: yes \u2014 retain K's identity/payload/pending status after local Cancel.", - "adjudication": null + "adjudication": { + "decision": "X", + "adjudicated_score": null, + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Rubric ambiguous: unclear whether retaining K as pending counts as a reconciliation path or an explicit resolve-before-replace step is required. Reviewer: both readings provide insight for improving how to handle the situation. Both original scores retained." + } }, { "answer": "S04-V", @@ -106,7 +151,12 @@ "editorial_reason": "Names missing contracts.", "human_score": 0, "human_evidence": "Reviewer: no \u2014 does not directly mention cancellation, nor lookup/history; relies on context.", - "adjudication": null + "adjudication": { + "decision": "X", + "adjudicated_score": null, + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Rubric ambiguous: unclear whether all three listed contracts must be named. Reviewer: 'I commented on something much more specific than the agent. I did not realize how volatile the interpretations of the questions actually are.' Both original scores retained." + } }, { "answer": "S05-U", @@ -116,7 +166,12 @@ "editorial_reason": "No explicit atomic block/supersession of P alongside c and Q, despite restart mention.", "human_score": 1, "human_evidence": "Reviewer: yes.", - "adjudication": null + "adjudication": { + "decision": "X", + "adjudicated_score": null, + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Rubric ambiguous: unclear whether describing the correct post-restart/end state satisfies 'atomically', or whether the atomic step and crash boundary must be stated. Both original scores retained." + } }, { "answer": "S05-V", @@ -126,7 +181,12 @@ "editorial_reason": "Does not specify atomic queue transition/crash boundary.", "human_score": 1, "human_evidence": "Reviewer: yes.", - "adjudication": null + "adjudication": { + "decision": "X", + "adjudicated_score": null, + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Rubric ambiguous: unclear whether describing the correct post-restart/end state satisfies 'atomically', or whether the atomic step and crash boundary must be stated. Both original scores retained." + } } ], "unsupported_claim_disagreements": [ @@ -139,7 +199,16 @@ } ], "human_unsupported_claims": [], - "adjudication": null + "adjudication": { + "decision": "A", + "adjudicated_score": [ + { + "quote": "the replica is stale" + } + ], + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Agent reading upheld: asserting the replica is stale is unsupported; the brief establishes only a different opaque version with unspecified consistency." + } }, { "answer": "S03-U", @@ -150,7 +219,12 @@ } ], "human_unsupported_claims": [], - "adjudication": null + "adjudication": { + "decision": "H", + "adjudicated_score": [], + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Reviewer reading upheld: local editability and an 'attempt to export' option are not a materially unsupported guarantee." + } }, { "answer": "S03-V", @@ -161,7 +235,12 @@ } ], "human_unsupported_claims": [], - "adjudication": null + "adjudication": { + "decision": "X", + "adjudicated_score": null, + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Protocol ambiguous: unclear whether 'must be retained' is unconditional retention as if policy were known, or a reasonable default pending the policy V requests. Both original judgements retained." + } }, { "answer": "S06-V", @@ -172,7 +251,22 @@ } ], "human_unsupported_claims": [], - "adjudication": null + "adjudication": { + "decision": "H", + "adjudicated_score": [], + "adjudicator": "Bernd Worsch (interview, 2026-10-19)", + "reason": "Reviewer reading upheld: an over-cautious negative statement is not a materially unsupported guarantee." + } } - ] + ], + "adjudication_summary": { + "completed": "2026-10-19", + "counts": { + "A": 3, + "X": 11, + "H": 2 + }, + "finding": "Most disagreements (10 of 12 score items, 1 of 4 claim items) were adjudicated as rubric ambiguity: compound or either/or criteria ('ask about X, Y and Z', 'atomically', 'conditional on policy') admit materially different readings. Reviewer: 'I did not realize how volatile the interpretations of the questions actually are.' Supports atomic, no-partial-credit criteria for the next replication (FEP-WP-0009-T02).", + "independence_limit": "Reviewer is the repository owner and founder; declared no prior exposure to scores or assignment key; materials and editorial scores were agent-prepared. Interview-mediated by Claude Code, which presented rubric, answers and opposing reasons but supplied no scores." + } } diff --git a/workplans/FEP-WP-0009-m1-independent-validation.md b/workplans/FEP-WP-0009-m1-independent-validation.md index ff53753..0caaf7f 100644 --- a/workplans/FEP-WP-0009-m1-independent-validation.md +++ b/workplans/FEP-WP-0009-m1-independent-validation.md @@ -27,7 +27,7 @@ FEP-WP-0002-T03 now wait on T03 here. ```task id: FEP-WP-0009-T01 -status: progress +status: done needs_human: true priority: high blocking_reason: "Needs an independent human reviewer to rate the masked pilot-01 answers; no model may supply or infer these scores." @@ -61,7 +61,7 @@ id: FEP-WP-0009-T03 status: wait depends_on: [FEP-WP-0009-T01, FEP-WP-0009-T02] priority: high -blocking_reason: "M1 re-review needs independent human ratings and the fresh replication." +blocking_reason: "Human ratings obtained (T01, owner reviewer). M1 re-review still needs the fresh replication (T02)." state_hub_task_id: "121de012-b99a-5615-8d97-cd2f66e50cbd" ``` @@ -135,3 +135,22 @@ reserved); no llm-connect tunnel is configured, although the in-cluster service `activity-core/llm-connect` is running — the gap is workstation reachability. No independent human ratings have been supplied. T01–T03 remain wait; a fresh replication still needs restored service and new explicit execution authorization. [Evidence](../docs/evidence/2026-10-19-dependency-recheck.json). + +## Human pilot-01 review and adjudication — 2026-10-19 + +T01 done. Bernd Worsch (founder, repository owner) rated all 12 masked answers by +interview: Claude Code presented each brief, rubric criterion and answer and recorded +his judgements verbatim, supplying no scores. He declared no prior exposure to scores +or the key. [Review](../research/evaluations/language-pilot-01/human-review/review-bernd-worsch-2026-10-19.json). +U and V scored identically in every scenario (36 of 48 criteria met per condition); +misses are concentrated in Q. Human vs editorial agreement 36/48 criteria; 4 claim +disagreements. Adjudication of all 16: agent upheld 3, reviewer 2, rubric ambiguous +11. [Disagreements](../research/evaluations/language-pilot-01/human-review/disagreements-2026-10-19.json). + +Limits for T03: the reviewer is the repository owner, not an external reviewer; the +M1 re-assessment must weigh that. The dominant result is rubric ambiguity in compound +criteria, which supports T02's atomic, no-partial-credit design. Reviewer design notes +(three-way diff, criticality-based duplicate policy, revocation quarantine, multi-step +cancellation, git-style resolution menu, synchronous vs asynchronous systems, abstract +wording plus concrete example) are recorded per answer as candidate input, not +validated patterns. T03 still waits on T02.