From 7cf3ca1738b0bb894baeefc08c691594bd21fa0e Mon Sep 17 00:00:00 2001 From: tegwick Date: Mon, 19 Oct 2026 23:34:01 +0200 Subject: [PATCH] Record human vs editorial pilot-01 disagreements (adjudication pending) Co-Authored-By: Claude Opus 5.5 Assistant: claude-code Assistant-Model: opus Assistant-Process: 182330@bnt-lap001 Assistant-Session: 242866cf-789d-4e24-ac5e-0a1d7b47a701 --- .../disagreements-2026-10-19.json | 178 ++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 research/evaluations/language-pilot-01/human-review/disagreements-2026-10-19.json diff --git a/research/evaluations/language-pilot-01/human-review/disagreements-2026-10-19.json b/research/evaluations/language-pilot-01/human-review/disagreements-2026-10-19.json new file mode 100644 index 0000000..8019c57 --- /dev/null +++ b/research/evaluations/language-pilot-01/human-review/disagreements-2026-10-19.json @@ -0,0 +1,178 @@ +{ + "compared_at": "2026-10-19", + "human_sheet": "review-bernd-worsch-2026-10-19.json", + "editorial_sheet": "../editorial-scores-masked.json", + "criteria_compared": 48, + "criteria_agreements": 36, + "criteria_disagreements": 12, + "pattern": "Human stricter on Q (7 of 12 answers: human 0, editorial 1); human more lenient on R/C1 (5: human 1, editorial 0) and on unsupported claims (editorial 4, human 0).", + "rule": "Both original judgements are retained unchanged; adjudication against the frozen rubric is recorded separately and never averages or overwrites.", + "score_disagreements": [ + { + "answer": "S01-U", + "criterion": "Q", + "editorial_score": 1, + "editorial_locator": "replica consistency contract needs to be explicitly defined", + "editorial_reason": "Identifies missing consistency contract.", + "human_score": 0, + "human_evidence": "Reviewer: no \u2014 only states that the replica consistency contract must be defined, not the specifics of the missing guarantee.", + "adjudication": null + }, + { + "answer": "S01-V", + "criterion": "R", + "editorial_score": 0, + "editorial_locator": "the replica is stale", + "editorial_reason": "Assumes a different opaque version is stale; no adequate response-order or unresolved-freshness action.", + "human_score": 1, + "human_evidence": "Reviewer: yes, with reservation: 'Operation A is accepted' could lead to shortcutting, because the conditions do not require visibility of the kept state. Flagged as ambiguous.", + "adjudication": null + }, + { + "answer": "S01-V", + "criterion": "Q", + "editorial_score": 1, + "editorial_locator": "replica consistency contract needs to be explicitly defined", + "editorial_reason": "Identifies missing contract.", + "human_score": 0, + "human_evidence": "Reviewer: no \u2014 only states that the replica consistency contract must be defined, not the specifics of the missing guarantee.", + "adjudication": null + }, + { + "answer": "S02-U", + "criterion": "Q", + "editorial_score": 1, + "editorial_locator": "lookup support and retention for operation IDs are critical", + "editorial_reason": "Names both missing mechanisms.", + "human_score": 0, + "human_evidence": "Reviewer: no \u2014 the old device is not mentioned. Old-device recovery may be complicated and demanding; users could be annoyed if bound to it or reminded intrusively.", + "adjudication": null + }, + { + "answer": "S02-V", + "criterion": "Q", + "editorial_score": 1, + "editorial_locator": "retention for operation IDs are critical but unspecified", + "editorial_reason": "Identifies missing retention contract.", + "human_score": 0, + "human_evidence": "Reviewer: no \u2014 the old device is not mentioned. Old-device recovery may be complicated and demanding; users could be annoyed if bound to it or reminded intrusively.", + "adjudication": null + }, + { + "answer": "S03-U", + "criterion": "R", + "editorial_score": 0, + "editorial_locator": "The notes remain editable locally", + "editorial_reason": "Unconditional access and export options precede the unresolved access policy.", + "human_score": 1, + "human_evidence": "Reviewer: yes.", + "adjudication": null + }, + { + "answer": "S03-V", + "criterion": "R", + "editorial_score": 0, + "editorial_locator": "must be retained", + "editorial_reason": "Unconditional confidential-content retention conflicts with the unknown policy.", + "human_score": 1, + "human_evidence": "Reviewer: yes \u2014 V's description is more explicit.", + "adjudication": null + }, + { + "answer": "S04-U", + "criterion": "Q", + "editorial_score": 1, + "editorial_locator": "lacks a documented operation-status lookup or deduplication retention guarantee", + "editorial_reason": "Identifies needed contracts.", + "human_score": 0, + "human_evidence": "Reviewer: no \u2014 does not directly mention the cancellation contract; relies on context.", + "adjudication": null + }, + { + "answer": "S04-V", + "criterion": "R", + "editorial_score": 0, + "editorial_locator": "should be treated as a new, distinct operation with a fresh identity", + "editorial_reason": "Does not specify reconciliation before replacement or remote reversal after known acceptance.", + "human_score": 1, + "human_evidence": "Reviewer: yes \u2014 retain K's identity/payload/pending status after local Cancel.", + "adjudication": null + }, + { + "answer": "S04-V", + "criterion": "Q", + "editorial_score": 1, + "editorial_locator": "server's behavior regarding duplicate submissions and its retention window", + "editorial_reason": "Names missing contracts.", + "human_score": 0, + "human_evidence": "Reviewer: no \u2014 does not directly mention cancellation, nor lookup/history; relies on context.", + "adjudication": null + }, + { + "answer": "S05-U", + "criterion": "C1", + "editorial_score": 0, + "editorial_locator": "original pending operation P and its identity must also be retained", + "editorial_reason": "No explicit atomic block/supersession of P alongside c and Q, despite restart mention.", + "human_score": 1, + "human_evidence": "Reviewer: yes.", + "adjudication": null + }, + { + "answer": "S05-V", + "criterion": "C1", + "editorial_score": 0, + "editorial_locator": "potentially flag that P's original intent was superseded", + "editorial_reason": "Does not specify atomic queue transition/crash boundary.", + "human_score": 1, + "human_evidence": "Reviewer: yes.", + "adjudication": null + } + ], + "unsupported_claim_disagreements": [ + { + "answer": "S01-V", + "editorial_unsupported_claims": [ + { + "quote": "the replica is stale", + "reason": "The scenario establishes a different opaque version and unspecified consistency, not that this observation is older." + } + ], + "human_unsupported_claims": [], + "adjudication": null + }, + { + "answer": "S03-U", + "editorial_unsupported_claims": [ + { + "quote": "The notes remain editable locally. ... options to either discard ... or attempt to export/save the content locally.", + "reason": "Grants local access/export before the missing revocation policy has been resolved." + } + ], + "human_unsupported_claims": [], + "adjudication": null + }, + { + "answer": "S03-V", + "editorial_unsupported_claims": [ + { + "quote": "The full local commit of confidential notes, including its revision and operation identity, must be retained.", + "reason": "Treats permanent/mandatory retention as decided before knowing post-revocation policy." + } + ], + "human_unsupported_claims": [], + "adjudication": null + }, + { + "answer": "S06-V", + "editorial_unsupported_claims": [ + { + "quote": "it cannot recover from power or device loss", + "reason": "Converts lack of tested power-loss/device-loss evidence into categorical inability. Unsupported certainty; not a claim of guaranteed survival." + } + ], + "human_unsupported_claims": [], + "adjudication": null + } + ] +}