Record human vs editorial pilot-01 disagreements (adjudication pending)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 182330@bnt-lap001 Assistant-Session: 242866cf-789d-4e24-ac5e-0a1d7b47a701
This commit is contained in:
parent
c53be82a0e
commit
7cf3ca1738
1 changed files with 178 additions and 0 deletions
|
|
@ -0,0 +1,178 @@
|
|||
{
|
||||
"compared_at": "2026-10-19",
|
||||
"human_sheet": "review-bernd-worsch-2026-10-19.json",
|
||||
"editorial_sheet": "../editorial-scores-masked.json",
|
||||
"criteria_compared": 48,
|
||||
"criteria_agreements": 36,
|
||||
"criteria_disagreements": 12,
|
||||
"pattern": "Human stricter on Q (7 of 12 answers: human 0, editorial 1); human more lenient on R/C1 (5: human 1, editorial 0) and on unsupported claims (editorial 4, human 0).",
|
||||
"rule": "Both original judgements are retained unchanged; adjudication against the frozen rubric is recorded separately and never averages or overwrites.",
|
||||
"score_disagreements": [
|
||||
{
|
||||
"answer": "S01-U",
|
||||
"criterion": "Q",
|
||||
"editorial_score": 1,
|
||||
"editorial_locator": "replica consistency contract needs to be explicitly defined",
|
||||
"editorial_reason": "Identifies missing consistency contract.",
|
||||
"human_score": 0,
|
||||
"human_evidence": "Reviewer: no \u2014 only states that the replica consistency contract must be defined, not the specifics of the missing guarantee.",
|
||||
"adjudication": null
|
||||
},
|
||||
{
|
||||
"answer": "S01-V",
|
||||
"criterion": "R",
|
||||
"editorial_score": 0,
|
||||
"editorial_locator": "the replica is stale",
|
||||
"editorial_reason": "Assumes a different opaque version is stale; no adequate response-order or unresolved-freshness action.",
|
||||
"human_score": 1,
|
||||
"human_evidence": "Reviewer: yes, with reservation: 'Operation A is accepted' could lead to shortcutting, because the conditions do not require visibility of the kept state. Flagged as ambiguous.",
|
||||
"adjudication": null
|
||||
},
|
||||
{
|
||||
"answer": "S01-V",
|
||||
"criterion": "Q",
|
||||
"editorial_score": 1,
|
||||
"editorial_locator": "replica consistency contract needs to be explicitly defined",
|
||||
"editorial_reason": "Identifies missing contract.",
|
||||
"human_score": 0,
|
||||
"human_evidence": "Reviewer: no \u2014 only states that the replica consistency contract must be defined, not the specifics of the missing guarantee.",
|
||||
"adjudication": null
|
||||
},
|
||||
{
|
||||
"answer": "S02-U",
|
||||
"criterion": "Q",
|
||||
"editorial_score": 1,
|
||||
"editorial_locator": "lookup support and retention for operation IDs are critical",
|
||||
"editorial_reason": "Names both missing mechanisms.",
|
||||
"human_score": 0,
|
||||
"human_evidence": "Reviewer: no \u2014 the old device is not mentioned. Old-device recovery may be complicated and demanding; users could be annoyed if bound to it or reminded intrusively.",
|
||||
"adjudication": null
|
||||
},
|
||||
{
|
||||
"answer": "S02-V",
|
||||
"criterion": "Q",
|
||||
"editorial_score": 1,
|
||||
"editorial_locator": "retention for operation IDs are critical but unspecified",
|
||||
"editorial_reason": "Identifies missing retention contract.",
|
||||
"human_score": 0,
|
||||
"human_evidence": "Reviewer: no \u2014 the old device is not mentioned. Old-device recovery may be complicated and demanding; users could be annoyed if bound to it or reminded intrusively.",
|
||||
"adjudication": null
|
||||
},
|
||||
{
|
||||
"answer": "S03-U",
|
||||
"criterion": "R",
|
||||
"editorial_score": 0,
|
||||
"editorial_locator": "The notes remain editable locally",
|
||||
"editorial_reason": "Unconditional access and export options precede the unresolved access policy.",
|
||||
"human_score": 1,
|
||||
"human_evidence": "Reviewer: yes.",
|
||||
"adjudication": null
|
||||
},
|
||||
{
|
||||
"answer": "S03-V",
|
||||
"criterion": "R",
|
||||
"editorial_score": 0,
|
||||
"editorial_locator": "must be retained",
|
||||
"editorial_reason": "Unconditional confidential-content retention conflicts with the unknown policy.",
|
||||
"human_score": 1,
|
||||
"human_evidence": "Reviewer: yes \u2014 V's description is more explicit.",
|
||||
"adjudication": null
|
||||
},
|
||||
{
|
||||
"answer": "S04-U",
|
||||
"criterion": "Q",
|
||||
"editorial_score": 1,
|
||||
"editorial_locator": "lacks a documented operation-status lookup or deduplication retention guarantee",
|
||||
"editorial_reason": "Identifies needed contracts.",
|
||||
"human_score": 0,
|
||||
"human_evidence": "Reviewer: no \u2014 does not directly mention the cancellation contract; relies on context.",
|
||||
"adjudication": null
|
||||
},
|
||||
{
|
||||
"answer": "S04-V",
|
||||
"criterion": "R",
|
||||
"editorial_score": 0,
|
||||
"editorial_locator": "should be treated as a new, distinct operation with a fresh identity",
|
||||
"editorial_reason": "Does not specify reconciliation before replacement or remote reversal after known acceptance.",
|
||||
"human_score": 1,
|
||||
"human_evidence": "Reviewer: yes \u2014 retain K's identity/payload/pending status after local Cancel.",
|
||||
"adjudication": null
|
||||
},
|
||||
{
|
||||
"answer": "S04-V",
|
||||
"criterion": "Q",
|
||||
"editorial_score": 1,
|
||||
"editorial_locator": "server's behavior regarding duplicate submissions and its retention window",
|
||||
"editorial_reason": "Names missing contracts.",
|
||||
"human_score": 0,
|
||||
"human_evidence": "Reviewer: no \u2014 does not directly mention cancellation, nor lookup/history; relies on context.",
|
||||
"adjudication": null
|
||||
},
|
||||
{
|
||||
"answer": "S05-U",
|
||||
"criterion": "C1",
|
||||
"editorial_score": 0,
|
||||
"editorial_locator": "original pending operation P and its identity must also be retained",
|
||||
"editorial_reason": "No explicit atomic block/supersession of P alongside c and Q, despite restart mention.",
|
||||
"human_score": 1,
|
||||
"human_evidence": "Reviewer: yes.",
|
||||
"adjudication": null
|
||||
},
|
||||
{
|
||||
"answer": "S05-V",
|
||||
"criterion": "C1",
|
||||
"editorial_score": 0,
|
||||
"editorial_locator": "potentially flag that P's original intent was superseded",
|
||||
"editorial_reason": "Does not specify atomic queue transition/crash boundary.",
|
||||
"human_score": 1,
|
||||
"human_evidence": "Reviewer: yes.",
|
||||
"adjudication": null
|
||||
}
|
||||
],
|
||||
"unsupported_claim_disagreements": [
|
||||
{
|
||||
"answer": "S01-V",
|
||||
"editorial_unsupported_claims": [
|
||||
{
|
||||
"quote": "the replica is stale",
|
||||
"reason": "The scenario establishes a different opaque version and unspecified consistency, not that this observation is older."
|
||||
}
|
||||
],
|
||||
"human_unsupported_claims": [],
|
||||
"adjudication": null
|
||||
},
|
||||
{
|
||||
"answer": "S03-U",
|
||||
"editorial_unsupported_claims": [
|
||||
{
|
||||
"quote": "The notes remain editable locally. ... options to either discard ... or attempt to export/save the content locally.",
|
||||
"reason": "Grants local access/export before the missing revocation policy has been resolved."
|
||||
}
|
||||
],
|
||||
"human_unsupported_claims": [],
|
||||
"adjudication": null
|
||||
},
|
||||
{
|
||||
"answer": "S03-V",
|
||||
"editorial_unsupported_claims": [
|
||||
{
|
||||
"quote": "The full local commit of confidential notes, including its revision and operation identity, must be retained.",
|
||||
"reason": "Treats permanent/mandatory retention as decided before knowing post-revocation policy."
|
||||
}
|
||||
],
|
||||
"human_unsupported_claims": [],
|
||||
"adjudication": null
|
||||
},
|
||||
{
|
||||
"answer": "S06-V",
|
||||
"editorial_unsupported_claims": [
|
||||
{
|
||||
"quote": "it cannot recover from power or device loss",
|
||||
"reason": "Converts lack of tested power-loss/device-loss evidence into categorical inability. Unsupported certainty; not a claim of guaranteed survival."
|
||||
}
|
||||
],
|
||||
"human_unsupported_claims": [],
|
||||
"adjudication": null
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue