diff --git a/research/evaluations/language-pilot-01/human-review/review-bernd-worsch-2026-10-19.json b/research/evaluations/language-pilot-01/human-review/review-bernd-worsch-2026-10-19.json new file mode 100644 index 0000000..d3b9368 --- /dev/null +++ b/research/evaluations/language-pilot-01/human-review/review-bernd-worsch-2026-10-19.json @@ -0,0 +1,226 @@ +{ + "reviewer": "Bernd Worsch (founder, Binky Hedgehog GmbH; repository owner)", + "reviewed_at": "2026-10-19", + "read_assignment_or_results_before_scoring": false, + "answers": [ + { + "answer": "S01-U", + "scores": { + "C1": 1, + "C2": 1, + "R": 1, + "Q": 0 + }, + "evidence": { + "C1": "Reviewer: 'confirmed accepted' \u2014 A is treated as saved.", + "C2": "Reviewer: yes \u2014 'draft B is preserved', 'keeping draft B visible'.", + "R": "Reviewer: yes; U is more clearly put ('superseded and ignored'; refresh based on accepted revision while keeping B).", + "Q": "Reviewer: no \u2014 only states that the replica consistency contract must be defined, not the specifics of the missing guarantee." + }, + "unsupported_claims": [], + "suspected_condition_leakage": "none", + "notes": "Reviewer guidance (S01, C2): the user's work stays primary, with an unobtrusive hint that it started from a stale state and a convenient resolution option: a three-way diff of the user's current input, the state the user started from, and the current backend state, with simple selection options and the ability to edit or re-edit before submitting. Reviewer guidance (S01, Q): stating the contract generically may help generality; naming the specific missing guarantee as an example would combine specificity and generality." + }, + { + "answer": "S01-V", + "scores": { + "C1": 1, + "C2": 1, + "R": 1, + "Q": 0 + }, + "evidence": { + "C1": "Reviewer: 'accepted' \u2014 A is treated as saved.", + "C2": "Reviewer: yes \u2014 'draft B is present', 'preserving draft B'.", + "R": "Reviewer: yes, with reservation: 'Operation A is accepted' could lead to shortcutting, because the conditions do not require visibility of the kept state. Flagged as ambiguous.", + "Q": "Reviewer: no \u2014 only states that the replica consistency contract must be defined, not the specifics of the missing guarantee." + }, + "unsupported_claims": [], + "suspected_condition_leakage": "none", + "notes": "Reviewer guidance (S01, C2): the user's work stays primary, with an unobtrusive hint that it started from a stale state and a convenient resolution option: a three-way diff of the user's current input, the state the user started from, and the current backend state, with simple selection options and the ability to edit or re-edit before submitting. Reviewer guidance (S01, Q): stating the contract generically may help generality; naming the specific missing guarantee as an example would combine specificity and generality.", + "ambiguity": "R: reviewer scored 1 but noted 'Operation A is accepted' could invite the acceptance-equals-freshness shortcut; U judged clearer." + }, + { + "answer": "S02-U", + "scores": { + "C1": 1, + "C2": 1, + "R": 1, + "Q": 0 + }, + "evidence": { + "C1": "Reviewer: yes \u2014 'delivery status is unknown due to a lost response'.", + "C2": "Reviewer: yes \u2014 fresh-ID retry 'is problematic \u2026 risks sending a duplicate'.", + "R": "Reviewer: yes \u2014 user prompted to manually reconcile, 'acknowledging potential duplication'.", + "Q": "Reviewer: no \u2014 the old device is not mentioned. Old-device recovery may be complicated and demanding; users could be annoyed if bound to it or reminded intrusively." + }, + "unsupported_claims": [], + "suspected_condition_leakage": "none", + "notes": "Reviewer guidance (S02, Q): choose policy by communication criticality \u2014 an accept-easily policy with a hint (optionally a remembered 'resend automatically' preference checkbox), or an avoid-duplicates policy for critical documents such as contracts or security-relevant information." + }, + { + "answer": "S02-V", + "scores": { + "C1": 1, + "C2": 1, + "R": 1, + "Q": 0 + }, + "evidence": { + "C1": "Reviewer: yes \u2014 'unknown server outcome for the initial send attempt'.", + "C2": "Reviewer: yes \u2014 fresh-ID automatic retry 'is flawed \u2026 risks duplicating the message'.", + "R": "Reviewer: yes \u2014 user 'must be prompted to manually confirm re-sending \u2026 acknowledging the potential for duplication'.", + "Q": "Reviewer: no \u2014 the old device is not mentioned. Old-device recovery may be complicated and demanding; users could be annoyed if bound to it or reminded intrusively." + }, + "unsupported_claims": [], + "suspected_condition_leakage": "none", + "notes": "Reviewer guidance (S02, Q): choose policy by communication criticality \u2014 an accept-easily policy with a hint (optionally a remembered 'resend automatically' preference checkbox), or an avoid-duplicates policy for critical documents such as contracts or security-relevant information." + }, + { + "answer": "S03-U", + "scores": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "evidence": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "unsupported_claims": null, + "suspected_condition_leakage": null, + "notes": null + }, + { + "answer": "S03-V", + "scores": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "evidence": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "unsupported_claims": null, + "suspected_condition_leakage": null, + "notes": null + }, + { + "answer": "S04-U", + "scores": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "evidence": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "unsupported_claims": null, + "suspected_condition_leakage": null, + "notes": null + }, + { + "answer": "S04-V", + "scores": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "evidence": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "unsupported_claims": null, + "suspected_condition_leakage": null, + "notes": null + }, + { + "answer": "S05-U", + "scores": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "evidence": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "unsupported_claims": null, + "suspected_condition_leakage": null, + "notes": null + }, + { + "answer": "S05-V", + "scores": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "evidence": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "unsupported_claims": null, + "suspected_condition_leakage": null, + "notes": null + }, + { + "answer": "S06-U", + "scores": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "evidence": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "unsupported_claims": null, + "suspected_condition_leakage": null, + "notes": null + }, + { + "answer": "S06-V", + "scores": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "evidence": { + "C1": null, + "C2": null, + "R": null, + "Q": null + }, + "unsupported_claims": null, + "suspected_condition_leakage": null, + "notes": null + } + ], + "review_mode": "Interview: Claude Code presented each brief, answer and criterion in turn and recorded the reviewer's own judgements verbatim; Claude supplied no scores or hints.", + "reviewer_declaration": "I have not seen scores for these answers and will provide my best judgement as founder of Binky Hedgehog GmbH." +} diff --git a/workplans/FEP-WP-0009-m1-independent-validation.md b/workplans/FEP-WP-0009-m1-independent-validation.md index 8cee3d4..ff53753 100644 --- a/workplans/FEP-WP-0009-m1-independent-validation.md +++ b/workplans/FEP-WP-0009-m1-independent-validation.md @@ -27,7 +27,7 @@ FEP-WP-0002-T03 now wait on T03 here. ```task id: FEP-WP-0009-T01 -status: wait +status: progress needs_human: true priority: high blocking_reason: "Needs an independent human reviewer to rate the masked pilot-01 answers; no model may supply or infer these scores."