Record twelve pilot sessions and masked editorial judgments
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a0cffe-206f-7323-89d3-b20ca4cf4bf9
This commit is contained in:
parent
6acbe6df33
commit
3690af12a5
63 changed files with 2120 additions and 0 deletions
|
|
@ -0,0 +1,383 @@
|
|||
{
|
||||
"reviewer": "Codex editorial assessment",
|
||||
"date": "2026-09-26",
|
||||
"stage": "before assignment key inspection",
|
||||
"independent_human": false,
|
||||
"masking_limit": "Assessor authored packs/rubrics; condition key and raw receipts not opened before these judgments. No pattern names in masked answers were noticed; inference from wording remains possible.",
|
||||
"criterion_order": [
|
||||
"C1",
|
||||
"C2",
|
||||
"R",
|
||||
"Q"
|
||||
],
|
||||
"answers": [
|
||||
{
|
||||
"answer": "S01-U",
|
||||
"answer_sha256": "5eda5ddfd9725e38aa5827ffd2c929d64074f8c0fd31ee9b5f667362a5c28691",
|
||||
"criteria": {
|
||||
"C1": {
|
||||
"score": 1,
|
||||
"locator": "Operation A is confirmed accepted",
|
||||
"reason": "Recognizes stronger acceptance evidence."
|
||||
},
|
||||
"C2": {
|
||||
"score": 1,
|
||||
"locator": "draft B is preserved",
|
||||
"reason": "Separates newer typing from A."
|
||||
},
|
||||
"R": {
|
||||
"score": 1,
|
||||
"locator": "replica read is superseded and ignored",
|
||||
"reason": "Rejects applying an earlier read; refresh remains contract-dependent."
|
||||
},
|
||||
"Q": {
|
||||
"score": 1,
|
||||
"locator": "replica consistency contract needs to be explicitly defined",
|
||||
"reason": "Identifies missing consistency contract."
|
||||
}
|
||||
},
|
||||
"unsupported_claims": [],
|
||||
"suspected_condition_leakage": false,
|
||||
"notes": ""
|
||||
},
|
||||
{
|
||||
"answer": "S01-V",
|
||||
"answer_sha256": "a63771e0792ab2df51720093a331e70e02a21120103f1adc7285e027721a8bf0",
|
||||
"criteria": {
|
||||
"C1": {
|
||||
"score": 1,
|
||||
"locator": "Operation A is accepted",
|
||||
"reason": "Does not regress A after promise rejection."
|
||||
},
|
||||
"C2": {
|
||||
"score": 1,
|
||||
"locator": "while preserving draft B",
|
||||
"reason": "Retains newer draft."
|
||||
},
|
||||
"R": {
|
||||
"score": 0,
|
||||
"locator": "the replica is stale",
|
||||
"reason": "Assumes a different opaque version is stale; no adequate response-order or unresolved-freshness action."
|
||||
},
|
||||
"Q": {
|
||||
"score": 1,
|
||||
"locator": "replica consistency contract needs to be explicitly defined",
|
||||
"reason": "Identifies missing contract."
|
||||
}
|
||||
},
|
||||
"unsupported_claims": [
|
||||
{
|
||||
"quote": "the replica is stale",
|
||||
"reason": "The scenario establishes a different opaque version and unspecified consistency, not that this observation is older."
|
||||
}
|
||||
],
|
||||
"suspected_condition_leakage": false,
|
||||
"notes": "The later request for a contract does not establish the asserted stale ordering."
|
||||
},
|
||||
{
|
||||
"answer": "S02-U",
|
||||
"answer_sha256": "5cda87a8d3d5f7952ab3d855213c10056d1d43c26e137c0de2225ced5113d6ba",
|
||||
"criteria": {
|
||||
"C1": {
|
||||
"score": 1,
|
||||
"locator": "delivery status is unknown",
|
||||
"reason": "Recognizes uncertain original outcome."
|
||||
},
|
||||
"C2": {
|
||||
"score": 1,
|
||||
"locator": "risks sending a duplicate message",
|
||||
"reason": "Rejects fresh-ID retry as duplicate-safe."
|
||||
},
|
||||
"R": {
|
||||
"score": 1,
|
||||
"locator": "asking if they want to resend it, acknowledging potential duplication",
|
||||
"reason": "Explicit informed reconciliation allowed by the rubric."
|
||||
},
|
||||
"Q": {
|
||||
"score": 1,
|
||||
"locator": "lookup support and retention for operation IDs are critical",
|
||||
"reason": "Names both missing mechanisms."
|
||||
}
|
||||
},
|
||||
"unsupported_claims": [],
|
||||
"suspected_condition_leakage": false,
|
||||
"notes": "Content lookup is suggested, not asserted to uniquely prove identity; no guarantee credited."
|
||||
},
|
||||
{
|
||||
"answer": "S02-V",
|
||||
"answer_sha256": "0cab89bf9c994d5a87913eb956bdd400611a5aa05a25c46e9473fda524b4403b",
|
||||
"criteria": {
|
||||
"C1": {
|
||||
"score": 1,
|
||||
"locator": "unknown server outcome",
|
||||
"reason": "Retains uncertainty."
|
||||
},
|
||||
"C2": {
|
||||
"score": 1,
|
||||
"locator": "automatically retry is flawed",
|
||||
"reason": "Rejects automatic fresh-ID recovery."
|
||||
},
|
||||
"R": {
|
||||
"score": 1,
|
||||
"locator": "manually confirm re-sending ... acknowledging the potential for duplication",
|
||||
"reason": "Proposes informed reconciliation; ellipsis locator paraphrases surrounding words."
|
||||
},
|
||||
"Q": {
|
||||
"score": 1,
|
||||
"locator": "retention for operation IDs are critical but unspecified",
|
||||
"reason": "Identifies missing retention contract."
|
||||
}
|
||||
},
|
||||
"unsupported_claims": [],
|
||||
"suspected_condition_leakage": false,
|
||||
"notes": "Factual imprecision: deduplication scope was supplied, although answer calls scope unspecified. No unsupported safety guarantee is asserted."
|
||||
},
|
||||
{
|
||||
"answer": "S03-U",
|
||||
"answer_sha256": "0c773739af4a505c87fbea5659da9ba4e0d9207643597992b9050d22f04372ac",
|
||||
"criteria": {
|
||||
"C1": {
|
||||
"score": 1,
|
||||
"locator": "locally committed ... explicitly rejected due to authorization failure",
|
||||
"reason": "Separates local save and remote denial."
|
||||
},
|
||||
"C2": {
|
||||
"score": 1,
|
||||
"locator": "Blindly retrying the upload with a new identity is rejected",
|
||||
"reason": "Does not use new identity to bypass authorization."
|
||||
},
|
||||
"R": {
|
||||
"score": 0,
|
||||
"locator": "The notes remain editable locally",
|
||||
"reason": "Unconditional access and export options precede the unresolved access policy."
|
||||
},
|
||||
"Q": {
|
||||
"score": 1,
|
||||
"locator": "can the user continue to view, edit, or export these notes, or should they be purged?",
|
||||
"reason": "Asks the material policy question."
|
||||
}
|
||||
},
|
||||
"unsupported_claims": [
|
||||
{
|
||||
"quote": "The notes remain editable locally. ... options to either discard ... or attempt to export/save the content locally.",
|
||||
"reason": "Grants local access/export before the missing revocation policy has been resolved."
|
||||
}
|
||||
],
|
||||
"suspected_condition_leakage": false,
|
||||
"notes": "Questioning the policy later does not make the earlier unqualified permissions conditional. The duplication explanation for an explicit rejection is also weak."
|
||||
},
|
||||
{
|
||||
"answer": "S03-V",
|
||||
"answer_sha256": "d1e51b0040acf54520444273b5445d45bee241d3941a0e82c7f7ac0b9b5f7804",
|
||||
"criteria": {
|
||||
"C1": {
|
||||
"score": 1,
|
||||
"locator": "accepted locally, rejected remotely",
|
||||
"reason": "Separates acknowledgement scopes."
|
||||
},
|
||||
"C2": {
|
||||
"score": 1,
|
||||
"locator": "attempting a re-upload with a new identity is inappropriate",
|
||||
"reason": "Rejects this retry route."
|
||||
},
|
||||
"R": {
|
||||
"score": 0,
|
||||
"locator": "must be retained",
|
||||
"reason": "Unconditional confidential-content retention conflicts with the unknown policy."
|
||||
},
|
||||
"Q": {
|
||||
"score": 1,
|
||||
"locator": "retention, viewing, and export ... This will dictate the specific recovery options",
|
||||
"reason": "Explicitly requests the policy."
|
||||
}
|
||||
},
|
||||
"unsupported_claims": [
|
||||
{
|
||||
"quote": "The full local commit of confidential notes, including its revision and operation identity, must be retained.",
|
||||
"reason": "Treats permanent/mandatory retention as decided before knowing post-revocation policy."
|
||||
}
|
||||
],
|
||||
"suspected_condition_leakage": false,
|
||||
"notes": "Borderline for human review: final sentence conditions recovery options, but not the earlier must-retain statement. Export itself is framed tentatively; do not count it as a second violation."
|
||||
},
|
||||
{
|
||||
"answer": "S04-U",
|
||||
"answer_sha256": "acdf9bc9bb7faadfeb32be0ebdc9926984135f262fcf12cec7ebd07c9742cd1b",
|
||||
"criteria": {
|
||||
"C1": {
|
||||
"score": 1,
|
||||
"locator": "may have accepted operation K despite the timeout and local cancellation",
|
||||
"reason": "Local Cancel is not remote reversal."
|
||||
},
|
||||
"C2": {
|
||||
"score": 1,
|
||||
"locator": "A new submission is unsafe",
|
||||
"reason": "Does not claim replacement is duplicate-safe."
|
||||
},
|
||||
"R": {
|
||||
"score": 1,
|
||||
"locator": "check with the service provider",
|
||||
"reason": "Provides authoritative escalation/reconciliation before replacement."
|
||||
},
|
||||
"Q": {
|
||||
"score": 1,
|
||||
"locator": "lacks a documented operation-status lookup or deduplication retention guarantee",
|
||||
"reason": "Identifies needed contracts."
|
||||
}
|
||||
},
|
||||
"unsupported_claims": [],
|
||||
"suspected_condition_leakage": false,
|
||||
"notes": "Overstates that both identity and intent were discarded: only pending-entry removal is given. No harmful recovery guarantee follows; flag for human review."
|
||||
},
|
||||
{
|
||||
"answer": "S04-V",
|
||||
"answer_sha256": "f388d3514a2c73c5a237c1559c7b47a945c2bac43bf42ba3fe604479206d3f2f",
|
||||
"criteria": {
|
||||
"C1": {
|
||||
"score": 1,
|
||||
"locator": "server's status for K is unknown",
|
||||
"reason": "Does not infer rejection from Cancel."
|
||||
},
|
||||
"C2": {
|
||||
"score": 1,
|
||||
"locator": "cannot safely assume a new submission will not result in a duplicate",
|
||||
"reason": "Explicitly rejects a safety guarantee."
|
||||
},
|
||||
"R": {
|
||||
"score": 0,
|
||||
"locator": "should be treated as a new, distinct operation with a fresh identity",
|
||||
"reason": "Does not specify reconciliation before replacement or remote reversal after known acceptance."
|
||||
},
|
||||
"Q": {
|
||||
"score": 1,
|
||||
"locator": "server's behavior regarding duplicate submissions and its retention window",
|
||||
"reason": "Names missing contracts."
|
||||
}
|
||||
},
|
||||
"unsupported_claims": [],
|
||||
"suspected_condition_leakage": false,
|
||||
"notes": "Borderline: recommends fresh identity but warns duplication. Score recovery omission; do not infer an explicit duplicate-free guarantee the answer does not make."
|
||||
},
|
||||
{
|
||||
"answer": "S05-U",
|
||||
"answer_sha256": "338eafdf7ce803f1cae6a5126596a6f75683a7eef0eb7c3ec7ca92b765337d00",
|
||||
"criteria": {
|
||||
"C1": {
|
||||
"score": 0,
|
||||
"locator": "original pending operation P and its identity must also be retained",
|
||||
"reason": "No explicit atomic block/supersession of P alongside c and Q, despite restart mention."
|
||||
},
|
||||
"C2": {
|
||||
"score": 1,
|
||||
"locator": "should not alter the active content or the status of Q",
|
||||
"reason": "Scopes old receipt to old operation."
|
||||
},
|
||||
"R": {
|
||||
"score": 1,
|
||||
"locator": "how the receiver handles an in-flight operation ... superseding operation",
|
||||
"reason": "Requires server handling of the in-flight operation, rather than trusting local retirement."
|
||||
},
|
||||
"Q": {
|
||||
"score": 1,
|
||||
"locator": "clear contract for the receiver's ancestry and precondition behavior",
|
||||
"reason": "Names missing contract."
|
||||
}
|
||||
},
|
||||
"unsupported_claims": [],
|
||||
"suspected_condition_leakage": false,
|
||||
"notes": ""
|
||||
},
|
||||
{
|
||||
"answer": "S05-V",
|
||||
"answer_sha256": "8685235bcfc59fc1f3b2e922e47cd6bb46eee18daf0c224d11a16775197fee7d",
|
||||
"criteria": {
|
||||
"C1": {
|
||||
"score": 0,
|
||||
"locator": "potentially flag that P's original intent was superseded",
|
||||
"reason": "Does not specify atomic queue transition/crash boundary."
|
||||
},
|
||||
"C2": {
|
||||
"score": 1,
|
||||
"locator": "update P's status to \"accepted\" but **not** reapply P's changes to revision c or Q",
|
||||
"reason": "Restricts receipt to P; no status promotion of Q."
|
||||
},
|
||||
"R": {
|
||||
"score": 1,
|
||||
"locator": "Without knowing how the receiver handles concurrent updates and what preconditions it enforces",
|
||||
"reason": "Does not assume local queue action prevents remote stale write."
|
||||
},
|
||||
"Q": {
|
||||
"score": 1,
|
||||
"locator": "critical missing contract is the receiver's ancestry and precondition behavior",
|
||||
"reason": "Explicit missing contract."
|
||||
}
|
||||
},
|
||||
"unsupported_claims": [],
|
||||
"suspected_condition_leakage": false,
|
||||
"notes": "C2 uses semantic reading of not applying P to Q; reviewer may ask for more explicit Q acknowledgement scope."
|
||||
},
|
||||
{
|
||||
"answer": "S06-U",
|
||||
"answer_sha256": "6c60c9c12851331cfd8f3f1af4ac6455cdf41e62b79d89fb7b37f464707ace12",
|
||||
"criteria": {
|
||||
"C1": {
|
||||
"score": 1,
|
||||
"locator": "cannot guarantee edits survive device loss or complete power failure",
|
||||
"reason": "Does not upgrade restart test evidence."
|
||||
},
|
||||
"C2": {
|
||||
"score": 1,
|
||||
"locator": "two copies ... same device ... sync protocol propagates deletions, which risks data loss",
|
||||
"reason": "Recognizes the stated copies/sync do not establish independent recovery."
|
||||
},
|
||||
"R": {
|
||||
"score": 1,
|
||||
"locator": "remove conflict review ... eliminates the user's ability to resolve divergent versions",
|
||||
"reason": "Retains accessible resolution."
|
||||
},
|
||||
"Q": {
|
||||
"score": 0,
|
||||
"locator": "no documented contract ... device loss or power failure",
|
||||
"reason": "Omits an independently recoverable backup/restore contract."
|
||||
}
|
||||
},
|
||||
"unsupported_claims": [],
|
||||
"suspected_condition_leakage": false,
|
||||
"notes": "C2 is semantically adequate about inadequate recovery, though it does not use the word backup; Q still requires independent restore contract."
|
||||
},
|
||||
{
|
||||
"answer": "S06-V",
|
||||
"answer_sha256": "f5965dd3ada0156bf46bbe5cdde96893b3caecd8600729d322d2b934ba4a4c0e",
|
||||
"criteria": {
|
||||
"C1": {
|
||||
"score": 1,
|
||||
"locator": "cannot guarantee survival ... power failure",
|
||||
"reason": "Correctly denies the promised guarantee."
|
||||
},
|
||||
"C2": {
|
||||
"score": 1,
|
||||
"locator": "same device ... sync protocol propagates deletions, which is insufficient for robust recovery",
|
||||
"reason": "Recognizes no independent recovery protection."
|
||||
},
|
||||
"R": {
|
||||
"score": 1,
|
||||
"locator": "Removing conflict review from the keyboard interface is premature and risky",
|
||||
"reason": "Rejects removal of accessible resolution."
|
||||
},
|
||||
"Q": {
|
||||
"score": 0,
|
||||
"locator": "no documented contract for restart-safe storage beyond process restart",
|
||||
"reason": "Omits independent backup and restore requirement."
|
||||
}
|
||||
},
|
||||
"unsupported_claims": [
|
||||
{
|
||||
"quote": "it cannot recover from power or device loss",
|
||||
"reason": "Converts lack of tested power-loss/device-loss evidence into categorical inability. Unsupported certainty; not a claim of guaranteed survival."
|
||||
}
|
||||
],
|
||||
"suspected_condition_leakage": false,
|
||||
"notes": "Flag for human adjudication: this is an unsupported negative capability assertion, separate from the correctly rejected guarantee. C1 still meets its narrow requirement."
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
# Human review handoff
|
||||
|
||||
Status: ready after masked answers are generated; no human scores yet.
|
||||
|
||||
Start here without opening assignment.json, raw/, results or editorial scores.
|
||||
Read [the scoring rules](../protocol.md) and [the scenarios/rubric](../scenarios.json).
|
||||
Each scenario has U and V responses in [masked](../masked/). These are the actual
|
||||
model answers without condition metadata; no semantic redaction was applied.
|
||||
|
||||
Copy [the empty scoring sheet](scores-template.json) to a new file. Enter your name,
|
||||
date and whether you saw the assignment or prior scores. For each answer, score
|
||||
C1, C2, R and Q as 1 or 0 using the frozen rubric, with quotations or locators.
|
||||
List unsupported guarantees separately (use [] only after checking). No credit for
|
||||
pattern vocabulary. Flag ambiguity and suspected condition clues. Return your sheet
|
||||
before inspecting the assignment key. No result is implied by this template.
|
||||
|
||||
This is 12 short answers across six pairs. Reviewing all is preferred; if reviewing
|
||||
a subset, record which were chosen before reading them. A partial review remains
|
||||
partial. The repository owner prepared the materials and editorial scores; a human
|
||||
reviewer provides the independent assessment still missing from this pilot.
|
||||
|
||||
After submission, compare disagreements against the same rubric and retain both
|
||||
original judgments and any adjudication. Do not replace disagreement with an average
|
||||
or silently overwrite the original sheet. Work owner: FEP-WP-0002 T03.
|
||||
|
|
@ -0,0 +1,223 @@
|
|||
{
|
||||
"reviewer": null,
|
||||
"reviewed_at": null,
|
||||
"read_assignment_or_results_before_scoring": null,
|
||||
"answers": [
|
||||
{
|
||||
"answer": "S01-U",
|
||||
"scores": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"evidence": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"unsupported_claims": null,
|
||||
"suspected_condition_leakage": null,
|
||||
"notes": null
|
||||
},
|
||||
{
|
||||
"answer": "S01-V",
|
||||
"scores": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"evidence": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"unsupported_claims": null,
|
||||
"suspected_condition_leakage": null,
|
||||
"notes": null
|
||||
},
|
||||
{
|
||||
"answer": "S02-U",
|
||||
"scores": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"evidence": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"unsupported_claims": null,
|
||||
"suspected_condition_leakage": null,
|
||||
"notes": null
|
||||
},
|
||||
{
|
||||
"answer": "S02-V",
|
||||
"scores": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"evidence": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"unsupported_claims": null,
|
||||
"suspected_condition_leakage": null,
|
||||
"notes": null
|
||||
},
|
||||
{
|
||||
"answer": "S03-U",
|
||||
"scores": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"evidence": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"unsupported_claims": null,
|
||||
"suspected_condition_leakage": null,
|
||||
"notes": null
|
||||
},
|
||||
{
|
||||
"answer": "S03-V",
|
||||
"scores": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"evidence": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"unsupported_claims": null,
|
||||
"suspected_condition_leakage": null,
|
||||
"notes": null
|
||||
},
|
||||
{
|
||||
"answer": "S04-U",
|
||||
"scores": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"evidence": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"unsupported_claims": null,
|
||||
"suspected_condition_leakage": null,
|
||||
"notes": null
|
||||
},
|
||||
{
|
||||
"answer": "S04-V",
|
||||
"scores": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"evidence": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"unsupported_claims": null,
|
||||
"suspected_condition_leakage": null,
|
||||
"notes": null
|
||||
},
|
||||
{
|
||||
"answer": "S05-U",
|
||||
"scores": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"evidence": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"unsupported_claims": null,
|
||||
"suspected_condition_leakage": null,
|
||||
"notes": null
|
||||
},
|
||||
{
|
||||
"answer": "S05-V",
|
||||
"scores": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"evidence": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"unsupported_claims": null,
|
||||
"suspected_condition_leakage": null,
|
||||
"notes": null
|
||||
},
|
||||
{
|
||||
"answer": "S06-U",
|
||||
"scores": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"evidence": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"unsupported_claims": null,
|
||||
"suspected_condition_leakage": null,
|
||||
"notes": null
|
||||
},
|
||||
{
|
||||
"answer": "S06-V",
|
||||
"scores": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"evidence": {
|
||||
"C1": null,
|
||||
"C2": null,
|
||||
"R": null,
|
||||
"Q": null
|
||||
},
|
||||
"unsupported_claims": null,
|
||||
"suspected_condition_leakage": null,
|
||||
"notes": null
|
||||
}
|
||||
]
|
||||
}
|
||||
6
research/evaluations/language-pilot-01/masked/S01-U.md
Normal file
6
research/evaluations/language-pilot-01/masked/S01-U.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
This architecture should result in the following:
|
||||
|
||||
* **Outcome/State:** Operation A is confirmed accepted, and draft B is preserved. The replica read is superseded and ignored.
|
||||
* **Preserved Information:** The accepted revision of A, the full content of draft B, and the identity of operation A are retained.
|
||||
* **Recovery Action and Rejected Alternative:** The system should refresh observations based on the accepted revision of A, while keeping draft B visible. Blindly overwriting draft B with the stale replica data is incorrect.
|
||||
* **Missing Contract or Clarification:** The replica consistency contract needs to be explicitly defined to determine how and when its data should be considered authoritative or how to reconcile it with confirmed server state.
|
||||
4
research/evaluations/language-pilot-01/masked/S01-V.md
Normal file
4
research/evaluations/language-pilot-01/masked/S01-V.md
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
* **Outcome/State:** Operation A is accepted, but the replica is stale, and draft B is present.
|
||||
* **Preserved Information:** Accepted revision of A, draft B, and the original identity of A.
|
||||
* **Recovery Action:** Refresh the affected observations while preserving draft B. Do not retry A with a new identity.
|
||||
* **Missing Contract/Clarification:** The replica consistency contract needs to be explicitly defined to understand how to handle the stale read.
|
||||
9
research/evaluations/language-pilot-01/masked/S02-U.md
Normal file
9
research/evaluations/language-pilot-01/masked/S02-U.md
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
Here's a recommendation for the scenario:
|
||||
|
||||
* **Outcome/State:** The original message was sent, but its delivery status is unknown due to a lost response. The local operation ID is gone, and the server's deduplication scope is device-specific.
|
||||
|
||||
* **Preserved Information:** Only the immutable message payload remains locally. The original operation identity and any server-side acceptance evidence are lost to the client.
|
||||
|
||||
* **Recovery Action and Rejected Alternative:** The product's suggestion to generate a fresh ID and retry is problematic. This risks sending a duplicate message if the original was accepted. Instead, the user should be prompted to manually reconcile the message. This could involve presenting the message to the user and asking if they want to resend it, acknowledging potential duplication.
|
||||
|
||||
* **Missing Contract or Clarification:** The server's lookup support and retention for operation IDs are critical. Without these, the client cannot determine if the original message was processed. A contract for server-side reconciliation or query by payload content would be beneficial.
|
||||
6
research/evaluations/language-pilot-01/masked/S02-V.md
Normal file
6
research/evaluations/language-pilot-01/masked/S02-V.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
Here's an evaluation and recovery plan:
|
||||
|
||||
* **Outcome/State:** The original message payload is present, but its operation identity is lost, leading to an unknown server outcome for the initial send attempt.
|
||||
* **Preserved Information:** Only the immutable message payload itself is retained across device changes and reinstallation.
|
||||
* **Recovery Action and Rejected Alternative:** The product's suggestion to generate a fresh ID and automatically retry is flawed. This risks duplicating the message if the original send was successful. Instead, the user must be prompted to manually confirm re-sending the message, acknowledging the potential for duplication.
|
||||
* **Missing Contract or Clarification:** The server's deduplication scope and retention for operation IDs are critical but unspecified. Without this, it's impossible to know if the original message was processed or if a fresh ID would truly be safe.
|
||||
9
research/evaluations/language-pilot-01/masked/S03-U.md
Normal file
9
research/evaluations/language-pilot-01/masked/S03-U.md
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
Here's a recommendation for the frontend architecture:
|
||||
|
||||
* **Outcome/State:** The editor should display the confidential notes as locally committed but with a clear, persistent indication that the pending upload was explicitly rejected due to authorization failure. The notes remain editable locally.
|
||||
|
||||
* **Preserved Information:** The full content of the confidential notes, their local commit identity, and the identity of the rejected upload attempt must be retained. The explicit server rejection message should also be preserved and associated with the failed upload.
|
||||
|
||||
* **Recovery Action and Rejected Alternative:** The user should be presented with options to either discard the local changes (if they choose to, understanding the implications) or attempt to export/save the content locally. Blindly retrying the upload with a new identity is rejected as it doesn't address the authorization issue and could lead to data duplication if the server's rejection was transient.
|
||||
|
||||
* **Missing Contract or Clarification:** A clear contract is needed regarding the retention and access policies for locally committed data when server authorization is revoked. Specifically, can the user continue to view, edit, or export these notes, or should they be purged?
|
||||
6
research/evaluations/language-pilot-01/masked/S03-V.md
Normal file
6
research/evaluations/language-pilot-01/masked/S03-V.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
This architecture needs refinement to handle authorization denial gracefully.
|
||||
|
||||
* **Outcome/State:** The local commit remains, but the pending upload is explicitly rejected due to authorization. The editor is in a state of "accepted locally, rejected remotely."
|
||||
* **Preserved Information:** The full local commit of confidential notes, including its revision and operation identity, must be retained. The server's explicit denial and the reason (authorization failure) should also be recorded.
|
||||
* **Recovery Action and Rejected Alternative:** The system should prompt the user to explicitly reconcile the rejected upload, perhaps by allowing them to save the notes locally or export them. Blindly discarding the local commit or attempting a re-upload with a new identity is inappropriate as it would lose data and violate the "never lose edits" requirement.
|
||||
* **Missing Contract or Clarification:** A clear contract is needed regarding the retention, viewing, and export of local data for users whose write authorization has been revoked. This will dictate the specific recovery options presented to the user.
|
||||
6
research/evaluations/language-pilot-01/masked/S04-U.md
Normal file
6
research/evaluations/language-pilot-01/masked/S04-U.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
The proposed architecture has a critical flaw for this scenario.
|
||||
|
||||
* **Outcome/State:** The booking's final state is unknown. The server may have accepted operation K despite the timeout and local cancellation.
|
||||
* **Preserved Information:** The client has discarded the original operation K's identity and intent.
|
||||
* **Recovery Action:** The only safe action is to inform the user that the previous booking's status is unknown and they should check with the service provider. A new submission is unsafe.
|
||||
* **Missing Contract:** The server lacks a documented operation-status lookup or deduplication retention guarantee, which is essential for safely retrying or confirming operations after client-side uncertainty.
|
||||
6
research/evaluations/language-pilot-01/masked/S04-V.md
Normal file
6
research/evaluations/language-pilot-01/masked/S04-V.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
Here's a recommendation for the scenario:
|
||||
|
||||
* **Outcome/State:** The client should not remove the pending entry for operation K upon local "Cancel." The server's status for K is unknown.
|
||||
* **Preserved Information:** The client must retain the original operation K's identity and payload, along with its "pending" status, even after local cancellation.
|
||||
* **Recovery Action and Rejected Alternative:** The user's request to submit the "same booking again" should be treated as a new, distinct operation with a fresh identity. Blindly retrying K with its original identity is rejected because the server lacks deduplication guarantees.
|
||||
* **Missing Contract or Clarification:** A critical missing contract is the server's behavior regarding duplicate submissions and its retention window for operation identities. Without this, the client cannot safely assume a new submission will not result in a duplicate booking if K was, in fact, accepted.
|
||||
9
research/evaluations/language-pilot-01/masked/S05-U.md
Normal file
9
research/evaluations/language-pilot-01/masked/S05-U.md
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
Here's a recommendation for the frontend architecture based on the scenario:
|
||||
|
||||
* **Outcome/State:** The system should transition to a state where revision `c` is the active local content, with pending operation `Q` awaiting remote acceptance. The late receipt for `P` should update its own operation history, marking `P` as remotely accepted, but it should not alter the active content or the status of `Q`.
|
||||
|
||||
* **Preserved Information:** Both divergent revisions `a` and `b` should be preserved with their identities and provenance until `c` is fully committed and resolved. The original pending operation `P` and its identity must also be retained, along with the new pending operation `Q`.
|
||||
|
||||
* **Recovery Action and Rejected Alternative:** Upon process restart, the system must recover `c` as the active content and `Q` as the pending operation. The late receipt for `P` should not trigger a re-application of `P` or a rollback to `a`, as this would erase the user's explicit resolution into `c`.
|
||||
|
||||
* **Missing Contract or Clarification:** The architecture needs a clear contract for the receiver's ancestry and precondition behavior. Specifically, it must define how the receiver handles an in-flight operation (`P`) when a superseding operation (`Q` derived from `c`) is subsequently sent, and how it would respond to `P` if `b` was already accepted.
|
||||
6
research/evaluations/language-pilot-01/masked/S05-V.md
Normal file
6
research/evaluations/language-pilot-01/masked/S05-V.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
This architecture has several issues that could lead to data loss or corruption.
|
||||
|
||||
* **Outcome/State:** The system will transition from having pending operations P and Q, and revisions a, b, and c, to a state where P's receipt arrives. The system needs to define how the late receipt for P interacts with the already-resolved revision c and the new pending operation Q.
|
||||
* **Preserved Information:** The system must preserve revisions a, b, and c, along with the identities of pending operations P and Q, and the fact that P was in flight. The receipt for P, when it arrives, must also be preserved with its associated operation identity.
|
||||
* **Recovery Action and Rejected Alternative:** Upon receiving the late receipt for P, the system should update P's status to "accepted" but **not** reapply P's changes to revision c or Q. Blindly applying P's changes would overwrite the explicit reconciliation that produced c. Instead, the system should acknowledge P's acceptance and potentially flag that P's original intent was superseded by the reconciliation.
|
||||
* **Missing Contract or Clarification:** The critical missing contract is the receiver's ancestry and precondition behavior. Without knowing how the receiver handles concurrent updates and what preconditions it enforces, it's impossible to guarantee that P's original intent wasn't already superseded on the server, or how the server would react to a re-submission of P after c was uploaded.
|
||||
6
research/evaluations/language-pilot-01/masked/S06-U.md
Normal file
6
research/evaluations/language-pilot-01/masked/S06-U.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
Here's an assessment of the proposed design:
|
||||
|
||||
* **Outcome/State:** The design cannot guarantee edits survive device loss or complete power failure, nor can it claim recovery is solved.
|
||||
* **Preserved Information:** Only two copies of a conflict are stored on the same device, and the sync protocol propagates deletions, which risks data loss.
|
||||
* **Recovery Action and Rejected Alternative:** The proposal to remove conflict review from the keyboard interface is premature and dangerous, as it eliminates the user's ability to resolve divergent versions.
|
||||
* **Missing Contract or Clarification:** There is no documented contract for restart-safe offline editing that covers device loss or power failure, nor is there a clear definition of how deletions are handled during conflict resolution.
|
||||
6
research/evaluations/language-pilot-01/masked/S06-V.md
Normal file
6
research/evaluations/language-pilot-01/masked/S06-V.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
Here's an assessment of the proposed frontend architecture:
|
||||
|
||||
* **Outcome/State:** The product cannot guarantee survival of edits through device loss or complete power failure, nor can it claim conflict resolution is solved.
|
||||
* **Preserved Information:** Only two copies of a conflict are retained on the same device, and the sync protocol propagates deletions, which is insufficient for robust recovery.
|
||||
* **Recovery Action and Rejected Alternative:** The current local transaction only handles process restarts; it cannot recover from power or device loss. Removing conflict review from the keyboard interface is premature and risky.
|
||||
* **Missing Contract or Clarification:** There's no documented contract for restart-safe storage beyond process restart, nor for how deletions are handled during conflict resolution or synchronization across devices.
|
||||
6
research/evaluations/language-pilot-01/raw/r01/answer.md
Normal file
6
research/evaluations/language-pilot-01/raw/r01/answer.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
This architecture should result in the following:
|
||||
|
||||
* **Outcome/State:** Operation A is confirmed accepted, and draft B is preserved. The replica read is superseded and ignored.
|
||||
* **Preserved Information:** The accepted revision of A, the full content of draft B, and the identity of operation A are retained.
|
||||
* **Recovery Action and Rejected Alternative:** The system should refresh observations based on the accepted revision of A, while keeping draft B visible. Blindly overwriting draft B with the stale replica data is incorrect.
|
||||
* **Missing Contract or Clarification:** The replica consistency contract needs to be explicitly defined to determine how and when its data should be considered authoritative or how to reconcile it with confirmed server state.
|
||||
43
research/evaluations/language-pilot-01/raw/r01/prompt.txt
Normal file
43
research/evaluations/language-pilot-01/raw/r01/prompt.txt
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
You are reviewing a proposed frontend architecture. Use the reference only where applicable. Return at most 220 words, using four short bullets: (1) outcome/state, (2) preserved information, (3) recovery action and rejected alternative, (4) missing contract or clarification. Explain the reasoning without naming or quoting reference headings, pattern names, or the presentation style. Do not infer undocumented guarantees. No tables, scores, source searching or tool calls. The task is a recommendation, not a claim you implemented or tested it.
|
||||
|
||||
REFERENCE
|
||||
Reference guidance. All mechanisms remain research hypotheses with limited evidence; apply only when their conditions hold. Distinguish assumed guarantees from observations. Do not infer missing backend, storage, access or retention contracts.
|
||||
|
||||
## Local Draft State
|
||||
|
||||
Keep the last observed record, unfinished edits and current server observation separate. A refresh or rejected write must not silently replace newer typing. Submitted content may move into a pending-operation record before the composer is cleared. Choose navigation/restart retention and privacy rules explicitly; local content alone is not authority.
|
||||
|
||||
## Version-Checked Mutation
|
||||
|
||||
When an update depends on an observed base, send that expected version. Require atomic server validation and commit; a stale rejection preserves intent for comparison with current data. Blindly substituting a new version on retry can erase concurrent intent. Version preconditions and retry identities solve different problems.
|
||||
|
||||
## Mutation Outcome Tracking
|
||||
|
||||
Separate pending, accepted, explicitly rejected and unknown outcomes using operation-specific evidence. A timeout, cancelled request, generic server error or rejected promise alone does not prove rejection. An authenticated matching server event may confirm acceptance before a request finishes. Later transport errors cannot reverse that evidence or clear newer work. Retry the same immutable payload and operation identity only within a documented deduplication scope and retention window. Missing identity or expired receipts require reconciliation, not a claim that a fresh ID is safe. Cancelling local tracking does not undo remote acceptance; reversal is a separate authorized action.
|
||||
|
||||
## Invalidate and Reproject
|
||||
|
||||
Acceptance and view freshness are separate. Refresh affected observations while preserving unfinished work. Local request generations can reject superseded reads, but cannot order opaque server versions or repair stale replicas. A read-after-write contract or documented server ordering is needed. Report accepted-with-unresolved-freshness when appropriate; a refresh failure does not undo a write.
|
||||
|
||||
## Local Commit Boundary
|
||||
|
||||
For restart-safe offline editing, acknowledge local save only after a recoverable commit of the revision and its pending upload intent. Immediate feedback is not a storage receipt. Ordinary process restart, power loss and device loss require different evidence. An in-memory queue alone cannot satisfy a restart promise. Local success does not imply remote acceptance.
|
||||
|
||||
## Pending Synchronization
|
||||
|
||||
Retain revision and operation identity with pending work across restart. A sent request is not an acknowledgement; receiver acceptance is not delivery to every device or backup. Define receiver deduplication, ancestry and authorization behavior. Lost acknowledgements retain the original identity. Block obsolete uploads on detected divergence; atomically supersede them when recording a replacement. Local retirement cannot stop an in-flight stale write: the receiver must enforce preconditions. A late receipt updates its own operation history, not a newer operation.
|
||||
|
||||
## Conflict Copy
|
||||
|
||||
When incompatible edits are detected, retain alternate content with identity and provenance before destructive replacement. Selecting an active copy is not semantic resolution. Protect both versions equally and expose unresolved status. Storage costs, unnoticed copies and missed conflict detection remain risks. Sync does not substitute for separately recoverable backup.
|
||||
|
||||
## Explicit Reconciliation
|
||||
|
||||
Let a person compare, choose or combine divergent versions and commit the decision before removing recovery material. Accessible notification, keyboard comparison and stable focus are requirements, not automatic consequences of storing copies. Two devices can reconcile concurrently; the resulting revisions still need divergence handling. A version store may preserve the same evidence without separate copy files; justified domain-aware merging is an alternative.
|
||||
|
||||
Connections: Local Draft State supports recovery from Version-Checked Mutation. Mutation Outcome Tracking supplies acceptance evidence to Invalidate and Reproject. Local Commit Boundary enables Pending Synchronization. Detected divergence can lead from Pending Synchronization to Conflict Copy, then Explicit Reconciliation, which creates another local commit. These connections are conditional, not universal ordering.
|
||||
|
||||
Evidence limits shared by both presentations: Joplin/Obsidian document conflict copies but do not establish our atomic crash contract. Kubernetes documents conditional updates; GitLab has a different, sometimes nonspecific failure surface. Stripe documents bounded idempotency. React Router illustrates revalidation but browser cancellation does not stop server writes. Pinned Zulip/Element source shows partial draft and pending-message mechanisms. A published Matrix SDK test with a synthetic receiver shows a rejected promise can coexist with a reconciled event, and fresh identity after local cancellation can duplicate simulated acceptance. This is not full application/server validation or independent full-pattern recurrence.
|
||||
|
||||
SCENARIO
|
||||
A server-validated record editor submits operation A. Its request promise rejects, but an authenticated server event identifying A and its accepted revision arrives. The user has already typed draft B. A previously started replica read then returns a different opaque version; the replica consistency contract is unspecified. Explain the correct visible states and next actions.
|
||||
58
research/evaluations/language-pilot-01/raw/r01/receipt.json
Normal file
58
research/evaluations/language-pilot-01/raw/r01/receipt.json
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"scenario": "S01",
|
||||
"condition": "patterns",
|
||||
"prompt_sha256": "4f0f952134b4f6e99ef1c9bca4180df301d545fc470206f257b1f5a7171bf6f9",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e",
|
||||
"status": "completed",
|
||||
"output_words": 116,
|
||||
"elapsed_seconds": 1.908,
|
||||
"responses": [
|
||||
{
|
||||
"stage": "decision-review",
|
||||
"model": "google/gemini-2.5-flash",
|
||||
"usage": {
|
||||
"prompt_tokens": 1042,
|
||||
"completion_tokens": 149,
|
||||
"total_tokens": 1191
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
"provider": "openrouter",
|
||||
"response_id": "gen-1790445703-TYHROpbBuwjMgFzp5iuo",
|
||||
"latency_seconds": 1.799,
|
||||
"profile": "custodian-triage-balanced",
|
||||
"profile_model": "google/gemini-2.5-flash",
|
||||
"profile_provider": "openrouter",
|
||||
"diagnostics": {
|
||||
"available": true,
|
||||
"effective_settings": {
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"reasoning_effort_present": false,
|
||||
"reasoning_present": false
|
||||
},
|
||||
"provider_usage": {
|
||||
"prompt_tokens": 1042,
|
||||
"completion_tokens": 149,
|
||||
"total_tokens": 1191,
|
||||
"cost": 0.0006851,
|
||||
"completion_tokens_details": {
|
||||
"reasoning_tokens": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"content_characters": 768,
|
||||
"content_sha256": "5eda5ddfd9725e38aa5827ffd2c929d64074f8c0fd31ee9b5f667362a5c28691"
|
||||
}
|
||||
]
|
||||
}
|
||||
18
research/evaluations/language-pilot-01/raw/r01/started.json
Normal file
18
research/evaluations/language-pilot-01/raw/r01/started.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"status": "attempting",
|
||||
"scenario": "S01",
|
||||
"condition": "patterns",
|
||||
"prompt_sha256": "4f0f952134b4f6e99ef1c9bca4180df301d545fc470206f257b1f5a7171bf6f9",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e"
|
||||
}
|
||||
4
research/evaluations/language-pilot-01/raw/r02/answer.md
Normal file
4
research/evaluations/language-pilot-01/raw/r02/answer.md
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
* **Outcome/State:** Operation A is accepted, but the replica is stale, and draft B is present.
|
||||
* **Preserved Information:** Accepted revision of A, draft B, and the original identity of A.
|
||||
* **Recovery Action:** Refresh the affected observations while preserving draft B. Do not retry A with a new identity.
|
||||
* **Missing Contract/Clarification:** The replica consistency contract needs to be explicitly defined to understand how to handle the stale read.
|
||||
27
research/evaluations/language-pilot-01/raw/r02/prompt.txt
Normal file
27
research/evaluations/language-pilot-01/raw/r02/prompt.txt
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
You are reviewing a proposed frontend architecture. Use the reference only where applicable. Return at most 220 words, using four short bullets: (1) outcome/state, (2) preserved information, (3) recovery action and rejected alternative, (4) missing contract or clarification. Explain the reasoning without naming or quoting reference headings, pattern names, or the presentation style. Do not infer undocumented guarantees. No tables, scores, source searching or tool calls. The task is a recommendation, not a claim you implemented or tested it.
|
||||
|
||||
REFERENCE
|
||||
Reference guidance. All mechanisms remain research hypotheses with limited evidence; apply only when their conditions hold. Distinguish assumed guarantees from observations. Do not infer missing backend, storage, access or retention contracts.
|
||||
|
||||
Keep the last observed record, unfinished edits and current server observation separate. A refresh or rejected write must not silently replace newer typing. Submitted content may move into a pending-operation record before the composer is cleared. Choose navigation/restart retention and privacy rules explicitly; local content alone is not authority.
|
||||
|
||||
When an update depends on an observed base, send that expected version. Require atomic server validation and commit; a stale rejection preserves intent for comparison with current data. Blindly substituting a new version on retry can erase concurrent intent. Version preconditions and retry identities solve different problems.
|
||||
|
||||
Separate pending, accepted, explicitly rejected and unknown outcomes using operation-specific evidence. A timeout, cancelled request, generic server error or rejected promise alone does not prove rejection. An authenticated matching server event may confirm acceptance before a request finishes. Later transport errors cannot reverse that evidence or clear newer work. Retry the same immutable payload and operation identity only within a documented deduplication scope and retention window. Missing identity or expired receipts require reconciliation, not a claim that a fresh ID is safe. Cancelling local tracking does not undo remote acceptance; reversal is a separate authorized action.
|
||||
|
||||
Acceptance and view freshness are separate. Refresh affected observations while preserving unfinished work. Local request generations can reject superseded reads, but cannot order opaque server versions or repair stale replicas. A read-after-write contract or documented server ordering is needed. Report accepted-with-unresolved-freshness when appropriate; a refresh failure does not undo a write.
|
||||
|
||||
For restart-safe offline editing, acknowledge local save only after a recoverable commit of the revision and its pending upload intent. Immediate feedback is not a storage receipt. Ordinary process restart, power loss and device loss require different evidence. An in-memory queue alone cannot satisfy a restart promise. Local success does not imply remote acceptance.
|
||||
|
||||
Retain revision and operation identity with pending work across restart. A sent request is not an acknowledgement; receiver acceptance is not delivery to every device or backup. Define receiver deduplication, ancestry and authorization behavior. Lost acknowledgements retain the original identity. Block obsolete uploads on detected divergence; atomically supersede them when recording a replacement. Local retirement cannot stop an in-flight stale write: the receiver must enforce preconditions. A late receipt updates its own operation history, not a newer operation.
|
||||
|
||||
When incompatible edits are detected, retain alternate content with identity and provenance before destructive replacement. Selecting an active copy is not semantic resolution. Protect both versions equally and expose unresolved status. Storage costs, unnoticed copies and missed conflict detection remain risks. Sync does not substitute for separately recoverable backup.
|
||||
|
||||
Let a person compare, choose or combine divergent versions and commit the decision before removing recovery material. Accessible notification, keyboard comparison and stable focus are requirements, not automatic consequences of storing copies. Two devices can reconcile concurrently; the resulting revisions still need divergence handling. A version store may preserve the same evidence without separate copy files; justified domain-aware merging is an alternative.
|
||||
|
||||
Connections in the workflow: preserved unfinished edits support recovery from rejected conditional writes. Operation-specific acceptance evidence informs refreshing observations. A local recoverable commit enables restartable propagation. Detected divergence during propagation can require preservation of alternate content, then an intentional resolution, which creates another local commit. These connections are conditional, not universal ordering.
|
||||
|
||||
Evidence limits shared by both presentations: Joplin/Obsidian document conflict copies but do not establish our atomic crash contract. Kubernetes documents conditional updates; GitLab has a different, sometimes nonspecific failure surface. Stripe documents bounded idempotency. React Router illustrates revalidation but browser cancellation does not stop server writes. Pinned Zulip/Element source shows partial draft and pending-message mechanisms. A published Matrix SDK test with a synthetic receiver shows a rejected promise can coexist with a reconciled event, and fresh identity after local cancellation can duplicate simulated acceptance. This is not full application/server validation or independent full-pattern recurrence.
|
||||
|
||||
SCENARIO
|
||||
A server-validated record editor submits operation A. Its request promise rejects, but an authenticated server event identifying A and its accepted revision arrives. The user has already typed draft B. A previously started replica read then returns a different opaque version; the replica consistency contract is unspecified. Explain the correct visible states and next actions.
|
||||
58
research/evaluations/language-pilot-01/raw/r02/receipt.json
Normal file
58
research/evaluations/language-pilot-01/raw/r02/receipt.json
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"scenario": "S01",
|
||||
"condition": "prose",
|
||||
"prompt_sha256": "8c4a2c61ede88cdf619abf15d7f6fe9d3cc47236514043479db1914cd4a69baf",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e",
|
||||
"status": "completed",
|
||||
"output_words": 70,
|
||||
"elapsed_seconds": 1.224,
|
||||
"responses": [
|
||||
{
|
||||
"stage": "decision-review",
|
||||
"model": "google/gemini-2.5-flash",
|
||||
"usage": {
|
||||
"prompt_tokens": 1001,
|
||||
"completion_tokens": 100,
|
||||
"total_tokens": 1101
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
"provider": "openrouter",
|
||||
"response_id": "gen-1790445704-t5Mfydsi6er6WDbR5h1h",
|
||||
"latency_seconds": 1.112,
|
||||
"profile": "custodian-triage-balanced",
|
||||
"profile_model": "google/gemini-2.5-flash",
|
||||
"profile_provider": "openrouter",
|
||||
"diagnostics": {
|
||||
"available": true,
|
||||
"effective_settings": {
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"reasoning_effort_present": false,
|
||||
"reasoning_present": false
|
||||
},
|
||||
"provider_usage": {
|
||||
"prompt_tokens": 1001,
|
||||
"completion_tokens": 100,
|
||||
"total_tokens": 1101,
|
||||
"cost": 0.0005503,
|
||||
"completion_tokens_details": {
|
||||
"reasoning_tokens": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"content_characters": 462,
|
||||
"content_sha256": "a63771e0792ab2df51720093a331e70e02a21120103f1adc7285e027721a8bf0"
|
||||
}
|
||||
]
|
||||
}
|
||||
18
research/evaluations/language-pilot-01/raw/r02/started.json
Normal file
18
research/evaluations/language-pilot-01/raw/r02/started.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"status": "attempting",
|
||||
"scenario": "S01",
|
||||
"condition": "prose",
|
||||
"prompt_sha256": "8c4a2c61ede88cdf619abf15d7f6fe9d3cc47236514043479db1914cd4a69baf",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e"
|
||||
}
|
||||
9
research/evaluations/language-pilot-01/raw/r03/answer.md
Normal file
9
research/evaluations/language-pilot-01/raw/r03/answer.md
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
Here's a recommendation for the scenario:
|
||||
|
||||
* **Outcome/State:** The original message was sent, but its delivery status is unknown due to a lost response. The local operation ID is gone, and the server's deduplication scope is device-specific.
|
||||
|
||||
* **Preserved Information:** Only the immutable message payload remains locally. The original operation identity and any server-side acceptance evidence are lost to the client.
|
||||
|
||||
* **Recovery Action and Rejected Alternative:** The product's suggestion to generate a fresh ID and retry is problematic. This risks sending a duplicate message if the original was accepted. Instead, the user should be prompted to manually reconcile the message. This could involve presenting the message to the user and asking if they want to resend it, acknowledging potential duplication.
|
||||
|
||||
* **Missing Contract or Clarification:** The server's lookup support and retention for operation IDs are critical. Without these, the client cannot determine if the original message was processed. A contract for server-side reconciliation or query by payload content would be beneficial.
|
||||
27
research/evaluations/language-pilot-01/raw/r03/prompt.txt
Normal file
27
research/evaluations/language-pilot-01/raw/r03/prompt.txt
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
You are reviewing a proposed frontend architecture. Use the reference only where applicable. Return at most 220 words, using four short bullets: (1) outcome/state, (2) preserved information, (3) recovery action and rejected alternative, (4) missing contract or clarification. Explain the reasoning without naming or quoting reference headings, pattern names, or the presentation style. Do not infer undocumented guarantees. No tables, scores, source searching or tool calls. The task is a recommendation, not a claim you implemented or tested it.
|
||||
|
||||
REFERENCE
|
||||
Reference guidance. All mechanisms remain research hypotheses with limited evidence; apply only when their conditions hold. Distinguish assumed guarantees from observations. Do not infer missing backend, storage, access or retention contracts.
|
||||
|
||||
Keep the last observed record, unfinished edits and current server observation separate. A refresh or rejected write must not silently replace newer typing. Submitted content may move into a pending-operation record before the composer is cleared. Choose navigation/restart retention and privacy rules explicitly; local content alone is not authority.
|
||||
|
||||
When an update depends on an observed base, send that expected version. Require atomic server validation and commit; a stale rejection preserves intent for comparison with current data. Blindly substituting a new version on retry can erase concurrent intent. Version preconditions and retry identities solve different problems.
|
||||
|
||||
Separate pending, accepted, explicitly rejected and unknown outcomes using operation-specific evidence. A timeout, cancelled request, generic server error or rejected promise alone does not prove rejection. An authenticated matching server event may confirm acceptance before a request finishes. Later transport errors cannot reverse that evidence or clear newer work. Retry the same immutable payload and operation identity only within a documented deduplication scope and retention window. Missing identity or expired receipts require reconciliation, not a claim that a fresh ID is safe. Cancelling local tracking does not undo remote acceptance; reversal is a separate authorized action.
|
||||
|
||||
Acceptance and view freshness are separate. Refresh affected observations while preserving unfinished work. Local request generations can reject superseded reads, but cannot order opaque server versions or repair stale replicas. A read-after-write contract or documented server ordering is needed. Report accepted-with-unresolved-freshness when appropriate; a refresh failure does not undo a write.
|
||||
|
||||
For restart-safe offline editing, acknowledge local save only after a recoverable commit of the revision and its pending upload intent. Immediate feedback is not a storage receipt. Ordinary process restart, power loss and device loss require different evidence. An in-memory queue alone cannot satisfy a restart promise. Local success does not imply remote acceptance.
|
||||
|
||||
Retain revision and operation identity with pending work across restart. A sent request is not an acknowledgement; receiver acceptance is not delivery to every device or backup. Define receiver deduplication, ancestry and authorization behavior. Lost acknowledgements retain the original identity. Block obsolete uploads on detected divergence; atomically supersede them when recording a replacement. Local retirement cannot stop an in-flight stale write: the receiver must enforce preconditions. A late receipt updates its own operation history, not a newer operation.
|
||||
|
||||
When incompatible edits are detected, retain alternate content with identity and provenance before destructive replacement. Selecting an active copy is not semantic resolution. Protect both versions equally and expose unresolved status. Storage costs, unnoticed copies and missed conflict detection remain risks. Sync does not substitute for separately recoverable backup.
|
||||
|
||||
Let a person compare, choose or combine divergent versions and commit the decision before removing recovery material. Accessible notification, keyboard comparison and stable focus are requirements, not automatic consequences of storing copies. Two devices can reconcile concurrently; the resulting revisions still need divergence handling. A version store may preserve the same evidence without separate copy files; justified domain-aware merging is an alternative.
|
||||
|
||||
Connections in the workflow: preserved unfinished edits support recovery from rejected conditional writes. Operation-specific acceptance evidence informs refreshing observations. A local recoverable commit enables restartable propagation. Detected divergence during propagation can require preservation of alternate content, then an intentional resolution, which creates another local commit. These connections are conditional, not universal ordering.
|
||||
|
||||
Evidence limits shared by both presentations: Joplin/Obsidian document conflict copies but do not establish our atomic crash contract. Kubernetes documents conditional updates; GitLab has a different, sometimes nonspecific failure surface. Stripe documents bounded idempotency. React Router illustrates revalidation but browser cancellation does not stop server writes. Pinned Zulip/Element source shows partial draft and pending-message mechanisms. A published Matrix SDK test with a synthetic receiver shows a rejected promise can coexist with a reconciled event, and fresh identity after local cancellation can duplicate simulated acceptance. This is not full application/server validation or independent full-pattern recurrence.
|
||||
|
||||
SCENARIO
|
||||
A mobile message composer queued an immutable payload and sent it. The response was lost. After logout, reinstall and login as a new device, only the payload remains; the original operation ID is lost. The server deduplication scope is device and endpoint. Lookup support and retention are unspecified. The product suggests generating a fresh ID and automatically retrying. Evaluate that suggestion and give a recovery plan.
|
||||
58
research/evaluations/language-pilot-01/raw/r03/receipt.json
Normal file
58
research/evaluations/language-pilot-01/raw/r03/receipt.json
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"scenario": "S02",
|
||||
"condition": "prose",
|
||||
"prompt_sha256": "29d62a36607a18d873212691418a36f800494ecb032d266326e78d8de265cdcb",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e",
|
||||
"status": "completed",
|
||||
"output_words": 162,
|
||||
"elapsed_seconds": 1.946,
|
||||
"responses": [
|
||||
{
|
||||
"stage": "decision-review",
|
||||
"model": "google/gemini-2.5-flash",
|
||||
"usage": {
|
||||
"prompt_tokens": 1015,
|
||||
"completion_tokens": 215,
|
||||
"total_tokens": 1230
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
"provider": "openrouter",
|
||||
"response_id": "gen-1790445706-dtVGZFmuL7mkmX1cY9ZB",
|
||||
"latency_seconds": 1.954,
|
||||
"profile": "custodian-triage-balanced",
|
||||
"profile_model": "google/gemini-2.5-flash",
|
||||
"profile_provider": "openrouter",
|
||||
"diagnostics": {
|
||||
"available": true,
|
||||
"effective_settings": {
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"reasoning_effort_present": false,
|
||||
"reasoning_present": false
|
||||
},
|
||||
"provider_usage": {
|
||||
"prompt_tokens": 1015,
|
||||
"completion_tokens": 215,
|
||||
"total_tokens": 1230,
|
||||
"cost": 0.000842,
|
||||
"completion_tokens_details": {
|
||||
"reasoning_tokens": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"content_characters": 1110,
|
||||
"content_sha256": "5cda87a8d3d5f7952ab3d855213c10056d1d43c26e137c0de2225ced5113d6ba"
|
||||
}
|
||||
]
|
||||
}
|
||||
18
research/evaluations/language-pilot-01/raw/r03/started.json
Normal file
18
research/evaluations/language-pilot-01/raw/r03/started.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"status": "attempting",
|
||||
"scenario": "S02",
|
||||
"condition": "prose",
|
||||
"prompt_sha256": "29d62a36607a18d873212691418a36f800494ecb032d266326e78d8de265cdcb",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e"
|
||||
}
|
||||
6
research/evaluations/language-pilot-01/raw/r04/answer.md
Normal file
6
research/evaluations/language-pilot-01/raw/r04/answer.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
Here's an evaluation and recovery plan:
|
||||
|
||||
* **Outcome/State:** The original message payload is present, but its operation identity is lost, leading to an unknown server outcome for the initial send attempt.
|
||||
* **Preserved Information:** Only the immutable message payload itself is retained across device changes and reinstallation.
|
||||
* **Recovery Action and Rejected Alternative:** The product's suggestion to generate a fresh ID and automatically retry is flawed. This risks duplicating the message if the original send was successful. Instead, the user must be prompted to manually confirm re-sending the message, acknowledging the potential for duplication.
|
||||
* **Missing Contract or Clarification:** The server's deduplication scope and retention for operation IDs are critical but unspecified. Without this, it's impossible to know if the original message was processed or if a fresh ID would truly be safe.
|
||||
43
research/evaluations/language-pilot-01/raw/r04/prompt.txt
Normal file
43
research/evaluations/language-pilot-01/raw/r04/prompt.txt
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
You are reviewing a proposed frontend architecture. Use the reference only where applicable. Return at most 220 words, using four short bullets: (1) outcome/state, (2) preserved information, (3) recovery action and rejected alternative, (4) missing contract or clarification. Explain the reasoning without naming or quoting reference headings, pattern names, or the presentation style. Do not infer undocumented guarantees. No tables, scores, source searching or tool calls. The task is a recommendation, not a claim you implemented or tested it.
|
||||
|
||||
REFERENCE
|
||||
Reference guidance. All mechanisms remain research hypotheses with limited evidence; apply only when their conditions hold. Distinguish assumed guarantees from observations. Do not infer missing backend, storage, access or retention contracts.
|
||||
|
||||
## Local Draft State
|
||||
|
||||
Keep the last observed record, unfinished edits and current server observation separate. A refresh or rejected write must not silently replace newer typing. Submitted content may move into a pending-operation record before the composer is cleared. Choose navigation/restart retention and privacy rules explicitly; local content alone is not authority.
|
||||
|
||||
## Version-Checked Mutation
|
||||
|
||||
When an update depends on an observed base, send that expected version. Require atomic server validation and commit; a stale rejection preserves intent for comparison with current data. Blindly substituting a new version on retry can erase concurrent intent. Version preconditions and retry identities solve different problems.
|
||||
|
||||
## Mutation Outcome Tracking
|
||||
|
||||
Separate pending, accepted, explicitly rejected and unknown outcomes using operation-specific evidence. A timeout, cancelled request, generic server error or rejected promise alone does not prove rejection. An authenticated matching server event may confirm acceptance before a request finishes. Later transport errors cannot reverse that evidence or clear newer work. Retry the same immutable payload and operation identity only within a documented deduplication scope and retention window. Missing identity or expired receipts require reconciliation, not a claim that a fresh ID is safe. Cancelling local tracking does not undo remote acceptance; reversal is a separate authorized action.
|
||||
|
||||
## Invalidate and Reproject
|
||||
|
||||
Acceptance and view freshness are separate. Refresh affected observations while preserving unfinished work. Local request generations can reject superseded reads, but cannot order opaque server versions or repair stale replicas. A read-after-write contract or documented server ordering is needed. Report accepted-with-unresolved-freshness when appropriate; a refresh failure does not undo a write.
|
||||
|
||||
## Local Commit Boundary
|
||||
|
||||
For restart-safe offline editing, acknowledge local save only after a recoverable commit of the revision and its pending upload intent. Immediate feedback is not a storage receipt. Ordinary process restart, power loss and device loss require different evidence. An in-memory queue alone cannot satisfy a restart promise. Local success does not imply remote acceptance.
|
||||
|
||||
## Pending Synchronization
|
||||
|
||||
Retain revision and operation identity with pending work across restart. A sent request is not an acknowledgement; receiver acceptance is not delivery to every device or backup. Define receiver deduplication, ancestry and authorization behavior. Lost acknowledgements retain the original identity. Block obsolete uploads on detected divergence; atomically supersede them when recording a replacement. Local retirement cannot stop an in-flight stale write: the receiver must enforce preconditions. A late receipt updates its own operation history, not a newer operation.
|
||||
|
||||
## Conflict Copy
|
||||
|
||||
When incompatible edits are detected, retain alternate content with identity and provenance before destructive replacement. Selecting an active copy is not semantic resolution. Protect both versions equally and expose unresolved status. Storage costs, unnoticed copies and missed conflict detection remain risks. Sync does not substitute for separately recoverable backup.
|
||||
|
||||
## Explicit Reconciliation
|
||||
|
||||
Let a person compare, choose or combine divergent versions and commit the decision before removing recovery material. Accessible notification, keyboard comparison and stable focus are requirements, not automatic consequences of storing copies. Two devices can reconcile concurrently; the resulting revisions still need divergence handling. A version store may preserve the same evidence without separate copy files; justified domain-aware merging is an alternative.
|
||||
|
||||
Connections: Local Draft State supports recovery from Version-Checked Mutation. Mutation Outcome Tracking supplies acceptance evidence to Invalidate and Reproject. Local Commit Boundary enables Pending Synchronization. Detected divergence can lead from Pending Synchronization to Conflict Copy, then Explicit Reconciliation, which creates another local commit. These connections are conditional, not universal ordering.
|
||||
|
||||
Evidence limits shared by both presentations: Joplin/Obsidian document conflict copies but do not establish our atomic crash contract. Kubernetes documents conditional updates; GitLab has a different, sometimes nonspecific failure surface. Stripe documents bounded idempotency. React Router illustrates revalidation but browser cancellation does not stop server writes. Pinned Zulip/Element source shows partial draft and pending-message mechanisms. A published Matrix SDK test with a synthetic receiver shows a rejected promise can coexist with a reconciled event, and fresh identity after local cancellation can duplicate simulated acceptance. This is not full application/server validation or independent full-pattern recurrence.
|
||||
|
||||
SCENARIO
|
||||
A mobile message composer queued an immutable payload and sent it. The response was lost. After logout, reinstall and login as a new device, only the payload remains; the original operation ID is lost. The server deduplication scope is device and endpoint. Lookup support and retention are unspecified. The product suggests generating a fresh ID and automatically retrying. Evaluate that suggestion and give a recovery plan.
|
||||
58
research/evaluations/language-pilot-01/raw/r04/receipt.json
Normal file
58
research/evaluations/language-pilot-01/raw/r04/receipt.json
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"scenario": "S02",
|
||||
"condition": "patterns",
|
||||
"prompt_sha256": "d6ee54d67b84d20dbb57884560aed951398427b19039c07323a9a8633bd06a2a",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e",
|
||||
"status": "completed",
|
||||
"output_words": 133,
|
||||
"elapsed_seconds": 2.002,
|
||||
"responses": [
|
||||
{
|
||||
"stage": "decision-review",
|
||||
"model": "google/gemini-2.5-flash",
|
||||
"usage": {
|
||||
"prompt_tokens": 1056,
|
||||
"completion_tokens": 178,
|
||||
"total_tokens": 1234
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
"provider": "openrouter",
|
||||
"response_id": "gen-1790445708-WURLWrZG9GkHw08EzqnN",
|
||||
"latency_seconds": 2.017,
|
||||
"profile": "custodian-triage-balanced",
|
||||
"profile_model": "google/gemini-2.5-flash",
|
||||
"profile_provider": "openrouter",
|
||||
"diagnostics": {
|
||||
"available": true,
|
||||
"effective_settings": {
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"reasoning_effort_present": false,
|
||||
"reasoning_present": false
|
||||
},
|
||||
"provider_usage": {
|
||||
"prompt_tokens": 1056,
|
||||
"completion_tokens": 178,
|
||||
"total_tokens": 1234,
|
||||
"cost": 0.0007618,
|
||||
"completion_tokens_details": {
|
||||
"reasoning_tokens": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"content_characters": 915,
|
||||
"content_sha256": "0cab89bf9c994d5a87913eb956bdd400611a5aa05a25c46e9473fda524b4403b"
|
||||
}
|
||||
]
|
||||
}
|
||||
18
research/evaluations/language-pilot-01/raw/r04/started.json
Normal file
18
research/evaluations/language-pilot-01/raw/r04/started.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"status": "attempting",
|
||||
"scenario": "S02",
|
||||
"condition": "patterns",
|
||||
"prompt_sha256": "d6ee54d67b84d20dbb57884560aed951398427b19039c07323a9a8633bd06a2a",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e"
|
||||
}
|
||||
6
research/evaluations/language-pilot-01/raw/r05/answer.md
Normal file
6
research/evaluations/language-pilot-01/raw/r05/answer.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
This architecture needs refinement to handle authorization denial gracefully.
|
||||
|
||||
* **Outcome/State:** The local commit remains, but the pending upload is explicitly rejected due to authorization. The editor is in a state of "accepted locally, rejected remotely."
|
||||
* **Preserved Information:** The full local commit of confidential notes, including its revision and operation identity, must be retained. The server's explicit denial and the reason (authorization failure) should also be recorded.
|
||||
* **Recovery Action and Rejected Alternative:** The system should prompt the user to explicitly reconcile the rejected upload, perhaps by allowing them to save the notes locally or export them. Blindly discarding the local commit or attempting a re-upload with a new identity is inappropriate as it would lose data and violate the "never lose edits" requirement.
|
||||
* **Missing Contract or Clarification:** A clear contract is needed regarding the retention, viewing, and export of local data for users whose write authorization has been revoked. This will dictate the specific recovery options presented to the user.
|
||||
43
research/evaluations/language-pilot-01/raw/r05/prompt.txt
Normal file
43
research/evaluations/language-pilot-01/raw/r05/prompt.txt
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
You are reviewing a proposed frontend architecture. Use the reference only where applicable. Return at most 220 words, using four short bullets: (1) outcome/state, (2) preserved information, (3) recovery action and rejected alternative, (4) missing contract or clarification. Explain the reasoning without naming or quoting reference headings, pattern names, or the presentation style. Do not infer undocumented guarantees. No tables, scores, source searching or tool calls. The task is a recommendation, not a claim you implemented or tested it.
|
||||
|
||||
REFERENCE
|
||||
Reference guidance. All mechanisms remain research hypotheses with limited evidence; apply only when their conditions hold. Distinguish assumed guarantees from observations. Do not infer missing backend, storage, access or retention contracts.
|
||||
|
||||
## Local Draft State
|
||||
|
||||
Keep the last observed record, unfinished edits and current server observation separate. A refresh or rejected write must not silently replace newer typing. Submitted content may move into a pending-operation record before the composer is cleared. Choose navigation/restart retention and privacy rules explicitly; local content alone is not authority.
|
||||
|
||||
## Version-Checked Mutation
|
||||
|
||||
When an update depends on an observed base, send that expected version. Require atomic server validation and commit; a stale rejection preserves intent for comparison with current data. Blindly substituting a new version on retry can erase concurrent intent. Version preconditions and retry identities solve different problems.
|
||||
|
||||
## Mutation Outcome Tracking
|
||||
|
||||
Separate pending, accepted, explicitly rejected and unknown outcomes using operation-specific evidence. A timeout, cancelled request, generic server error or rejected promise alone does not prove rejection. An authenticated matching server event may confirm acceptance before a request finishes. Later transport errors cannot reverse that evidence or clear newer work. Retry the same immutable payload and operation identity only within a documented deduplication scope and retention window. Missing identity or expired receipts require reconciliation, not a claim that a fresh ID is safe. Cancelling local tracking does not undo remote acceptance; reversal is a separate authorized action.
|
||||
|
||||
## Invalidate and Reproject
|
||||
|
||||
Acceptance and view freshness are separate. Refresh affected observations while preserving unfinished work. Local request generations can reject superseded reads, but cannot order opaque server versions or repair stale replicas. A read-after-write contract or documented server ordering is needed. Report accepted-with-unresolved-freshness when appropriate; a refresh failure does not undo a write.
|
||||
|
||||
## Local Commit Boundary
|
||||
|
||||
For restart-safe offline editing, acknowledge local save only after a recoverable commit of the revision and its pending upload intent. Immediate feedback is not a storage receipt. Ordinary process restart, power loss and device loss require different evidence. An in-memory queue alone cannot satisfy a restart promise. Local success does not imply remote acceptance.
|
||||
|
||||
## Pending Synchronization
|
||||
|
||||
Retain revision and operation identity with pending work across restart. A sent request is not an acknowledgement; receiver acceptance is not delivery to every device or backup. Define receiver deduplication, ancestry and authorization behavior. Lost acknowledgements retain the original identity. Block obsolete uploads on detected divergence; atomically supersede them when recording a replacement. Local retirement cannot stop an in-flight stale write: the receiver must enforce preconditions. A late receipt updates its own operation history, not a newer operation.
|
||||
|
||||
## Conflict Copy
|
||||
|
||||
When incompatible edits are detected, retain alternate content with identity and provenance before destructive replacement. Selecting an active copy is not semantic resolution. Protect both versions equally and expose unresolved status. Storage costs, unnoticed copies and missed conflict detection remain risks. Sync does not substitute for separately recoverable backup.
|
||||
|
||||
## Explicit Reconciliation
|
||||
|
||||
Let a person compare, choose or combine divergent versions and commit the decision before removing recovery material. Accessible notification, keyboard comparison and stable focus are requirements, not automatic consequences of storing copies. Two devices can reconcile concurrently; the resulting revisions still need divergence handling. A version store may preserve the same evidence without separate copy files; justified domain-aware merging is an alternative.
|
||||
|
||||
Connections: Local Draft State supports recovery from Version-Checked Mutation. Mutation Outcome Tracking supplies acceptance evidence to Invalidate and Reproject. Local Commit Boundary enables Pending Synchronization. Detected divergence can lead from Pending Synchronization to Conflict Copy, then Explicit Reconciliation, which creates another local commit. These connections are conditional, not universal ordering.
|
||||
|
||||
Evidence limits shared by both presentations: Joplin/Obsidian document conflict copies but do not establish our atomic crash contract. Kubernetes documents conditional updates; GitLab has a different, sometimes nonspecific failure surface. Stripe documents bounded idempotency. React Router illustrates revalidation but browser cancellation does not stop server writes. Pinned Zulip/Element source shows partial draft and pending-message mechanisms. A published Matrix SDK test with a synthetic receiver shows a rejected promise can coexist with a reconciled event, and fresh identity after local cancellation can duplicate simulated acceptance. This is not full application/server validation or independent full-pattern recurrence.
|
||||
|
||||
SCENARIO
|
||||
An offline field editor has a completed local commit of confidential notes and a pending upload. On reconnect, the server explicitly denies write authorization. The product requirement only says "never lose edits"; it says nothing about whether revoked users may retain, view or export local copies. Design the next states and recovery behavior without inventing an access policy.
|
||||
58
research/evaluations/language-pilot-01/raw/r05/receipt.json
Normal file
58
research/evaluations/language-pilot-01/raw/r05/receipt.json
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"scenario": "S03",
|
||||
"condition": "patterns",
|
||||
"prompt_sha256": "68c000a8625f08d4a944e13b3b9141029445efadbece190afe831bd28f960e11",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e",
|
||||
"status": "completed",
|
||||
"output_words": 163,
|
||||
"elapsed_seconds": 1.992,
|
||||
"responses": [
|
||||
{
|
||||
"stage": "decision-review",
|
||||
"model": "google/gemini-2.5-flash",
|
||||
"usage": {
|
||||
"prompt_tokens": 1044,
|
||||
"completion_tokens": 209,
|
||||
"total_tokens": 1253
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
"provider": "openrouter",
|
||||
"response_id": "gen-1790445710-EuMPTKA7DE2yPmtYv8MI",
|
||||
"latency_seconds": 1.987,
|
||||
"profile": "custodian-triage-balanced",
|
||||
"profile_model": "google/gemini-2.5-flash",
|
||||
"profile_provider": "openrouter",
|
||||
"diagnostics": {
|
||||
"available": true,
|
||||
"effective_settings": {
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"reasoning_effort_present": false,
|
||||
"reasoning_present": false
|
||||
},
|
||||
"provider_usage": {
|
||||
"prompt_tokens": 1044,
|
||||
"completion_tokens": 209,
|
||||
"total_tokens": 1253,
|
||||
"cost": 0.0008357,
|
||||
"completion_tokens_details": {
|
||||
"reasoning_tokens": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"content_characters": 1115,
|
||||
"content_sha256": "d1e51b0040acf54520444273b5445d45bee241d3941a0e82c7f7ac0b9b5f7804"
|
||||
}
|
||||
]
|
||||
}
|
||||
18
research/evaluations/language-pilot-01/raw/r05/started.json
Normal file
18
research/evaluations/language-pilot-01/raw/r05/started.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"status": "attempting",
|
||||
"scenario": "S03",
|
||||
"condition": "patterns",
|
||||
"prompt_sha256": "68c000a8625f08d4a944e13b3b9141029445efadbece190afe831bd28f960e11",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e"
|
||||
}
|
||||
9
research/evaluations/language-pilot-01/raw/r06/answer.md
Normal file
9
research/evaluations/language-pilot-01/raw/r06/answer.md
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
Here's a recommendation for the frontend architecture:
|
||||
|
||||
* **Outcome/State:** The editor should display the confidential notes as locally committed but with a clear, persistent indication that the pending upload was explicitly rejected due to authorization failure. The notes remain editable locally.
|
||||
|
||||
* **Preserved Information:** The full content of the confidential notes, their local commit identity, and the identity of the rejected upload attempt must be retained. The explicit server rejection message should also be preserved and associated with the failed upload.
|
||||
|
||||
* **Recovery Action and Rejected Alternative:** The user should be presented with options to either discard the local changes (if they choose to, understanding the implications) or attempt to export/save the content locally. Blindly retrying the upload with a new identity is rejected as it doesn't address the authorization issue and could lead to data duplication if the server's rejection was transient.
|
||||
|
||||
* **Missing Contract or Clarification:** A clear contract is needed regarding the retention and access policies for locally committed data when server authorization is revoked. Specifically, can the user continue to view, edit, or export these notes, or should they be purged?
|
||||
27
research/evaluations/language-pilot-01/raw/r06/prompt.txt
Normal file
27
research/evaluations/language-pilot-01/raw/r06/prompt.txt
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
You are reviewing a proposed frontend architecture. Use the reference only where applicable. Return at most 220 words, using four short bullets: (1) outcome/state, (2) preserved information, (3) recovery action and rejected alternative, (4) missing contract or clarification. Explain the reasoning without naming or quoting reference headings, pattern names, or the presentation style. Do not infer undocumented guarantees. No tables, scores, source searching or tool calls. The task is a recommendation, not a claim you implemented or tested it.
|
||||
|
||||
REFERENCE
|
||||
Reference guidance. All mechanisms remain research hypotheses with limited evidence; apply only when their conditions hold. Distinguish assumed guarantees from observations. Do not infer missing backend, storage, access or retention contracts.
|
||||
|
||||
Keep the last observed record, unfinished edits and current server observation separate. A refresh or rejected write must not silently replace newer typing. Submitted content may move into a pending-operation record before the composer is cleared. Choose navigation/restart retention and privacy rules explicitly; local content alone is not authority.
|
||||
|
||||
When an update depends on an observed base, send that expected version. Require atomic server validation and commit; a stale rejection preserves intent for comparison with current data. Blindly substituting a new version on retry can erase concurrent intent. Version preconditions and retry identities solve different problems.
|
||||
|
||||
Separate pending, accepted, explicitly rejected and unknown outcomes using operation-specific evidence. A timeout, cancelled request, generic server error or rejected promise alone does not prove rejection. An authenticated matching server event may confirm acceptance before a request finishes. Later transport errors cannot reverse that evidence or clear newer work. Retry the same immutable payload and operation identity only within a documented deduplication scope and retention window. Missing identity or expired receipts require reconciliation, not a claim that a fresh ID is safe. Cancelling local tracking does not undo remote acceptance; reversal is a separate authorized action.
|
||||
|
||||
Acceptance and view freshness are separate. Refresh affected observations while preserving unfinished work. Local request generations can reject superseded reads, but cannot order opaque server versions or repair stale replicas. A read-after-write contract or documented server ordering is needed. Report accepted-with-unresolved-freshness when appropriate; a refresh failure does not undo a write.
|
||||
|
||||
For restart-safe offline editing, acknowledge local save only after a recoverable commit of the revision and its pending upload intent. Immediate feedback is not a storage receipt. Ordinary process restart, power loss and device loss require different evidence. An in-memory queue alone cannot satisfy a restart promise. Local success does not imply remote acceptance.
|
||||
|
||||
Retain revision and operation identity with pending work across restart. A sent request is not an acknowledgement; receiver acceptance is not delivery to every device or backup. Define receiver deduplication, ancestry and authorization behavior. Lost acknowledgements retain the original identity. Block obsolete uploads on detected divergence; atomically supersede them when recording a replacement. Local retirement cannot stop an in-flight stale write: the receiver must enforce preconditions. A late receipt updates its own operation history, not a newer operation.
|
||||
|
||||
When incompatible edits are detected, retain alternate content with identity and provenance before destructive replacement. Selecting an active copy is not semantic resolution. Protect both versions equally and expose unresolved status. Storage costs, unnoticed copies and missed conflict detection remain risks. Sync does not substitute for separately recoverable backup.
|
||||
|
||||
Let a person compare, choose or combine divergent versions and commit the decision before removing recovery material. Accessible notification, keyboard comparison and stable focus are requirements, not automatic consequences of storing copies. Two devices can reconcile concurrently; the resulting revisions still need divergence handling. A version store may preserve the same evidence without separate copy files; justified domain-aware merging is an alternative.
|
||||
|
||||
Connections in the workflow: preserved unfinished edits support recovery from rejected conditional writes. Operation-specific acceptance evidence informs refreshing observations. A local recoverable commit enables restartable propagation. Detected divergence during propagation can require preservation of alternate content, then an intentional resolution, which creates another local commit. These connections are conditional, not universal ordering.
|
||||
|
||||
Evidence limits shared by both presentations: Joplin/Obsidian document conflict copies but do not establish our atomic crash contract. Kubernetes documents conditional updates; GitLab has a different, sometimes nonspecific failure surface. Stripe documents bounded idempotency. React Router illustrates revalidation but browser cancellation does not stop server writes. Pinned Zulip/Element source shows partial draft and pending-message mechanisms. A published Matrix SDK test with a synthetic receiver shows a rejected promise can coexist with a reconciled event, and fresh identity after local cancellation can duplicate simulated acceptance. This is not full application/server validation or independent full-pattern recurrence.
|
||||
|
||||
SCENARIO
|
||||
An offline field editor has a completed local commit of confidential notes and a pending upload. On reconnect, the server explicitly denies write authorization. The product requirement only says "never lose edits"; it says nothing about whether revoked users may retain, view or export local copies. Design the next states and recovery behavior without inventing an access policy.
|
||||
58
research/evaluations/language-pilot-01/raw/r06/receipt.json
Normal file
58
research/evaluations/language-pilot-01/raw/r06/receipt.json
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"scenario": "S03",
|
||||
"condition": "prose",
|
||||
"prompt_sha256": "870c04ec6189e7ca5cbb825da7eb6cbacd41f825972f5be9adc063eebe8c0a6b",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e",
|
||||
"status": "completed",
|
||||
"output_words": 185,
|
||||
"elapsed_seconds": 2.595,
|
||||
"responses": [
|
||||
{
|
||||
"stage": "decision-review",
|
||||
"model": "google/gemini-2.5-flash",
|
||||
"usage": {
|
||||
"prompt_tokens": 1003,
|
||||
"completion_tokens": 234,
|
||||
"total_tokens": 1237
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
"provider": "openrouter",
|
||||
"response_id": "gen-1790445712-Pq447WaJuHb0Wb9IsdEv",
|
||||
"latency_seconds": 2.68,
|
||||
"profile": "custodian-triage-balanced",
|
||||
"profile_model": "google/gemini-2.5-flash",
|
||||
"profile_provider": "openrouter",
|
||||
"diagnostics": {
|
||||
"available": true,
|
||||
"effective_settings": {
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"reasoning_effort_present": false,
|
||||
"reasoning_present": false
|
||||
},
|
||||
"provider_usage": {
|
||||
"prompt_tokens": 1003,
|
||||
"completion_tokens": 234,
|
||||
"total_tokens": 1237,
|
||||
"cost": 0.0008859,
|
||||
"completion_tokens_details": {
|
||||
"reasoning_tokens": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"content_characters": 1264,
|
||||
"content_sha256": "0c773739af4a505c87fbea5659da9ba4e0d9207643597992b9050d22f04372ac"
|
||||
}
|
||||
]
|
||||
}
|
||||
18
research/evaluations/language-pilot-01/raw/r06/started.json
Normal file
18
research/evaluations/language-pilot-01/raw/r06/started.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"status": "attempting",
|
||||
"scenario": "S03",
|
||||
"condition": "prose",
|
||||
"prompt_sha256": "870c04ec6189e7ca5cbb825da7eb6cbacd41f825972f5be9adc063eebe8c0a6b",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e"
|
||||
}
|
||||
6
research/evaluations/language-pilot-01/raw/r07/answer.md
Normal file
6
research/evaluations/language-pilot-01/raw/r07/answer.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
Here's a recommendation for the scenario:
|
||||
|
||||
* **Outcome/State:** The client should not remove the pending entry for operation K upon local "Cancel." The server's status for K is unknown.
|
||||
* **Preserved Information:** The client must retain the original operation K's identity and payload, along with its "pending" status, even after local cancellation.
|
||||
* **Recovery Action and Rejected Alternative:** The user's request to submit the "same booking again" should be treated as a new, distinct operation with a fresh identity. Blindly retrying K with its original identity is rejected because the server lacks deduplication guarantees.
|
||||
* **Missing Contract or Clarification:** A critical missing contract is the server's behavior regarding duplicate submissions and its retention window for operation identities. Without this, the client cannot safely assume a new submission will not result in a duplicate booking if K was, in fact, accepted.
|
||||
27
research/evaluations/language-pilot-01/raw/r07/prompt.txt
Normal file
27
research/evaluations/language-pilot-01/raw/r07/prompt.txt
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
You are reviewing a proposed frontend architecture. Use the reference only where applicable. Return at most 220 words, using four short bullets: (1) outcome/state, (2) preserved information, (3) recovery action and rejected alternative, (4) missing contract or clarification. Explain the reasoning without naming or quoting reference headings, pattern names, or the presentation style. Do not infer undocumented guarantees. No tables, scores, source searching or tool calls. The task is a recommendation, not a claim you implemented or tested it.
|
||||
|
||||
REFERENCE
|
||||
Reference guidance. All mechanisms remain research hypotheses with limited evidence; apply only when their conditions hold. Distinguish assumed guarantees from observations. Do not infer missing backend, storage, access or retention contracts.
|
||||
|
||||
Keep the last observed record, unfinished edits and current server observation separate. A refresh or rejected write must not silently replace newer typing. Submitted content may move into a pending-operation record before the composer is cleared. Choose navigation/restart retention and privacy rules explicitly; local content alone is not authority.
|
||||
|
||||
When an update depends on an observed base, send that expected version. Require atomic server validation and commit; a stale rejection preserves intent for comparison with current data. Blindly substituting a new version on retry can erase concurrent intent. Version preconditions and retry identities solve different problems.
|
||||
|
||||
Separate pending, accepted, explicitly rejected and unknown outcomes using operation-specific evidence. A timeout, cancelled request, generic server error or rejected promise alone does not prove rejection. An authenticated matching server event may confirm acceptance before a request finishes. Later transport errors cannot reverse that evidence or clear newer work. Retry the same immutable payload and operation identity only within a documented deduplication scope and retention window. Missing identity or expired receipts require reconciliation, not a claim that a fresh ID is safe. Cancelling local tracking does not undo remote acceptance; reversal is a separate authorized action.
|
||||
|
||||
Acceptance and view freshness are separate. Refresh affected observations while preserving unfinished work. Local request generations can reject superseded reads, but cannot order opaque server versions or repair stale replicas. A read-after-write contract or documented server ordering is needed. Report accepted-with-unresolved-freshness when appropriate; a refresh failure does not undo a write.
|
||||
|
||||
For restart-safe offline editing, acknowledge local save only after a recoverable commit of the revision and its pending upload intent. Immediate feedback is not a storage receipt. Ordinary process restart, power loss and device loss require different evidence. An in-memory queue alone cannot satisfy a restart promise. Local success does not imply remote acceptance.
|
||||
|
||||
Retain revision and operation identity with pending work across restart. A sent request is not an acknowledgement; receiver acceptance is not delivery to every device or backup. Define receiver deduplication, ancestry and authorization behavior. Lost acknowledgements retain the original identity. Block obsolete uploads on detected divergence; atomically supersede them when recording a replacement. Local retirement cannot stop an in-flight stale write: the receiver must enforce preconditions. A late receipt updates its own operation history, not a newer operation.
|
||||
|
||||
When incompatible edits are detected, retain alternate content with identity and provenance before destructive replacement. Selecting an active copy is not semantic resolution. Protect both versions equally and expose unresolved status. Storage costs, unnoticed copies and missed conflict detection remain risks. Sync does not substitute for separately recoverable backup.
|
||||
|
||||
Let a person compare, choose or combine divergent versions and commit the decision before removing recovery material. Accessible notification, keyboard comparison and stable focus are requirements, not automatic consequences of storing copies. Two devices can reconcile concurrently; the resulting revisions still need divergence handling. A version store may preserve the same evidence without separate copy files; justified domain-aware merging is an alternative.
|
||||
|
||||
Connections in the workflow: preserved unfinished edits support recovery from rejected conditional writes. Operation-specific acceptance evidence informs refreshing observations. A local recoverable commit enables restartable propagation. Detected divergence during propagation can require preservation of alternate content, then an intentional resolution, which creates another local commit. These connections are conditional, not universal ordering.
|
||||
|
||||
Evidence limits shared by both presentations: Joplin/Obsidian document conflict copies but do not establish our atomic crash contract. Kubernetes documents conditional updates; GitLab has a different, sometimes nonspecific failure surface. Stripe documents bounded idempotency. React Router illustrates revalidation but browser cancellation does not stop server writes. Pinned Zulip/Element source shows partial draft and pending-message mechanisms. A published Matrix SDK test with a synthetic receiver shows a rejected promise can coexist with a reconciled event, and fresh identity after local cancellation can duplicate simulated acceptance. This is not full application/server validation or independent full-pattern recurrence.
|
||||
|
||||
SCENARIO
|
||||
A server-confirmed booking UI times out after submitting operation K. Before receiving authoritative evidence, the user presses Cancel; the client removes its pending entry. The user then asks to submit the same booking again. The server exposes no documented operation-status lookup or deduplication retention guarantee. Specify what Cancel can honestly mean and whether a new submission is safe.
|
||||
58
research/evaluations/language-pilot-01/raw/r07/receipt.json
Normal file
58
research/evaluations/language-pilot-01/raw/r07/receipt.json
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"scenario": "S04",
|
||||
"condition": "prose",
|
||||
"prompt_sha256": "ba387924d68aa1c41d779a5f948d96eae129cb7344b50cc2891d93cdfbc35ff6",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e",
|
||||
"status": "completed",
|
||||
"output_words": 140,
|
||||
"elapsed_seconds": 1.616,
|
||||
"responses": [
|
||||
{
|
||||
"stage": "decision-review",
|
||||
"model": "google/gemini-2.5-flash",
|
||||
"usage": {
|
||||
"prompt_tokens": 1007,
|
||||
"completion_tokens": 192,
|
||||
"total_tokens": 1199
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
"provider": "openrouter",
|
||||
"response_id": "gen-1790445715-N9k9LMA7rueb5RTOIPEp",
|
||||
"latency_seconds": 1.561,
|
||||
"profile": "custodian-triage-balanced",
|
||||
"profile_model": "google/gemini-2.5-flash",
|
||||
"profile_provider": "openrouter",
|
||||
"diagnostics": {
|
||||
"available": true,
|
||||
"effective_settings": {
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"reasoning_effort_present": false,
|
||||
"reasoning_present": false
|
||||
},
|
||||
"provider_usage": {
|
||||
"prompt_tokens": 1007,
|
||||
"completion_tokens": 192,
|
||||
"total_tokens": 1199,
|
||||
"cost": 0.0007821,
|
||||
"completion_tokens_details": {
|
||||
"reasoning_tokens": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"content_characters": 947,
|
||||
"content_sha256": "f388d3514a2c73c5a237c1559c7b47a945c2bac43bf42ba3fe604479206d3f2f"
|
||||
}
|
||||
]
|
||||
}
|
||||
18
research/evaluations/language-pilot-01/raw/r07/started.json
Normal file
18
research/evaluations/language-pilot-01/raw/r07/started.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"status": "attempting",
|
||||
"scenario": "S04",
|
||||
"condition": "prose",
|
||||
"prompt_sha256": "ba387924d68aa1c41d779a5f948d96eae129cb7344b50cc2891d93cdfbc35ff6",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e"
|
||||
}
|
||||
6
research/evaluations/language-pilot-01/raw/r08/answer.md
Normal file
6
research/evaluations/language-pilot-01/raw/r08/answer.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
The proposed architecture has a critical flaw for this scenario.
|
||||
|
||||
* **Outcome/State:** The booking's final state is unknown. The server may have accepted operation K despite the timeout and local cancellation.
|
||||
* **Preserved Information:** The client has discarded the original operation K's identity and intent.
|
||||
* **Recovery Action:** The only safe action is to inform the user that the previous booking's status is unknown and they should check with the service provider. A new submission is unsafe.
|
||||
* **Missing Contract:** The server lacks a documented operation-status lookup or deduplication retention guarantee, which is essential for safely retrying or confirming operations after client-side uncertainty.
|
||||
43
research/evaluations/language-pilot-01/raw/r08/prompt.txt
Normal file
43
research/evaluations/language-pilot-01/raw/r08/prompt.txt
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
You are reviewing a proposed frontend architecture. Use the reference only where applicable. Return at most 220 words, using four short bullets: (1) outcome/state, (2) preserved information, (3) recovery action and rejected alternative, (4) missing contract or clarification. Explain the reasoning without naming or quoting reference headings, pattern names, or the presentation style. Do not infer undocumented guarantees. No tables, scores, source searching or tool calls. The task is a recommendation, not a claim you implemented or tested it.
|
||||
|
||||
REFERENCE
|
||||
Reference guidance. All mechanisms remain research hypotheses with limited evidence; apply only when their conditions hold. Distinguish assumed guarantees from observations. Do not infer missing backend, storage, access or retention contracts.
|
||||
|
||||
## Local Draft State
|
||||
|
||||
Keep the last observed record, unfinished edits and current server observation separate. A refresh or rejected write must not silently replace newer typing. Submitted content may move into a pending-operation record before the composer is cleared. Choose navigation/restart retention and privacy rules explicitly; local content alone is not authority.
|
||||
|
||||
## Version-Checked Mutation
|
||||
|
||||
When an update depends on an observed base, send that expected version. Require atomic server validation and commit; a stale rejection preserves intent for comparison with current data. Blindly substituting a new version on retry can erase concurrent intent. Version preconditions and retry identities solve different problems.
|
||||
|
||||
## Mutation Outcome Tracking
|
||||
|
||||
Separate pending, accepted, explicitly rejected and unknown outcomes using operation-specific evidence. A timeout, cancelled request, generic server error or rejected promise alone does not prove rejection. An authenticated matching server event may confirm acceptance before a request finishes. Later transport errors cannot reverse that evidence or clear newer work. Retry the same immutable payload and operation identity only within a documented deduplication scope and retention window. Missing identity or expired receipts require reconciliation, not a claim that a fresh ID is safe. Cancelling local tracking does not undo remote acceptance; reversal is a separate authorized action.
|
||||
|
||||
## Invalidate and Reproject
|
||||
|
||||
Acceptance and view freshness are separate. Refresh affected observations while preserving unfinished work. Local request generations can reject superseded reads, but cannot order opaque server versions or repair stale replicas. A read-after-write contract or documented server ordering is needed. Report accepted-with-unresolved-freshness when appropriate; a refresh failure does not undo a write.
|
||||
|
||||
## Local Commit Boundary
|
||||
|
||||
For restart-safe offline editing, acknowledge local save only after a recoverable commit of the revision and its pending upload intent. Immediate feedback is not a storage receipt. Ordinary process restart, power loss and device loss require different evidence. An in-memory queue alone cannot satisfy a restart promise. Local success does not imply remote acceptance.
|
||||
|
||||
## Pending Synchronization
|
||||
|
||||
Retain revision and operation identity with pending work across restart. A sent request is not an acknowledgement; receiver acceptance is not delivery to every device or backup. Define receiver deduplication, ancestry and authorization behavior. Lost acknowledgements retain the original identity. Block obsolete uploads on detected divergence; atomically supersede them when recording a replacement. Local retirement cannot stop an in-flight stale write: the receiver must enforce preconditions. A late receipt updates its own operation history, not a newer operation.
|
||||
|
||||
## Conflict Copy
|
||||
|
||||
When incompatible edits are detected, retain alternate content with identity and provenance before destructive replacement. Selecting an active copy is not semantic resolution. Protect both versions equally and expose unresolved status. Storage costs, unnoticed copies and missed conflict detection remain risks. Sync does not substitute for separately recoverable backup.
|
||||
|
||||
## Explicit Reconciliation
|
||||
|
||||
Let a person compare, choose or combine divergent versions and commit the decision before removing recovery material. Accessible notification, keyboard comparison and stable focus are requirements, not automatic consequences of storing copies. Two devices can reconcile concurrently; the resulting revisions still need divergence handling. A version store may preserve the same evidence without separate copy files; justified domain-aware merging is an alternative.
|
||||
|
||||
Connections: Local Draft State supports recovery from Version-Checked Mutation. Mutation Outcome Tracking supplies acceptance evidence to Invalidate and Reproject. Local Commit Boundary enables Pending Synchronization. Detected divergence can lead from Pending Synchronization to Conflict Copy, then Explicit Reconciliation, which creates another local commit. These connections are conditional, not universal ordering.
|
||||
|
||||
Evidence limits shared by both presentations: Joplin/Obsidian document conflict copies but do not establish our atomic crash contract. Kubernetes documents conditional updates; GitLab has a different, sometimes nonspecific failure surface. Stripe documents bounded idempotency. React Router illustrates revalidation but browser cancellation does not stop server writes. Pinned Zulip/Element source shows partial draft and pending-message mechanisms. A published Matrix SDK test with a synthetic receiver shows a rejected promise can coexist with a reconciled event, and fresh identity after local cancellation can duplicate simulated acceptance. This is not full application/server validation or independent full-pattern recurrence.
|
||||
|
||||
SCENARIO
|
||||
A server-confirmed booking UI times out after submitting operation K. Before receiving authoritative evidence, the user presses Cancel; the client removes its pending entry. The user then asks to submit the same booking again. The server exposes no documented operation-status lookup or deduplication retention guarantee. Specify what Cancel can honestly mean and whether a new submission is safe.
|
||||
58
research/evaluations/language-pilot-01/raw/r08/receipt.json
Normal file
58
research/evaluations/language-pilot-01/raw/r08/receipt.json
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"scenario": "S04",
|
||||
"condition": "patterns",
|
||||
"prompt_sha256": "64c54d8397f1b117bc8aa939f1d767b7912e21ce16951f5d7729d3fc33b44e39",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e",
|
||||
"status": "completed",
|
||||
"output_words": 103,
|
||||
"elapsed_seconds": 1.679,
|
||||
"responses": [
|
||||
{
|
||||
"stage": "decision-review",
|
||||
"model": "google/gemini-2.5-flash",
|
||||
"usage": {
|
||||
"prompt_tokens": 1048,
|
||||
"completion_tokens": 142,
|
||||
"total_tokens": 1190
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
"provider": "openrouter",
|
||||
"response_id": "gen-1790445717-tSznapdMvP2gf37tW5r0",
|
||||
"latency_seconds": 1.618,
|
||||
"profile": "custodian-triage-balanced",
|
||||
"profile_model": "google/gemini-2.5-flash",
|
||||
"profile_provider": "openrouter",
|
||||
"diagnostics": {
|
||||
"available": true,
|
||||
"effective_settings": {
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"reasoning_effort_present": false,
|
||||
"reasoning_present": false
|
||||
},
|
||||
"provider_usage": {
|
||||
"prompt_tokens": 1048,
|
||||
"completion_tokens": 142,
|
||||
"total_tokens": 1190,
|
||||
"cost": 0.0006694,
|
||||
"completion_tokens_details": {
|
||||
"reasoning_tokens": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"content_characters": 719,
|
||||
"content_sha256": "acdf9bc9bb7faadfeb32be0ebdc9926984135f262fcf12cec7ebd07c9742cd1b"
|
||||
}
|
||||
]
|
||||
}
|
||||
18
research/evaluations/language-pilot-01/raw/r08/started.json
Normal file
18
research/evaluations/language-pilot-01/raw/r08/started.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"status": "attempting",
|
||||
"scenario": "S04",
|
||||
"condition": "patterns",
|
||||
"prompt_sha256": "64c54d8397f1b117bc8aa939f1d767b7912e21ce16951f5d7729d3fc33b44e39",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e"
|
||||
}
|
||||
6
research/evaluations/language-pilot-01/raw/r09/answer.md
Normal file
6
research/evaluations/language-pilot-01/raw/r09/answer.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
This architecture has several issues that could lead to data loss or corruption.
|
||||
|
||||
* **Outcome/State:** The system will transition from having pending operations P and Q, and revisions a, b, and c, to a state where P's receipt arrives. The system needs to define how the late receipt for P interacts with the already-resolved revision c and the new pending operation Q.
|
||||
* **Preserved Information:** The system must preserve revisions a, b, and c, along with the identities of pending operations P and Q, and the fact that P was in flight. The receipt for P, when it arrives, must also be preserved with its associated operation identity.
|
||||
* **Recovery Action and Rejected Alternative:** Upon receiving the late receipt for P, the system should update P's status to "accepted" but **not** reapply P's changes to revision c or Q. Blindly applying P's changes would overwrite the explicit reconciliation that produced c. Instead, the system should acknowledge P's acceptance and potentially flag that P's original intent was superseded by the reconciliation.
|
||||
* **Missing Contract or Clarification:** The critical missing contract is the receiver's ancestry and precondition behavior. Without knowing how the receiver handles concurrent updates and what preconditions it enforces, it's impossible to guarantee that P's original intent wasn't already superseded on the server, or how the server would react to a re-submission of P after c was uploaded.
|
||||
43
research/evaluations/language-pilot-01/raw/r09/prompt.txt
Normal file
43
research/evaluations/language-pilot-01/raw/r09/prompt.txt
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
You are reviewing a proposed frontend architecture. Use the reference only where applicable. Return at most 220 words, using four short bullets: (1) outcome/state, (2) preserved information, (3) recovery action and rejected alternative, (4) missing contract or clarification. Explain the reasoning without naming or quoting reference headings, pattern names, or the presentation style. Do not infer undocumented guarantees. No tables, scores, source searching or tool calls. The task is a recommendation, not a claim you implemented or tested it.
|
||||
|
||||
REFERENCE
|
||||
Reference guidance. All mechanisms remain research hypotheses with limited evidence; apply only when their conditions hold. Distinguish assumed guarantees from observations. Do not infer missing backend, storage, access or retention contracts.
|
||||
|
||||
## Local Draft State
|
||||
|
||||
Keep the last observed record, unfinished edits and current server observation separate. A refresh or rejected write must not silently replace newer typing. Submitted content may move into a pending-operation record before the composer is cleared. Choose navigation/restart retention and privacy rules explicitly; local content alone is not authority.
|
||||
|
||||
## Version-Checked Mutation
|
||||
|
||||
When an update depends on an observed base, send that expected version. Require atomic server validation and commit; a stale rejection preserves intent for comparison with current data. Blindly substituting a new version on retry can erase concurrent intent. Version preconditions and retry identities solve different problems.
|
||||
|
||||
## Mutation Outcome Tracking
|
||||
|
||||
Separate pending, accepted, explicitly rejected and unknown outcomes using operation-specific evidence. A timeout, cancelled request, generic server error or rejected promise alone does not prove rejection. An authenticated matching server event may confirm acceptance before a request finishes. Later transport errors cannot reverse that evidence or clear newer work. Retry the same immutable payload and operation identity only within a documented deduplication scope and retention window. Missing identity or expired receipts require reconciliation, not a claim that a fresh ID is safe. Cancelling local tracking does not undo remote acceptance; reversal is a separate authorized action.
|
||||
|
||||
## Invalidate and Reproject
|
||||
|
||||
Acceptance and view freshness are separate. Refresh affected observations while preserving unfinished work. Local request generations can reject superseded reads, but cannot order opaque server versions or repair stale replicas. A read-after-write contract or documented server ordering is needed. Report accepted-with-unresolved-freshness when appropriate; a refresh failure does not undo a write.
|
||||
|
||||
## Local Commit Boundary
|
||||
|
||||
For restart-safe offline editing, acknowledge local save only after a recoverable commit of the revision and its pending upload intent. Immediate feedback is not a storage receipt. Ordinary process restart, power loss and device loss require different evidence. An in-memory queue alone cannot satisfy a restart promise. Local success does not imply remote acceptance.
|
||||
|
||||
## Pending Synchronization
|
||||
|
||||
Retain revision and operation identity with pending work across restart. A sent request is not an acknowledgement; receiver acceptance is not delivery to every device or backup. Define receiver deduplication, ancestry and authorization behavior. Lost acknowledgements retain the original identity. Block obsolete uploads on detected divergence; atomically supersede them when recording a replacement. Local retirement cannot stop an in-flight stale write: the receiver must enforce preconditions. A late receipt updates its own operation history, not a newer operation.
|
||||
|
||||
## Conflict Copy
|
||||
|
||||
When incompatible edits are detected, retain alternate content with identity and provenance before destructive replacement. Selecting an active copy is not semantic resolution. Protect both versions equally and expose unresolved status. Storage costs, unnoticed copies and missed conflict detection remain risks. Sync does not substitute for separately recoverable backup.
|
||||
|
||||
## Explicit Reconciliation
|
||||
|
||||
Let a person compare, choose or combine divergent versions and commit the decision before removing recovery material. Accessible notification, keyboard comparison and stable focus are requirements, not automatic consequences of storing copies. Two devices can reconcile concurrently; the resulting revisions still need divergence handling. A version store may preserve the same evidence without separate copy files; justified domain-aware merging is an alternative.
|
||||
|
||||
Connections: Local Draft State supports recovery from Version-Checked Mutation. Mutation Outcome Tracking supplies acceptance evidence to Invalidate and Reproject. Local Commit Boundary enables Pending Synchronization. Detected divergence can lead from Pending Synchronization to Conflict Copy, then Explicit Reconciliation, which creates another local commit. These connections are conditional, not universal ordering.
|
||||
|
||||
Evidence limits shared by both presentations: Joplin/Obsidian document conflict copies but do not establish our atomic crash contract. Kubernetes documents conditional updates; GitLab has a different, sometimes nonspecific failure surface. Stripe documents bounded idempotency. React Router illustrates revalidation but browser cancellation does not stop server writes. Pinned Zulip/Element source shows partial draft and pending-message mechanisms. A published Matrix SDK test with a synthetic receiver shows a rejected promise can coexist with a reconciled event, and fresh identity after local cancellation can duplicate simulated acceptance. This is not full application/server validation or independent full-pattern recurrence.
|
||||
|
||||
SCENARIO
|
||||
Device A durably recorded offline revision a with pending operation P. Another device uploaded divergent revision b. A preserves both and resolves them into revision c with new pending operation Q. P was already in flight. Later a receipt for P arrives. Local commits survive process restart, but receiver ancestry/precondition behavior is unspecified. Specify queue transitions, crash boundaries and what the late receipt may change.
|
||||
58
research/evaluations/language-pilot-01/raw/r09/receipt.json
Normal file
58
research/evaluations/language-pilot-01/raw/r09/receipt.json
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"scenario": "S05",
|
||||
"condition": "patterns",
|
||||
"prompt_sha256": "ca81a6294d2ea8014d778b9ce583e0245369c45412ef3e26cd110b284bd8b652",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e",
|
||||
"status": "completed",
|
||||
"output_words": 228,
|
||||
"elapsed_seconds": 2.748,
|
||||
"responses": [
|
||||
{
|
||||
"stage": "decision-review",
|
||||
"model": "google/gemini-2.5-flash",
|
||||
"usage": {
|
||||
"prompt_tokens": 1056,
|
||||
"completion_tokens": 302,
|
||||
"total_tokens": 1358
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
"provider": "openrouter",
|
||||
"response_id": "gen-1790445719-GDPHsgiTRDN2F3A47F4A",
|
||||
"latency_seconds": 2.776,
|
||||
"profile": "custodian-triage-balanced",
|
||||
"profile_model": "google/gemini-2.5-flash",
|
||||
"profile_provider": "openrouter",
|
||||
"diagnostics": {
|
||||
"available": true,
|
||||
"effective_settings": {
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"reasoning_effort_present": false,
|
||||
"reasoning_present": false
|
||||
},
|
||||
"provider_usage": {
|
||||
"prompt_tokens": 1056,
|
||||
"completion_tokens": 302,
|
||||
"total_tokens": 1358,
|
||||
"cost": 0.0010718,
|
||||
"completion_tokens_details": {
|
||||
"reasoning_tokens": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"content_characters": 1453,
|
||||
"content_sha256": "8685235bcfc59fc1f3b2e922e47cd6bb46eee18daf0c224d11a16775197fee7d"
|
||||
}
|
||||
]
|
||||
}
|
||||
18
research/evaluations/language-pilot-01/raw/r09/started.json
Normal file
18
research/evaluations/language-pilot-01/raw/r09/started.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"status": "attempting",
|
||||
"scenario": "S05",
|
||||
"condition": "patterns",
|
||||
"prompt_sha256": "ca81a6294d2ea8014d778b9ce583e0245369c45412ef3e26cd110b284bd8b652",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e"
|
||||
}
|
||||
9
research/evaluations/language-pilot-01/raw/r10/answer.md
Normal file
9
research/evaluations/language-pilot-01/raw/r10/answer.md
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
Here's a recommendation for the frontend architecture based on the scenario:
|
||||
|
||||
* **Outcome/State:** The system should transition to a state where revision `c` is the active local content, with pending operation `Q` awaiting remote acceptance. The late receipt for `P` should update its own operation history, marking `P` as remotely accepted, but it should not alter the active content or the status of `Q`.
|
||||
|
||||
* **Preserved Information:** Both divergent revisions `a` and `b` should be preserved with their identities and provenance until `c` is fully committed and resolved. The original pending operation `P` and its identity must also be retained, along with the new pending operation `Q`.
|
||||
|
||||
* **Recovery Action and Rejected Alternative:** Upon process restart, the system must recover `c` as the active content and `Q` as the pending operation. The late receipt for `P` should not trigger a re-application of `P` or a rollback to `a`, as this would erase the user's explicit resolution into `c`.
|
||||
|
||||
* **Missing Contract or Clarification:** The architecture needs a clear contract for the receiver's ancestry and precondition behavior. Specifically, it must define how the receiver handles an in-flight operation (`P`) when a superseding operation (`Q` derived from `c`) is subsequently sent, and how it would respond to `P` if `b` was already accepted.
|
||||
27
research/evaluations/language-pilot-01/raw/r10/prompt.txt
Normal file
27
research/evaluations/language-pilot-01/raw/r10/prompt.txt
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
You are reviewing a proposed frontend architecture. Use the reference only where applicable. Return at most 220 words, using four short bullets: (1) outcome/state, (2) preserved information, (3) recovery action and rejected alternative, (4) missing contract or clarification. Explain the reasoning without naming or quoting reference headings, pattern names, or the presentation style. Do not infer undocumented guarantees. No tables, scores, source searching or tool calls. The task is a recommendation, not a claim you implemented or tested it.
|
||||
|
||||
REFERENCE
|
||||
Reference guidance. All mechanisms remain research hypotheses with limited evidence; apply only when their conditions hold. Distinguish assumed guarantees from observations. Do not infer missing backend, storage, access or retention contracts.
|
||||
|
||||
Keep the last observed record, unfinished edits and current server observation separate. A refresh or rejected write must not silently replace newer typing. Submitted content may move into a pending-operation record before the composer is cleared. Choose navigation/restart retention and privacy rules explicitly; local content alone is not authority.
|
||||
|
||||
When an update depends on an observed base, send that expected version. Require atomic server validation and commit; a stale rejection preserves intent for comparison with current data. Blindly substituting a new version on retry can erase concurrent intent. Version preconditions and retry identities solve different problems.
|
||||
|
||||
Separate pending, accepted, explicitly rejected and unknown outcomes using operation-specific evidence. A timeout, cancelled request, generic server error or rejected promise alone does not prove rejection. An authenticated matching server event may confirm acceptance before a request finishes. Later transport errors cannot reverse that evidence or clear newer work. Retry the same immutable payload and operation identity only within a documented deduplication scope and retention window. Missing identity or expired receipts require reconciliation, not a claim that a fresh ID is safe. Cancelling local tracking does not undo remote acceptance; reversal is a separate authorized action.
|
||||
|
||||
Acceptance and view freshness are separate. Refresh affected observations while preserving unfinished work. Local request generations can reject superseded reads, but cannot order opaque server versions or repair stale replicas. A read-after-write contract or documented server ordering is needed. Report accepted-with-unresolved-freshness when appropriate; a refresh failure does not undo a write.
|
||||
|
||||
For restart-safe offline editing, acknowledge local save only after a recoverable commit of the revision and its pending upload intent. Immediate feedback is not a storage receipt. Ordinary process restart, power loss and device loss require different evidence. An in-memory queue alone cannot satisfy a restart promise. Local success does not imply remote acceptance.
|
||||
|
||||
Retain revision and operation identity with pending work across restart. A sent request is not an acknowledgement; receiver acceptance is not delivery to every device or backup. Define receiver deduplication, ancestry and authorization behavior. Lost acknowledgements retain the original identity. Block obsolete uploads on detected divergence; atomically supersede them when recording a replacement. Local retirement cannot stop an in-flight stale write: the receiver must enforce preconditions. A late receipt updates its own operation history, not a newer operation.
|
||||
|
||||
When incompatible edits are detected, retain alternate content with identity and provenance before destructive replacement. Selecting an active copy is not semantic resolution. Protect both versions equally and expose unresolved status. Storage costs, unnoticed copies and missed conflict detection remain risks. Sync does not substitute for separately recoverable backup.
|
||||
|
||||
Let a person compare, choose or combine divergent versions and commit the decision before removing recovery material. Accessible notification, keyboard comparison and stable focus are requirements, not automatic consequences of storing copies. Two devices can reconcile concurrently; the resulting revisions still need divergence handling. A version store may preserve the same evidence without separate copy files; justified domain-aware merging is an alternative.
|
||||
|
||||
Connections in the workflow: preserved unfinished edits support recovery from rejected conditional writes. Operation-specific acceptance evidence informs refreshing observations. A local recoverable commit enables restartable propagation. Detected divergence during propagation can require preservation of alternate content, then an intentional resolution, which creates another local commit. These connections are conditional, not universal ordering.
|
||||
|
||||
Evidence limits shared by both presentations: Joplin/Obsidian document conflict copies but do not establish our atomic crash contract. Kubernetes documents conditional updates; GitLab has a different, sometimes nonspecific failure surface. Stripe documents bounded idempotency. React Router illustrates revalidation but browser cancellation does not stop server writes. Pinned Zulip/Element source shows partial draft and pending-message mechanisms. A published Matrix SDK test with a synthetic receiver shows a rejected promise can coexist with a reconciled event, and fresh identity after local cancellation can duplicate simulated acceptance. This is not full application/server validation or independent full-pattern recurrence.
|
||||
|
||||
SCENARIO
|
||||
Device A durably recorded offline revision a with pending operation P. Another device uploaded divergent revision b. A preserves both and resolves them into revision c with new pending operation Q. P was already in flight. Later a receipt for P arrives. Local commits survive process restart, but receiver ancestry/precondition behavior is unspecified. Specify queue transitions, crash boundaries and what the late receipt may change.
|
||||
58
research/evaluations/language-pilot-01/raw/r10/receipt.json
Normal file
58
research/evaluations/language-pilot-01/raw/r10/receipt.json
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"scenario": "S05",
|
||||
"condition": "prose",
|
||||
"prompt_sha256": "7f7a1018ef79ada7f306093e3cf5b5495fe6d98cdc04da5d18bbe46d0fb3440d",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e",
|
||||
"status": "completed",
|
||||
"output_words": 211,
|
||||
"elapsed_seconds": 2.466,
|
||||
"responses": [
|
||||
{
|
||||
"stage": "decision-review",
|
||||
"model": "google/gemini-2.5-flash",
|
||||
"usage": {
|
||||
"prompt_tokens": 1015,
|
||||
"completion_tokens": 297,
|
||||
"total_tokens": 1312
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
"provider": "openrouter",
|
||||
"response_id": "gen-1790445722-fqRN9QZGf8oEhYcEqRmw",
|
||||
"latency_seconds": 2.528,
|
||||
"profile": "custodian-triage-balanced",
|
||||
"profile_model": "google/gemini-2.5-flash",
|
||||
"profile_provider": "openrouter",
|
||||
"diagnostics": {
|
||||
"available": true,
|
||||
"effective_settings": {
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"reasoning_effort_present": false,
|
||||
"reasoning_present": false
|
||||
},
|
||||
"provider_usage": {
|
||||
"prompt_tokens": 1015,
|
||||
"completion_tokens": 297,
|
||||
"total_tokens": 1312,
|
||||
"cost": 0.001047,
|
||||
"completion_tokens_details": {
|
||||
"reasoning_tokens": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"content_characters": 1359,
|
||||
"content_sha256": "338eafdf7ce803f1cae6a5126596a6f75683a7eef0eb7c3ec7ca92b765337d00"
|
||||
}
|
||||
]
|
||||
}
|
||||
18
research/evaluations/language-pilot-01/raw/r10/started.json
Normal file
18
research/evaluations/language-pilot-01/raw/r10/started.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"status": "attempting",
|
||||
"scenario": "S05",
|
||||
"condition": "prose",
|
||||
"prompt_sha256": "7f7a1018ef79ada7f306093e3cf5b5495fe6d98cdc04da5d18bbe46d0fb3440d",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e"
|
||||
}
|
||||
6
research/evaluations/language-pilot-01/raw/r11/answer.md
Normal file
6
research/evaluations/language-pilot-01/raw/r11/answer.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
Here's an assessment of the proposed design:
|
||||
|
||||
* **Outcome/State:** The design cannot guarantee edits survive device loss or complete power failure, nor can it claim recovery is solved.
|
||||
* **Preserved Information:** Only two copies of a conflict are stored on the same device, and the sync protocol propagates deletions, which risks data loss.
|
||||
* **Recovery Action and Rejected Alternative:** The proposal to remove conflict review from the keyboard interface is premature and dangerous, as it eliminates the user's ability to resolve divergent versions.
|
||||
* **Missing Contract or Clarification:** There is no documented contract for restart-safe offline editing that covers device loss or power failure, nor is there a clear definition of how deletions are handled during conflict resolution.
|
||||
27
research/evaluations/language-pilot-01/raw/r11/prompt.txt
Normal file
27
research/evaluations/language-pilot-01/raw/r11/prompt.txt
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
You are reviewing a proposed frontend architecture. Use the reference only where applicable. Return at most 220 words, using four short bullets: (1) outcome/state, (2) preserved information, (3) recovery action and rejected alternative, (4) missing contract or clarification. Explain the reasoning without naming or quoting reference headings, pattern names, or the presentation style. Do not infer undocumented guarantees. No tables, scores, source searching or tool calls. The task is a recommendation, not a claim you implemented or tested it.
|
||||
|
||||
REFERENCE
|
||||
Reference guidance. All mechanisms remain research hypotheses with limited evidence; apply only when their conditions hold. Distinguish assumed guarantees from observations. Do not infer missing backend, storage, access or retention contracts.
|
||||
|
||||
Keep the last observed record, unfinished edits and current server observation separate. A refresh or rejected write must not silently replace newer typing. Submitted content may move into a pending-operation record before the composer is cleared. Choose navigation/restart retention and privacy rules explicitly; local content alone is not authority.
|
||||
|
||||
When an update depends on an observed base, send that expected version. Require atomic server validation and commit; a stale rejection preserves intent for comparison with current data. Blindly substituting a new version on retry can erase concurrent intent. Version preconditions and retry identities solve different problems.
|
||||
|
||||
Separate pending, accepted, explicitly rejected and unknown outcomes using operation-specific evidence. A timeout, cancelled request, generic server error or rejected promise alone does not prove rejection. An authenticated matching server event may confirm acceptance before a request finishes. Later transport errors cannot reverse that evidence or clear newer work. Retry the same immutable payload and operation identity only within a documented deduplication scope and retention window. Missing identity or expired receipts require reconciliation, not a claim that a fresh ID is safe. Cancelling local tracking does not undo remote acceptance; reversal is a separate authorized action.
|
||||
|
||||
Acceptance and view freshness are separate. Refresh affected observations while preserving unfinished work. Local request generations can reject superseded reads, but cannot order opaque server versions or repair stale replicas. A read-after-write contract or documented server ordering is needed. Report accepted-with-unresolved-freshness when appropriate; a refresh failure does not undo a write.
|
||||
|
||||
For restart-safe offline editing, acknowledge local save only after a recoverable commit of the revision and its pending upload intent. Immediate feedback is not a storage receipt. Ordinary process restart, power loss and device loss require different evidence. An in-memory queue alone cannot satisfy a restart promise. Local success does not imply remote acceptance.
|
||||
|
||||
Retain revision and operation identity with pending work across restart. A sent request is not an acknowledgement; receiver acceptance is not delivery to every device or backup. Define receiver deduplication, ancestry and authorization behavior. Lost acknowledgements retain the original identity. Block obsolete uploads on detected divergence; atomically supersede them when recording a replacement. Local retirement cannot stop an in-flight stale write: the receiver must enforce preconditions. A late receipt updates its own operation history, not a newer operation.
|
||||
|
||||
When incompatible edits are detected, retain alternate content with identity and provenance before destructive replacement. Selecting an active copy is not semantic resolution. Protect both versions equally and expose unresolved status. Storage costs, unnoticed copies and missed conflict detection remain risks. Sync does not substitute for separately recoverable backup.
|
||||
|
||||
Let a person compare, choose or combine divergent versions and commit the decision before removing recovery material. Accessible notification, keyboard comparison and stable focus are requirements, not automatic consequences of storing copies. Two devices can reconcile concurrently; the resulting revisions still need divergence handling. A version store may preserve the same evidence without separate copy files; justified domain-aware merging is an alternative.
|
||||
|
||||
Connections in the workflow: preserved unfinished edits support recovery from rejected conditional writes. Operation-specific acceptance evidence informs refreshing observations. A local recoverable commit enables restartable propagation. Detected divergence during propagation can require preservation of alternate content, then an intentional resolution, which creates another local commit. These connections are conditional, not universal ordering.
|
||||
|
||||
Evidence limits shared by both presentations: Joplin/Obsidian document conflict copies but do not establish our atomic crash contract. Kubernetes documents conditional updates; GitLab has a different, sometimes nonspecific failure surface. Stripe documents bounded idempotency. React Router illustrates revalidation but browser cancellation does not stop server writes. Pinned Zulip/Element source shows partial draft and pending-message mechanisms. A published Matrix SDK test with a synthetic receiver shows a rejected promise can coexist with a reconciled event, and fresh identity after local cancellation can duplicate simulated acceptance. This is not full application/server validation or independent full-pattern recurrence.
|
||||
|
||||
SCENARIO
|
||||
A field-note product promises edits survive device loss and complete power failure. The chosen local transaction has only been tested against ordinary process restart. Two copies of a conflict are stored on the same device, and the sync protocol propagates deletions. The team proposes declaring recovery solved and removing conflict review from the keyboard interface. Assess that design and state what can actually be promised.
|
||||
58
research/evaluations/language-pilot-01/raw/r11/receipt.json
Normal file
58
research/evaluations/language-pilot-01/raw/r11/receipt.json
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"scenario": "S06",
|
||||
"condition": "prose",
|
||||
"prompt_sha256": "1fcc92360175d793653e731c94615a2e049d15c2711d9715bc9b753a2eb88d3a",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e",
|
||||
"status": "completed",
|
||||
"output_words": 118,
|
||||
"elapsed_seconds": 1.47,
|
||||
"responses": [
|
||||
{
|
||||
"stage": "decision-review",
|
||||
"model": "google/gemini-2.5-flash",
|
||||
"usage": {
|
||||
"prompt_tokens": 1010,
|
||||
"completion_tokens": 154,
|
||||
"total_tokens": 1164
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
"provider": "openrouter",
|
||||
"response_id": "gen-1790445724-MzXKkIGMPijfZl6aFxjQ",
|
||||
"latency_seconds": 1.317,
|
||||
"profile": "custodian-triage-balanced",
|
||||
"profile_model": "google/gemini-2.5-flash",
|
||||
"profile_provider": "openrouter",
|
||||
"diagnostics": {
|
||||
"available": true,
|
||||
"effective_settings": {
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"reasoning_effort_present": false,
|
||||
"reasoning_present": false
|
||||
},
|
||||
"provider_usage": {
|
||||
"prompt_tokens": 1010,
|
||||
"completion_tokens": 154,
|
||||
"total_tokens": 1164,
|
||||
"cost": 0.000688,
|
||||
"completion_tokens_details": {
|
||||
"reasoning_tokens": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"content_characters": 796,
|
||||
"content_sha256": "6c60c9c12851331cfd8f3f1af4ac6455cdf41e62b79d89fb7b37f464707ace12"
|
||||
}
|
||||
]
|
||||
}
|
||||
18
research/evaluations/language-pilot-01/raw/r11/started.json
Normal file
18
research/evaluations/language-pilot-01/raw/r11/started.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"status": "attempting",
|
||||
"scenario": "S06",
|
||||
"condition": "prose",
|
||||
"prompt_sha256": "1fcc92360175d793653e731c94615a2e049d15c2711d9715bc9b753a2eb88d3a",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e"
|
||||
}
|
||||
6
research/evaluations/language-pilot-01/raw/r12/answer.md
Normal file
6
research/evaluations/language-pilot-01/raw/r12/answer.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
Here's an assessment of the proposed frontend architecture:
|
||||
|
||||
* **Outcome/State:** The product cannot guarantee survival of edits through device loss or complete power failure, nor can it claim conflict resolution is solved.
|
||||
* **Preserved Information:** Only two copies of a conflict are retained on the same device, and the sync protocol propagates deletions, which is insufficient for robust recovery.
|
||||
* **Recovery Action and Rejected Alternative:** The current local transaction only handles process restarts; it cannot recover from power or device loss. Removing conflict review from the keyboard interface is premature and risky.
|
||||
* **Missing Contract or Clarification:** There's no documented contract for restart-safe storage beyond process restart, nor for how deletions are handled during conflict resolution or synchronization across devices.
|
||||
43
research/evaluations/language-pilot-01/raw/r12/prompt.txt
Normal file
43
research/evaluations/language-pilot-01/raw/r12/prompt.txt
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
You are reviewing a proposed frontend architecture. Use the reference only where applicable. Return at most 220 words, using four short bullets: (1) outcome/state, (2) preserved information, (3) recovery action and rejected alternative, (4) missing contract or clarification. Explain the reasoning without naming or quoting reference headings, pattern names, or the presentation style. Do not infer undocumented guarantees. No tables, scores, source searching or tool calls. The task is a recommendation, not a claim you implemented or tested it.
|
||||
|
||||
REFERENCE
|
||||
Reference guidance. All mechanisms remain research hypotheses with limited evidence; apply only when their conditions hold. Distinguish assumed guarantees from observations. Do not infer missing backend, storage, access or retention contracts.
|
||||
|
||||
## Local Draft State
|
||||
|
||||
Keep the last observed record, unfinished edits and current server observation separate. A refresh or rejected write must not silently replace newer typing. Submitted content may move into a pending-operation record before the composer is cleared. Choose navigation/restart retention and privacy rules explicitly; local content alone is not authority.
|
||||
|
||||
## Version-Checked Mutation
|
||||
|
||||
When an update depends on an observed base, send that expected version. Require atomic server validation and commit; a stale rejection preserves intent for comparison with current data. Blindly substituting a new version on retry can erase concurrent intent. Version preconditions and retry identities solve different problems.
|
||||
|
||||
## Mutation Outcome Tracking
|
||||
|
||||
Separate pending, accepted, explicitly rejected and unknown outcomes using operation-specific evidence. A timeout, cancelled request, generic server error or rejected promise alone does not prove rejection. An authenticated matching server event may confirm acceptance before a request finishes. Later transport errors cannot reverse that evidence or clear newer work. Retry the same immutable payload and operation identity only within a documented deduplication scope and retention window. Missing identity or expired receipts require reconciliation, not a claim that a fresh ID is safe. Cancelling local tracking does not undo remote acceptance; reversal is a separate authorized action.
|
||||
|
||||
## Invalidate and Reproject
|
||||
|
||||
Acceptance and view freshness are separate. Refresh affected observations while preserving unfinished work. Local request generations can reject superseded reads, but cannot order opaque server versions or repair stale replicas. A read-after-write contract or documented server ordering is needed. Report accepted-with-unresolved-freshness when appropriate; a refresh failure does not undo a write.
|
||||
|
||||
## Local Commit Boundary
|
||||
|
||||
For restart-safe offline editing, acknowledge local save only after a recoverable commit of the revision and its pending upload intent. Immediate feedback is not a storage receipt. Ordinary process restart, power loss and device loss require different evidence. An in-memory queue alone cannot satisfy a restart promise. Local success does not imply remote acceptance.
|
||||
|
||||
## Pending Synchronization
|
||||
|
||||
Retain revision and operation identity with pending work across restart. A sent request is not an acknowledgement; receiver acceptance is not delivery to every device or backup. Define receiver deduplication, ancestry and authorization behavior. Lost acknowledgements retain the original identity. Block obsolete uploads on detected divergence; atomically supersede them when recording a replacement. Local retirement cannot stop an in-flight stale write: the receiver must enforce preconditions. A late receipt updates its own operation history, not a newer operation.
|
||||
|
||||
## Conflict Copy
|
||||
|
||||
When incompatible edits are detected, retain alternate content with identity and provenance before destructive replacement. Selecting an active copy is not semantic resolution. Protect both versions equally and expose unresolved status. Storage costs, unnoticed copies and missed conflict detection remain risks. Sync does not substitute for separately recoverable backup.
|
||||
|
||||
## Explicit Reconciliation
|
||||
|
||||
Let a person compare, choose or combine divergent versions and commit the decision before removing recovery material. Accessible notification, keyboard comparison and stable focus are requirements, not automatic consequences of storing copies. Two devices can reconcile concurrently; the resulting revisions still need divergence handling. A version store may preserve the same evidence without separate copy files; justified domain-aware merging is an alternative.
|
||||
|
||||
Connections: Local Draft State supports recovery from Version-Checked Mutation. Mutation Outcome Tracking supplies acceptance evidence to Invalidate and Reproject. Local Commit Boundary enables Pending Synchronization. Detected divergence can lead from Pending Synchronization to Conflict Copy, then Explicit Reconciliation, which creates another local commit. These connections are conditional, not universal ordering.
|
||||
|
||||
Evidence limits shared by both presentations: Joplin/Obsidian document conflict copies but do not establish our atomic crash contract. Kubernetes documents conditional updates; GitLab has a different, sometimes nonspecific failure surface. Stripe documents bounded idempotency. React Router illustrates revalidation but browser cancellation does not stop server writes. Pinned Zulip/Element source shows partial draft and pending-message mechanisms. A published Matrix SDK test with a synthetic receiver shows a rejected promise can coexist with a reconciled event, and fresh identity after local cancellation can duplicate simulated acceptance. This is not full application/server validation or independent full-pattern recurrence.
|
||||
|
||||
SCENARIO
|
||||
A field-note product promises edits survive device loss and complete power failure. The chosen local transaction has only been tested against ordinary process restart. Two copies of a conflict are stored on the same device, and the sync protocol propagates deletions. The team proposes declaring recovery solved and removing conflict review from the keyboard interface. Assess that design and state what can actually be promised.
|
||||
58
research/evaluations/language-pilot-01/raw/r12/receipt.json
Normal file
58
research/evaluations/language-pilot-01/raw/r12/receipt.json
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"scenario": "S06",
|
||||
"condition": "patterns",
|
||||
"prompt_sha256": "e69635967b0a309b74affbbeb9f14b22b42a81f2a6bf7bb7a7d98a3ecb39f288",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e",
|
||||
"status": "completed",
|
||||
"output_words": 120,
|
||||
"elapsed_seconds": 1.668,
|
||||
"responses": [
|
||||
{
|
||||
"stage": "decision-review",
|
||||
"model": "google/gemini-2.5-flash",
|
||||
"usage": {
|
||||
"prompt_tokens": 1051,
|
||||
"completion_tokens": 157,
|
||||
"total_tokens": 1208
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
"provider": "openrouter",
|
||||
"response_id": "gen-1790445726-UEDG24Nb5EZJxRw3TYhh",
|
||||
"latency_seconds": 1.657,
|
||||
"profile": "custodian-triage-balanced",
|
||||
"profile_model": "google/gemini-2.5-flash",
|
||||
"profile_provider": "openrouter",
|
||||
"diagnostics": {
|
||||
"available": true,
|
||||
"effective_settings": {
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"reasoning_effort_present": false,
|
||||
"reasoning_present": false
|
||||
},
|
||||
"provider_usage": {
|
||||
"prompt_tokens": 1051,
|
||||
"completion_tokens": 157,
|
||||
"total_tokens": 1208,
|
||||
"cost": 0.0007078,
|
||||
"completion_tokens_details": {
|
||||
"reasoning_tokens": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"content_characters": 858,
|
||||
"content_sha256": "f5965dd3ada0156bf46bbe5cdde96893b3caecd8600729d322d2b934ba4a4c0e"
|
||||
}
|
||||
]
|
||||
}
|
||||
18
research/evaluations/language-pilot-01/raw/r12/started.json
Normal file
18
research/evaluations/language-pilot-01/raw/r12/started.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"status": "attempting",
|
||||
"scenario": "S06",
|
||||
"condition": "patterns",
|
||||
"prompt_sha256": "e69635967b0a309b74affbbeb9f14b22b42a81f2a6bf7bb7a7d98a3ecb39f288",
|
||||
"service_requests_attempted": 1,
|
||||
"endpoint": "http://127.0.0.1:18081",
|
||||
"config": {
|
||||
"model_name": "custodian-triage-balanced",
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1400,
|
||||
"model_params": {},
|
||||
"max_depth": 3,
|
||||
"skip_if_exists": true,
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"freeze_sha256": "dea6d1f7053c492093f91ec55fe516736f9d704e2d2f52b0842de455d56c008e"
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue