Investigate repeated owner login failures with isolated callback probes

Assistant: codex
Assistant-Model: gpt-6.1-sol
Assistant-Session: 01a1abf1-7e95-7141-87c9-ee78332bf790

docs/evidence/2026-11-06-vergabe-owner-login-investigation.json

@@ -0,0 +1,127 @@
+{
+  "schema": "railiance.vergabe-pilot-owner-login-investigation.v1",
+  "evidence_kind": "user_reported_terminal_output_and_agent_synthetic_probe",
+  "recorded_at": "2026-11-06T13:54:02.318076+00:00",
+  "workplan": "RAPPS-WP-0014",
+  "task": "RAPPS-WP-0014-T03",
+  "run_id": "20261106T100825Z-acc2a468a535",
+  "user_observation": "Browser reacts very slowly; feedback/confirmation page seems unresponsive. User suspects timeouts.",
+  "attempts": [
+    {
+      "login_diagnostic": {
+        "category": "callback_timeout",
+        "client_exit": 2,
+        "helper_present": false,
+        "stdout_allowed": true,
+        "stderr_allowed": false,
+        "command_handoff_completed": false
+      },
+      "provider_receipt": {
+        "status": "owner_login_or_child_not_completed",
+        "provider_outcome": "unconfirmed",
+        "warden_exit_code": 10,
+        "receipt_file": "/home/worsch/.local/state/vergabe-pilot-recovery/20261106T100825Z-acc2a468a535/provider-receipt-7b66913de63c.json",
+        "session_cleanup_confirmed": null,
+        "session_revocation_confirmed": null,
+        "session_storage_cleanup_confirmed": null
+      },
+      "final": {
+        "status": "attended_backup_failed",
+        "failure": "owner_operation_incomplete",
+        "details_suppressed": true
+      }
+    },
+    {
+      "login_diagnostic": {
+        "category": "authentication_error",
+        "client_exit": 2,
+        "helper_present": false,
+        "stdout_allowed": true,
+        "stderr_allowed": false,
+        "command_handoff_completed": false
+      },
+      "provider_receipt": {
+        "status": "owner_login_or_child_not_completed",
+        "provider_outcome": "unconfirmed",
+        "warden_exit_code": 10,
+        "receipt_file": "/home/worsch/.local/state/vergabe-pilot-recovery/20261106T100825Z-acc2a468a535/provider-receipt-20d1f47a82ee.json",
+        "session_cleanup_confirmed": null,
+        "session_revocation_confirmed": null,
+        "session_storage_cleanup_confirmed": null
+      },
+      "final": {
+        "status": "attended_backup_failed",
+        "failure": "owner_operation_incomplete",
+        "details_suppressed": true
+      }
+    }
+  ],
+  "native_provider_child_started": false,
+  "fresh_point_provider_verified": false,
+  "fresh_point_restore_verified": false,
+  "revocation_outcome": "unconfirmed",
+  "agent_inspected_real_browser_profile_or_token_helper": false,
+  "read_only_http_checks": [
+    {
+      "host": "kc.coulomb.social",
+      "path": "/.well-known/openid-configuration",
+      "status": 200,
+      "elapsed_seconds": 0.054,
+      "issuer_matches": true
+    },
+    {
+      "host": "vergabe-teilnahme.coulomb.social",
+      "path": "/demo-company/health/",
+      "status": 200,
+      "elapsed_seconds": 0.054
+    },
+    {
+      "host": "127.0.0.1:18200",
+      "path": "/v1/sys/health",
+      "status": 200,
+      "elapsed_seconds": 0.033
+    }
+  ],
+  "synthetic_probe": {
+    "source": "tests/vergabe_operator_callback_fixture.cjs",
+    "sha256": "b6a735fe8a20b837662889e2b22ce57ef7fca66e7e765df4d7500ce9493c5112",
+    "browser_version": "151.0.7922.34",
+    "native_login": false,
+    "real_profile_or_credentials_used": false,
+    "headless": {
+      "launch_ms": 300,
+      "localhost_callback_ms": 190,
+      "ipv4_callback_ms": 150
+    },
+    "headed": {
+      "launch_ms": 598,
+      "localhost_callback_ms": 354,
+      "ipv4_callback_ms": 224
+    },
+    "both_completed": true,
+    "scope": "Fresh Playwright contexts, public discovery document, synthetic Continue link from that origin to a disposable local HTTP listener. Headed probe launched with explicit GUI approval. No native OIDC authorization request or identity exchange."
+  },
+  "source_review": {
+    "diagnosis_page": "key-cape/src/internal/server/oidc/signin_diagnosis.go",
+    "contract": "net-kingdom/docs/sign-in-diagnosis.md",
+    "diagnosis_continue_required": true,
+    "diagnosis_page_fields": [
+      "uid",
+      "groups",
+      "assurance",
+      "client_id"
+    ],
+    "local_callback_role_metadata_precedent": "railiance-platform/docs/evidence/2026-09-24-platform-admin-callback-prune.json",
+    "local_callback_precedent": [
+      "http://localhost:8250/oidc/callback",
+      "http://127.0.0.1:8250/oidc/callback"
+    ]
+  },
+  "conclusions": [
+    "Public/tunnel checks and synthetic callbacks were responsive at observation time; this does not reproduce the founder login or human input latency.",
+    "Neither an IPv4/IPv6 callback defect nor slow rendering is established by these observations.",
+    "The authentication_error classification is broader than callback_timeout and does not establish a specific credentials, assurance or claims defect.",
+    "Real stalled page hostname and heading/button text were requested without URLs, credentials or OTPs.",
+    "Pause native retries until the stalled page is identified; preserve the same completed snapshot. No browser or issuer setting, allowlist, callback registration or token-helper control was changed."
+  ]
+}

docs/proposals/2026-11-06-vergabe-attended-backups/README.md

@@ -28,6 +28,21 @@ verifies the private encrypted pair and uses a fresh contained login; it does
 not pause or recapture the live application. The previous failure receipt remains
 as historical evidence. No allowlist, token helper or login control is bypassed.
 
+Subsequent same-point retries also failed before child handoff: first callback
+timeout, then authentication_error (both Warden 10). The founder reports slow
+browser interaction and an unresponsive feedback/confirmation page. **Pause
+further native retries pending diagnosis.** The [investigation receipt](../../evidence/2026-11-06-vergabe-owner-login-investigation.json)
+records responsive public/tunnel checks and synthetic local callbacks through
+both hostname and IPv4, in headless and headed fresh browser contexts. Those
+checks do not reproduce the actual identity flow or human input latency and do
+not establish a browser, IPv6, credentials or claims cause. The stalled page's
+hostname and heading/button text are requested, without full URLs or credentials.
+The KeyCape diagnosis page has a Continue link that must complete the waiting
+CLI callback; it is distinct from an already-completed callback's feedback page.
+No setting, callback registration, output allowlist or authentication requirement
+was changed. The completed capture remains available; native retry/restore proof
+and possible session revocation remain unconfirmed.
+
 ## First operating run — founder terminal
 
 ```bash

docs/vergabe-demo-company-operating-acceptance.proposed.md

@@ -53,6 +53,12 @@ timeout before provider command handoff. Retry `--publish` for that same run wit
 `--verify-restore` and return the sanitized backup/worker/transport receipts.
 The [return](evidence/2026-11-06-vergabe-attended-callback-timeout-return.json)
 does not establish fresh provider publication, restore or session revocation.
+Two further retries report callback_timeout and authentication_error before
+provider handoff. Further native retries await the founder's stalled-page
+hostname/heading details; see the [login investigation](evidence/2026-11-06-vergabe-owner-login-investigation.json).
+Public/tunnel and fresh synthetic headed/headless callback checks pass, but do
+not reproduce the real identity flow or establish a root cause. The completed
+point remains available and no authentication control was changed.
 Review provider retention previews
 and execute only when eligible excess points exist; no artificial deletion is
 needed to accept an empty retention plan.

tests/vergabe_operator_callback_fixture.cjs

@@ -0,0 +1,46 @@
+const http = require('http');
+const fs = require('fs');
+const {chromium} = require('/home/worsch/coulomb-social/scripts/node_modules/playwright');
+async function main() {
+  const spec = JSON.parse(fs.readFileSync('/home/worsch/coulomb-social/scripts/node_modules/playwright-core/browsers.json'));
+  const rev = spec.browsers.find(x => x.name === 'chromium').revision;
+  const executablePath = `/home/worsch/.cache/ms-playwright/chromium-${rev}/chrome-linux64/chrome`;
+  const server = http.createServer((req,res) => {res.writeHead(200,{'Content-Type':'text/html'});res.end('<h1>Synthetic callback received</h1>');});
+  await new Promise(resolve => server.listen(0,'127.0.0.1',resolve));
+  const port = server.address().port;
+  let browser;
+  const results = {synthetic_only:true,native_login:false,headless:!process.argv.includes("--headed")};
+  let stage = 'browser_launch';
+  try {
+    let start = Date.now();
+    browser = await chromium.launch({executablePath,headless:results.headless,chromiumSandbox:true,timeout:20000});
+    results.launch_ms = Date.now()-start;
+    results.version = browser.version();
+    results.callbacks = [];
+    for(const host of ['localhost','127.0.0.1']) {
+      const context = await browser.newContext();
+      const page = await context.newPage();
+      stage = 'public_discovery';start=Date.now();
+      await page.goto('https://kc.coulomb.social/.well-known/openid-configuration',{waitUntil:'domcontentloaded',timeout:15000});
+      const discovery_ms = Date.now()-start;
+      await page.setContent(`<h1>You are signed in — synthetic</h1><a href="http://${host}:${port}/oidc/callback?state=synthetic">Continue</a>`);
+      stage = 'synthetic_callback';start=Date.now();
+      let error_kind = null;
+      try {
+        await Promise.all([page.waitForURL(`http://${host}:${port}/oidc/callback?state=synthetic`,{timeout:10000,waitUntil:'domcontentloaded'}),page.getByRole('link',{name:'Continue'}).click({timeout:10000})]);
+        await page.getByRole('heading',{name:'Synthetic callback received'}).waitFor({timeout:1000});
+      } catch(error) {error_kind=/ERR_CONNECTION_RESET/.test(error.message)?'connection_reset':/ERR_CONNECTION_REFUSED/.test(error.message)?'connection_refused':/Timeout/.test(error.name)?'timeout':'browser_error';}
+      results.callbacks.push({host,discovery_ms,callback_ms:Date.now()-start,error_kind});
+      await context.close();
+    }
+    results.status='complete';
+  } catch(error) {
+    results.status='failed';results.stage=stage;results.error_kind=error.name;
+    results.permission_error=/Operation not permitted|Permission denied|No usable sandbox/.test(error.message);
+  } finally {
+    if(browser)await browser.close();
+    await new Promise(resolve => server.close(resolve));
+  }
+  process.stdout.write(JSON.stringify(results,null,2)+'\n');
+}
+main().catch(() => {process.stdout.write('{"status":"probe_setup_failed","details_suppressed":true}\n');process.exitCode=1;});

workplans/RAPPS-WP-0014-vergabe-invited-pilot.md

@@ -196,10 +196,10 @@ id: RAPPS-WP-0014-T03
 status: wait
 needs_human: true
 decision_id: d06307e5-2737-461a-a2f4-a12c3b492a9a
-intervention_note: "Baseline native recovery passes; attended model selected. First operating capture/resume and encrypted transfer pass for 20261106T100825Z-acc2a468a535 (39 tables/249 rows, 83.11s). Contained login callback times out before provider child (Warden 10); possible session revocation unconfirmed. Retry python3 tools/vergabe_pilot_backup.py --publish 20261106T100825Z-acc2a468a535 --verify-restore with fresh browser login. No recapture/pause needed. T03 waits for this point's provider/restore receipts."
+intervention_note: "Baseline recovery and fresh attended capture/transfer pass. Same-point login retries fail before provider child: callback_timeout, then authentication_error; possible revocation unconfirmed. Founder reports slow/unresponsive confirmation page. Pause retries. Public discovery/tunnel and synthetic headed/headless Continue-to-loopback probes pass; actual stalled page hostname/heading requested without URLs/credentials. Preserve run 20261106T100825Z-acc2a468a535; publication/restore still pending."
 priority: high
 assignee: the-custodian
-blocking_reason: "Baseline recovery and accepted attended model pass; repeatable capture/transfer now pass. First fresh operating point still awaits contained owner login, provider publication/readback/decryption and restore after callback timeout. Restored native SSO/ingress and broader version rollback remain explicit limits."
+blocking_reason: "Baseline recovery and accepted attended model pass; repeatable capture/transfer now pass. Contained owner login fails before provider handoff with callback_timeout/authentication_error; slow confirmation-page cause awaits identification. Same fresh point still needs provider/readback/decryption and restore. Restored native SSO/ingress and broader version rollback remain explicit limits."
 state_hub_task_id: "dd069c6d-fcc1-5bac-b233-976f2f0d5cd1"
 ```
 
@@ -574,6 +574,26 @@ No code/control change is indicated by this timeout. Retry only
 and complete its fresh contained browser login. The app is already resumed;
 the completed capture does not need repeating. T03 remains wait.
 
+2026-11-06 login investigation: two same-point retries fail before provider
+child handoff with callback_timeout and then authentication_error, both client
+exit 2/helper absent/Warden 10. The founder reports slow browser reaction and
+an unresponsive feedback/confirmation page. Further native retries are paused
+for diagnosis; no credentials/assurance/claims root cause is inferred from the
+generic authentication_error category. Public issuer discovery and OpenBao
+tunnel health respond in 54ms/33ms. A fresh synthetic Chromium 151 probe passes
+Continue navigation to both localhost and 127.0.0.1 callbacks: 190ms/150ms
+headless and 354ms/224ms headed; no native identity request, real profile or
+credential is used. Headed launch was explicitly approved. These observations
+do not reproduce the founder's actual identity flow or human input latency.
+The KeyCape diagnosis-page source/contract requires Continue to finish the
+waiting callback. Requested only the stalled page's hostname and heading/button
+text, not its full URL, credentials or OTP. Receipt:
+`docs/evidence/2026-11-06-vergabe-owner-login-investigation.json`; reproducible
+synthetic fixture `tests/vergabe_operator_callback_fixture.cjs`. No browser,
+issuer, callback, output allowlist or token-helper control was changed. Preserve
+the completed run; publication/restore and possible session revocation remain
+unconfirmed. T03 retains the existing wait.
+
 
 ## Validate synthetic sizing against the exact release and shared contention