Implement audited lost-factor recovery and track remaining P04 acceptance

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a092fe-13b1-7f12-ac74-7d258af4d79c

.forgejo/workflows/acceptance.yaml

@@ -2,7 +2,7 @@ name: Authentication acceptance
 on:
   push:
     branches: [main]
-    paths: ["src/**", "scripts/provider-onboarding-contract.py", ".forgejo/workflows/acceptance.yaml"]
+    paths: ["src/**", "scripts/provider-onboarding-contract.py", "scripts/factor_recovery.py", "scripts/test_factor_recovery.py", ".forgejo/workflows/acceptance.yaml"]
   workflow_dispatch:
 jobs:
   acceptance:
@@ -36,5 +36,6 @@ jobs:
                   archive.write(response.read());archive.seek(0)
                   with tarfile.open(fileobj=archive,mode='r:gz') as tar:tar.extractall(root,filter='data')
               script=next(root.glob('*/scripts/provider-onboarding-contract.py'))
+              subprocess.run(['python3','-m','unittest','discover','-s',str(script.parent),'-p','test_factor_recovery.py'],check=True,timeout=30)
               subprocess.run(['python3',str(script)],check=True,timeout=90)
           PYCODE

scripts/factor_recovery.py

@@ -0,0 +1,94 @@
+"""Bounded provider recovery: preview, stale-state guard, disable, audit and replay."""
+import hashlib,json,re
+
+ACTION='keycape.factor.recovery'
+class RecoveryError(Exception):pass
+
+def fingerprint(snapshot):
+ return hashlib.sha256(json.dumps(snapshot,sort_keys=True,separators=(',',':')).encode()).hexdigest()
+
+def recover(store, request):
+ for key in ('user','serial','realm','actor','reference'):
+  value=request.get(key)
+  if not isinstance(value,str) or not value or len(value)>200 or not re.fullmatch(r'[A-Za-z0-9_.@:/-]+',value):raise RecoveryError('invalid_request')
+ if request['realm']!=store.realm:raise RecoveryError('wrong_realm')
+ snapshot=store.snapshot(request['user'],request['serial'])
+ if snapshot is None:raise RecoveryError('factor_not_owned_by_target')
+ version=fingerprint(snapshot)
+ base={k:request[k] for k in ('user','serial','realm','actor','reference')}
+ base.update(scope='shared_identity_across_applications',operation='disable_one_factor',version=version)
+ if not request.get('apply'):
+  return dict(base,status='preview',active=snapshot['active'],changes_applied=False)
+ if request.get('identity_verified') is not True:raise RecoveryError('identity_verification_required')
+ expected=request.get('expected_version')
+ if not isinstance(expected,str) or not re.fullmatch(r'[0-9a-f]{64}',expected):raise RecoveryError('preview_required')
+ prior=store.receipt(request['reference'])
+ if prior:
+  if any(prior.get(k)!=request[k] for k in ('user','serial','realm','actor')) or prior.get('version')!=expected:raise RecoveryError('reference_conflict')
+  if not snapshot['active']:
+   if fingerprint(dict(snapshot,active=True))!=expected:raise RecoveryError('factor_changed_after_recovery')
+   if not prior['complete']:store.record(base,expected,True)
+   return dict(base,status='recovered',changes_applied=False,replayed=True)
+  if prior['complete']:raise RecoveryError('factor_changed_after_recovery')
+ if version!=expected:raise RecoveryError('stale_preview')
+ if not snapshot['active']:raise RecoveryError('factor_already_inactive')
+ store.record(base,expected,False)  # An audit outage prevents mutation.
+ store.disable(request['user'],request['serial'],expected)
+ current=store.snapshot(request['user'],request['serial'])
+ if current is None or current['active']:raise RecoveryError('readback_failed')
+ store.record(base,expected,True)
+ return dict(base,status='recovered',changes_applied=True,replayed=False)
+
+class ProviderStore:
+ def __init__(self, app, realm='coulomb'):
+  self.app=app;self.realm=realm
+ def _token(self,user,serial):
+  from privacyidea.lib.token import get_tokens
+  from privacyidea.lib.user import User
+  values=get_tokens(user=User(user,self.realm),serial=serial)
+  if len(values)!=1:return None
+  return values[0].token
+ def snapshot(self,user,serial):
+  token=self._token(user,serial)
+  if token is None:return None
+  data=token.get_vars()
+  return {k:data[k] for k in ('id','serial','user_id','resolver','realms','active','rollout_state','tokentype')}
+ def disable(self,user,serial,expected):
+  from privacyidea.models import db,Token
+  token=self._token(user,serial)
+  if token is None:raise RecoveryError('factor_not_owned_by_target')
+  locked=db.session.query(Token).filter(Token.id==token.id).with_for_update().one()
+  db.session.refresh(locked)
+  if fingerprint(self.snapshot(user,serial))!=expected:
+   db.session.rollback();raise RecoveryError('stale_preview')
+  locked.active=False
+  db.session.commit()
+ def _audit(self):
+  from privacyidea.lib.audit import getAudit
+  return getAudit(self.app.config)
+ def receipt(self,reference):
+  page=self._audit().search({'action':ACTION,'action_detail':reference},page_size=1,sortorder='desc')
+  if not page.auditdata:return None
+  row=page.auditdata[0]
+  return {'user':row['user'],'serial':row['serial'],'realm':row['realm'],'actor':row['administrator'],'version':row['info'],'complete':bool(row['success'])}
+ def record(self,request,version,complete):
+  audit=self._audit()
+  audit.log({'action':ACTION,'action_detail':request['reference'],'user':request['user'],'serial':request['serial'],'realm':request['realm'],'administrator':request['actor'],'info':version,'success':bool(complete)})
+  audit.finalize_log()
+  # Do not treat a silently failed audit append as durable evidence.
+  row=self.receipt(request['reference'])
+  if row is None or any(row.get(k)!=request[k] for k in ('user','serial','realm','actor')) or row['version']!=version or row['complete']!=bool(complete):raise RecoveryError('audit_unavailable')
+
+def main():
+ import contextlib,io,logging,sys
+ request=json.load(sys.stdin);result={'success':False}
+ with contextlib.redirect_stdout(io.StringIO()),contextlib.redirect_stderr(io.StringIO()):
+  try:
+   logging.disable(logging.CRITICAL)
+   from privacyidea.app import create_app
+   app=create_app(config_name='production',silent=True)
+   with app.app_context():result=dict(recover(ProviderStore(app),request),success=True)
+  except RecoveryError as e:result['failure']=str(e)
+  except Exception:result['failure']='provider_operation_unavailable'
+ print(json.dumps(result));return 0 if result['success'] else 1
+if __name__=='__main__':raise SystemExit(main())

scripts/provider-onboarding-contract.py

@@ -13,6 +13,7 @@ def run():
  with tempfile.TemporaryDirectory(prefix="provider-contract-") as d:
   root=Path(d);(root/'enckey').write_bytes(os.urandom(96))
   cfg=root/'fixture.cfg';cfg.write_text("SQLALCHEMY_DATABASE_URI='sqlite:///:memory:'\nSECRET_KEY='isolated-fixture-only'\nPI_PEPPER='isolated-fixture-only'\nPI_NO_RESPONSE_SIGN=True\nPI_AUDIT_NO_SIGN=True\nPI_LOGFILE="+repr(str(root/'log'))+"\nPI_ENCFILE="+repr(str(root/'enckey'))+"\nPI_TRUSTED_JWT=[]\n")
+  with cfg.open('a') as f:f.write('PI_AUDIT_SQL_URI='+repr('sqlite:///'+str(root/'audit.sqlite'))+'\n')
   from privacyidea.app import create_app
   from privacyidea.models import db
   from privacyidea.lib.policy import set_policy,enable_policy
@@ -24,6 +25,9 @@ def run():
   phase="fixture_database"
   with app.app_context():
    check('database_isolated',str(db.engine.url)=='sqlite:///:memory:');db.create_all()
+   from privacyidea.lib.audit import getAudit
+   from privacyidea.lib.auditmodules.sqlaudit import LogEntry
+   LogEntry.__table__.create(getAudit(app.config).engine,checkfirst=True)
    passwd=root/'users';passwd.write_text('alice:'+crypt_ctx.hash('fixture-password',scheme='sha512_crypt')+':1001:1001:Fixture:/tmp:/bin/false\n')
    save_resolver({'resolver':'fixture-users','type':'passwdresolver','fileName':str(passwd)})
    set_realm('fixture',[{'name':'fixture-users'}])
@@ -85,6 +89,18 @@ def run():
   time.sleep(3)
   code,_=req('GET','/token/',token=short)
   check('expired_provider_jwt_rejected',code==401)
+  phase='platform_admin_recovery'
+  from factor_recovery import ProviderStore,recover,RecoveryError
+  with app.app_context():
+   store=ProviderStore(app,realm='fixture')
+   request=dict(user='alice',serial=serial,realm='fixture',actor='fixture-platform-operator',reference='fixture-recovery-001')
+   preview=recover(store,request)
+   check('recovery_preview_preserves_factor',preview['active'] is True and preview['changes_applied'] is False)
+   approved=dict(request,apply=True,identity_verified=True,expected_version=preview['version'])
+   applied=recover(store,approved)
+   check('platform_recovery_disabled_factor',applied['status']=='recovered' and store.snapshot('alice',serial)['active'] is False)
+   check('platform_recovery_audit_durable',store.receipt(request['reference'])['complete'] is True)
+   check('platform_recovery_replay_no_mutation',recover(store,approved)['replayed'] is True)
   phase='finished';result['success']=True
 with contextlib.redirect_stdout(io.StringIO()),contextlib.redirect_stderr(io.StringIO()):
  try:run()

scripts/test_factor_recovery.py

@@ -0,0 +1,47 @@
+import copy,unittest
+from factor_recovery import recover,fingerprint,RecoveryError
+class Store:
+ realm='coulomb'
+ def __init__(self):
+  self.state={'id':1,'serial':'T1','user_id':'u1','resolver':'directory','realms':['coulomb'],'active':True,'rollout_state':'enrolled','tokentype':'totp'};self.rows={};self.writes=0;self.fail_audit=False;self.fail_complete=False;self.race=False
+ def snapshot(self,user,serial):return copy.deepcopy(self.state) if user=='alice' and serial=='T1' else None
+ def receipt(self,ref):return self.rows.get(ref)
+ def record(self,request,version,complete):
+  if self.fail_audit or self.fail_complete and complete:raise RecoveryError('audit_unavailable')
+  self.rows[request['reference']]=dict(request,version=version,complete=complete)
+ def disable(self,user,serial,expected):
+  if self.race:self.state['id']=2
+  if fingerprint(self.state)!=expected:raise RecoveryError('stale_preview')
+  self.state['active']=False;self.writes+=1
+class RecoveryTests(unittest.TestCase):
+ def setUp(self):self.store=Store();self.request=dict(user='alice',serial='T1',realm='coulomb',actor='operator-entity',reference='support-1')
+ def approval(self):return dict(self.request,apply=True,identity_verified=True,expected_version=recover(self.store,self.request)['version'])
+ def test_preview_never_mutates_and_names_global_scope(self):
+  p=recover(self.store,self.request);self.assertFalse(p['changes_applied']);self.assertEqual(p['scope'],'shared_identity_across_applications');self.assertEqual(self.store.writes,0)
+ def test_verified_recovery_and_replay_disable_exactly_once(self):
+  r=self.approval();self.assertTrue(recover(self.store,r)['changes_applied']);self.assertTrue(recover(self.store,r)['replayed']);self.assertEqual(self.store.writes,1)
+ def test_verification_and_ownership_denied_before_mutation(self):
+  for r in [dict(self.approval(),identity_verified=False),dict(self.request,user='bob'),dict(self.request,realm='other')]:
+   with self.assertRaises(RecoveryError):recover(self.store,r)
+  self.assertEqual(self.store.writes,0)
+ def test_stale_confirmation_and_concurrent_replacement_rejected(self):
+  r=self.approval();self.store.state['id']=2
+  with self.assertRaises(RecoveryError):recover(self.store,r)
+  self.store.state['id']=1;self.store.race=True
+  with self.assertRaises(RecoveryError):recover(self.store,r)
+  self.assertEqual(self.store.writes,0)
+ def test_audit_failure_prevents_mutation(self):
+  r=self.approval();self.store.fail_audit=True
+  with self.assertRaises(RecoveryError):recover(self.store,r)
+  self.assertEqual(self.store.writes,0)
+ def test_completion_audit_retry_does_not_repeat_mutation(self):
+  r=self.approval();self.store.fail_complete=True
+  with self.assertRaises(RecoveryError):recover(self.store,r)
+  self.store.fail_complete=False;self.assertTrue(recover(self.store,r)['replayed']);self.assertEqual(self.store.writes,1)
+ def test_conflicting_reference_rejected(self):
+  r=self.approval();recover(self.store,r)
+  with self.assertRaises(RecoveryError):recover(self.store,dict(r,actor='different-operator'))
+
+ def test_replaced_inactive_factor_cannot_replay_old_recovery(self):
+  r=self.approval();recover(self.store,r);self.store.state['id']=2
+  with self.assertRaises(RecoveryError):recover(self.store,r)

workplans/KEY-WP-0036-platform-factor-recovery.md

@@ -0,0 +1,70 @@
+---
+id: KEY-WP-0036
+type: workplan
+title: "Audited platform administrator lost-factor recovery"
+domain: infotech
+repo: key-cape
+status: active
+owner: codex
+topic_slug: infotech
+created: "2026-09-13"
+updated: "2026-09-13"
+state_hub_workstream_id: "f9ba00a6-5758-5677-9a49-da1e35875f7a"
+---
+
+Implements the provider part of P04 under USER-WP-0030-T03. The existing factor
+read credential is deliberately not elevated to recovery authority. Recovery
+currently runs only as a provider-local operator procedure, not a portal API.
+
+## Implement bounded and audited provider recovery
+
+```task
+id: KEY-WP-0036-T01
+status: done
+priority: high
+state_hub_task_id: "f3287544-8412-5b1f-9d16-9f7b9e895ceb"
+```
+
+`scripts/factor_recovery.py` previews exact ownership and shared-identity scope,
+requires an operator verification attestation and matching metadata version,
+disables one factor with a row lock, requires pre-mutation audit durability,
+reads back state, records completion and reconciles retries. Reject changed or
+replaced factors and reference conflicts. Eight isolated unit tests pass.
+The installed-provider Job provider-recovery-contract-02 passed real database
+and audit persistence, disable/readback and replay assertions in an isolated
+fixture. Acceptance CI includes both suites. No real user's factor was changed.
+
+## Validate the attended owner entry point
+
+```task
+id: KEY-WP-0036-T02
+status: progress
+priority: high
+state_hub_task_id: "2d59da13-782e-5aeb-8957-69e41a15ddb1"
+```
+
+railiance-platform/scripts/keycape_factor_recovery.py derives actor from an
+OpenBao platform-admin entity, rejects root/workload substitutes, checks cluster
+identity, and emits only a private metadata receipt. Runbook:
+railiance-platform/docs/keycape-factor-recovery.md. Offline owner tests pass.
+Remaining: exercise the reviewed attended wrapper with a disposable provider
+identity and prove denied identity, preview, verified apply, audit and retry.
+Serialize owner operations until a concurrent authenticated service exists.
+
+## Integrate platform support recovery with the browser journey
+
+```task
+id: KEY-WP-0036-T03
+status: todo
+priority: high
+state_hub_task_id: "f7248d5e-6f68-5637-ada5-2527436f5f9e"
+```
+
+Implement an authenticated bounded recovery service and platform-only portal
+flow with fresh assurance, server-derived actor, CSRF, ownership-verification
+process, exact target/scope confirmation, audit/reference readback and safe
+retry. Never grant Kubernetes access to the portal. Test unauthorized/tenant
+admin denial, stale confirmation, provider failure, retry and replacement OTP
+enrollment using disposable identities. Mandatory-AAL2 access is restored only
+after possession of a replacement factor is proven. Keep P04 incomplete until
+browser success, failure and recovery paths pass; CLI tests alone do not close it.