diff --git a/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/source-scanner.json b/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/source-scanner.json new file mode 100644 index 00000000..f72d2039 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/source-scanner.json @@ -0,0 +1,5 @@ +{ + "exit_code": 0, + "stdout": "{\"blocked_blob_ids\": [], \"scanned_blobs\": 7, \"scanner_sha256\": \"88f91962aa2f93ac6ab281d553b9e125f5197bbbce38f9f2437f7299c32e5509\", \"scanner_version\": \"8.30.1\", \"scope\": \"changed-staged-blobs\", \"status\": \"clean\"}\n", + "stderr": "" +} diff --git a/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/sync_projection.py b/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/sync_projection.py new file mode 100644 index 00000000..6a28e5e7 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/sync_projection.py @@ -0,0 +1,15 @@ +from pathlib import Path +import subprocess,json,sys +base=Path(__file__).resolve().parent;owner=Path('/home/worsch/net-kingdom');label=sys.argv[1] +r=subprocess.run(['statehub','fix-consistency','--repo','net-kingdom','--json'],cwd=owner,capture_output=True,text=True) +(base/(label+'.stdout.json')).write_text(r.stdout) +(base/(label+'.stderr.txt')).write_text(r.stderr) +(base/(label+'.exit.json')).write_text(json.dumps({'exit_code':r.returncode})+'\n') +backup=base/'publication-backup' +(owner/'.custodian-brief.md').write_bytes((backup/'brief.worktree').read_bytes()) +mode,oid,_,name=(backup/'index-record.txt').read_text().split() +subprocess.run(['git','-C',str(owner),'update-index','--cacheinfo',mode,oid,name],check=True) +assert subprocess.check_output(['git','-C',str(owner),'show',':.custodian-brief.md'])==(backup/'brief.index').read_bytes() +print('Projection exit',r.returncode,'; original staged/working brief restored; exact stdout/stderr retained.') +print(r.stderr) +raise SystemExit(r.returncode) diff --git a/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/test_real_checker_adapter.py b/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/test_real_checker_adapter.py new file mode 100644 index 00000000..d02bc2b0 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/test_real_checker_adapter.py @@ -0,0 +1,107 @@ +"""Disposable adapter fixtures using exact compiled checker, no checker mocks.""" +from dataclasses import asdict,replace +import base64,copy,importlib.util,json,os +from pathlib import Path +import unittest +from unittest.mock import patch +from contextlib import chdir + +ROOT=Path(os.environ.get('NETKINGDOM_FIXTURE_ROOT', '/home/worsch/net-kingdom')) +spec=importlib.util.spec_from_file_location('owner_registration_tests',ROOT/'sso-mfa/k8s/keycape/test_keycape_registration_adapter.py') +t=importlib.util.module_from_spec(spec);spec.loader.exec_module(t) +BINARY=Path(os.environ.get('KEYCAPE_TEST_CHECKER_BINARY', '/tmp/nk-checker-artifact-next-r3/keycape')) + +class ArtifactAdapterChecks(t.RegistrationTests): + def setUp(self): + super().setUp() + self.check_patch.stop() # The real maintained subprocess checker runs. + self.binary.write_bytes(BINARY.read_bytes()) + self.binary.chmod(0o700) + desc=t.a.descriptor(t.digest(self.binary.read_bytes())) + self.desc_bytes=json.dumps(asdict(desc),sort_keys=True,separators=(',',':')).encode() + self.admitted=replace(self.admitted,pins=replace(self.admitted.pins,adapter=t.digest(self.desc_bytes))) + self.inv=replace(self.admitted,attempt_id='attempt-one',operation='run') + self.kw.update(admitted=self.admitted,check_config_digest=t.digest(self.binary.read_bytes())) + self.missing_env=False + clients=copy.deepcopy(t.f.EXISTING) + clients[1].update(displayName='Synthetic legacy approval', + redirectUris=['https://approval.example.invalid/cb'],allowedScopes=['openid'],grantTypes=['authorization_code']) + self.secret,self.raw=t.f.fixture(clients,tail='port: 8080\nprivateKeyPem: /nonexistent/synthetic-key.pem\ntokenLifetime: 15m\n') + + def kube(self,args,payload=None): + value=super().kube(args,payload) + if args[:5]==['-n','sso','get','deployment','keycape'] and not self.missing_env: + value['spec']['template']['spec']['containers'][0]['env'].append({ + 'name':'KEYCAPE_SECRETS_ENGINE_APPROVAL_CLIENT_SECRET', + 'valueFrom':{'secretKeyRef':{'name':'synthetic-legacy','key':'client-secret'}}}) + return value + + def test_real_checker_rejects_invalid_startup_config_before_cas(self): + raw=base64.b64decode(self.secret['data']['config.yaml']).decode().replace('port: 8080','port: 0') + self.secret['data']['config.yaml']=base64.b64encode(raw.encode()).decode() + with self.assertRaises(t.Refused):t.a.build(**self.kw).invoke(self.inv) + self.assertFalse(self.effects) + self.assertEqual(self.revocations,[self.inv]) + + def test_real_checker_rejects_missing_declared_secret_env_before_cas(self): + self.missing_env=True + with self.assertRaises(t.Refused):t.a.build(**self.kw).invoke(self.inv) + self.assertFalse(self.effects) + self.assertEqual(self.revocations,[self.inv]) + + def test_ambient_urls_cannot_reject_supplied_valid_config(self): + with patch.dict(os.environ,{'KEYCAPE_ACCOUNT_PORTAL_URL':'http://polluted.example.invalid', + 'KEYCAPE_BROWSER_LOGOUT_URL':'http://polluted.example.invalid/logout'}): + result=t.a.build(**self.kw).invoke(self.inv) + self.assertEqual(result.outcome,t.Outcome.SUCCESS) + self.assertEqual(len(self.effects),1) + + def test_ambient_urls_cannot_admit_supplied_invalid_config(self): + raw=base64.b64decode(self.secret['data']['config.yaml']).decode() + raw+='accountPortalURL: http://invalid.example.invalid\nbrowserLogoutURL: http://invalid.example.invalid/logout\n' + self.secret['data']['config.yaml']=base64.b64encode(raw.encode()).decode() + with patch.dict(os.environ,{'KEYCAPE_ACCOUNT_PORTAL_URL':'https://polluted.example.invalid', + 'KEYCAPE_BROWSER_LOGOUT_URL':'https://polluted.example.invalid/logout'}): + with self.assertRaises(t.Refused):t.a.build(**self.kw).invoke(self.inv) + self.assertFalse(self.effects) + self.assertEqual(self.revocations,[self.inv]) + + def test_relative_selected_real_binary_is_not_replaced_by_path_decoy(self): + decoy=Path(self.temp.name)/'decoy';decoy.mkdir() + trap=decoy/'keycape';trap.write_text('#!/bin/sh\nexit 27\n');trap.chmod(0o700) + with chdir(self.temp.name),patch.dict(os.environ,{'PATH':str(decoy)}): + result=t.a.build(**dict(self.kw,check_config=Path('keycape'))).invoke(self.inv) + self.assertEqual(result.outcome,t.Outcome.SUCCESS) + self.assertEqual(len(self.effects),1) + + def test_selected_path_replacement_after_snapshot_cannot_change_executed_checker(self): + original = t.a.helper.check_config + def replace_path(*args, **kwargs): + trap = self.binary.with_name('replacement-checker') + trap.write_text('#!/bin/sh\nexit 27\n') + trap.chmod(0o700) + trap.replace(self.binary) + return original(*args, **kwargs) + with patch.object(t.a.helper, 'check_config', side_effect=replace_path): + result = t.a.build(**self.kw).invoke(self.inv) + self.assertEqual(result.outcome, t.Outcome.SUCCESS) + self.assertEqual(len(self.effects), 1) + self.assertEqual(self.revocations, [self.inv]) + +if __name__=='__main__': + selected=[ + 'test_selected_path_replacement_after_snapshot_cannot_change_executed_checker', + 'test_actual_add_and_created_only_rollback_preserve_original_bytes', + 'test_actual_kernel_admission_dispatch_and_rollback_with_fixture_only_trust', + 'test_kernel_lost_readback_retains_uncertain_and_refuses_replay_and_undo', + 'test_kernel_external_revision_change_retains_uncertain_rollback', + 'test_stale_revision_refuses_mutation', + 'test_substituting_caller_selected_checker_under_same_descriptor_is_refused', + 'test_real_checker_rejects_invalid_startup_config_before_cas', + 'test_real_checker_rejects_missing_declared_secret_env_before_cas', + 'test_ambient_urls_cannot_reject_supplied_valid_config', + 'test_ambient_urls_cannot_admit_supplied_invalid_config', + 'test_relative_selected_real_binary_is_not_replaced_by_path_decoy'] + suite=unittest.TestSuite(ArtifactAdapterChecks(name) for name in selected) + result=unittest.TextTestRunner(verbosity=2).run(suite) + raise SystemExit(not result.wasSuccessful()) diff --git a/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/workplan-hashes.json b/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/workplan-hashes.json new file mode 100644 index 00000000..d4ed7b56 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/workplan-hashes.json @@ -0,0 +1,5 @@ +{ + "path": "workplans/NK-WP-0048-security-case-types.md", + "before": "6765312428c4bb78380d7736de1132f0d005e6059dc17008792d0b3f2999d441", + "after": "4b14db289cee2ff598484c050814efee1de743a316ac7ba7cb549efc08e73ecc" +} diff --git a/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/workplan-postimage.md b/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/workplan-postimage.md new file mode 100644 index 00000000..66a40fc9 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/workplan-postimage.md @@ -0,0 +1,871 @@ +--- +id: NK-WP-0048 +type: workplan +title: "Establish SecurityCaseTypes so that known security cases are light" +domain: infotech +repo: net-kingdom +status: active +flavor: implementation +owner: claude-code +topic_slug: netkingdom +created: "2026-10-08" +updated: "2026-10-16" +depends_on: [] +related_repos: + - ops-mason + - ops-warden + - key-cape + - railiance-platform + - informed-decision +state_hub_workstream_id: "e31f3f9d-0748-5137-b0a3-7b3ca46a66eb" +--- + +# NK-WP-0048 - Establish SecurityCaseTypes so that known security cases are light + +## Why + +The founder's direction (2026-10-08): setting up a security integration of a +kind that has been done before should be easy, meaning accept the case, then +ops-mason arranges it and ops-warden follows through. Setting up a new kind +needs ample documentation and tests and an informed-decision, because its shape +is part of the security architecture. + +Today every integration is hand-built: per-client helper scripts, per-lane +custody, founder checklists. The trigger was KEY-WP-0038 (Forgejo as an OIDC +client), whose next step was going to be yet another hand-written rollout +script. + +This plan continues the pattern work of NK-WP-0008 (pattern infospace), +NK-WP-0009 (tutorials) and NK-WP-0030 (scenario composition). It turns patterns +into **SecurityCaseTypes** that ops-mason and ops-warden can execute. Canon: +`canon/standards/security-case-types_v0.1.md` (proposed). + +Companion plans: ops-mason `MASON-WP-0010` (materialization) and ops-warden +`WARDEN-WP-0044` (follow-through). + +## Tasks + +```task +id: NK-WP-0048-T01 +status: done +priority: high +state_hub_task_id: "930ac3e9-20ec-5473-98cd-3fe8fddb730f" +``` + +### T01 — Accept the terminology and the canon + +Founder review of `canon/standards/security-case-types_v0.1.md`, including its +section 10 open questions. On acceptance the canon status goes from `proposed` +to `accepted`. + +2026-10-08: accepted by the founder (in session). Canon status `accepted`; +section 10's open questions are carried forward to the first type (T04, T05). + +```task +id: NK-WP-0048-T02 +status: done +priority: high +state_hub_task_id: "19c25d96-a55c-58f0-9168-3fdbc4955fc2" +``` + +### T02 — Inventory the de-facto types already built by hand + +Read-only. List every security integration built so far and group them by shape. +Known starting points: + +- browser OIDC relying parties on key-cape: informed-decision-approver, + coulomb-social, user-engine portal, openbao-admin, and Forgejo (pending) +- confidential service clients with `client_credentials`: secrets-engine + approval, requester and openbao clients, approval-engine operator +- workload KV read lanes on OpenBao: the railiance-platform credential lane + designs and CCRs, for example `railiance-backup-offsite-lane` +- short-lived SSH certificates through ops-warden (`warden sign`) +- tunnelled operator access (ops-bridge) + +For each: the repositories involved, the hand-built steps, attended human steps, +custody paths, verification evidence, and rollback (if any). Output +`docs/security-case-type-inventory.md`, ranking candidate types by how often the +shape recurs and how much manual work each instance costs today. + +2026-10-08: done, `docs/security-case-type-inventory.md`. 36 of 37 CCRs are +already `request_type: workload-kv-read`, a de-facto type reviewed each time as +if it were new. Browser clients: 5 hand-built plus Forgejo, each with its own +rollout script. Confidential service clients need a registration plus two or +three KV-read CCRs each. A forge read identity is needed in bulk by +ORGREF-WP-0002-T06. Recommendation: oidc-browser-relying-party first (unblocks +the forge), workload-kv-read second (largest payoff), then forge-read-identity. + +```task +id: NK-WP-0048-T03 +status: done +depends_on: [NK-WP-0048-T01] +priority: high +state_hub_task_id: "e487975b-73b1-516c-971d-da4cbbb93063" +``` + +### T03 — Schema, template and validator + +- `canon/schemas/security-case-type_v0.1.schema.json` and + `security-case_v0.1.schema.json` +- `security-case-types/_template/` with `type.yaml`, `README.md` and tests to copy +- a validator, extending `tools/security-scenario-composer/`, that checks a type, + checks a case against its type, and renders the case into the composer's + deterministic plan (still `execution.permitted: false`) + +Acceptance: the validator rejects missing sections, secret-shaped parameter +values, unpinned providers and cases against non-`implemented` types, with tests +for each. + +2026-10-08: done. `canon/schemas/security-case-type_v0.1.schema.json` and +`security-case_v0.1.schema.json`; `tools/security-case-types/security_case_validator.py` +(`validate-type`, `validate-case`, `render`, reusing the scenario composer); +`security-case-types/_template/` (a valid, runnable example type over +`net-kingdom.local-identity`, with README sections, an example case and tests that +travel with every copy). 31 tests pass, including the composer's existing suite. + +```task +id: NK-WP-0048-T04 +status: done +depends_on: [NK-WP-0048-T02, NK-WP-0048-T03] +priority: high +state_hub_task_id: "2fffe112-7d55-5680-9952-78d74f44f87f" +``` + +### T04 — Define the first type: browser OIDC relying party + +`security-case-types/oidc-browser-relying-party/`, version 1.0: + +- parameters: client id, display name, exact redirect URIs, scopes, client type + (confidential or public), MFA pin, the groups the consuming service maps, + custody path for the secret (confidential only), consuming service and its + owner +- composition: key-cape client registration (key-cape declares it as a playbook + capability), secret minting into custody (railiance-platform), LLDAP group + existence and membership (net-kingdom identity-provisioner), handover to the + consuming service +- materialization (ops-mason) and follow-through (ops-warden) sections +- verification: discovery lists the redirect URI; a test sign-in returns the + expected claims for one human; anonymous access to the custody path is denied +- rollback: remove exactly this client and its custody path +- README: threat model, MFA rationale, why redirect URIs are exact, which steps + still need an attended login and how that is minimised +- reference instances: informed-decision-approver, coulomb-social, user-engine + portal, openbao-admin + +Acceptance: complete per canon section 3, tests green, status `proposed`. + +2026-10-08: done. `security-case-types/oidc-browser-relying-party/` 1.0 (proposed): +`type.yaml` with 12 parameters (the custody reference is required only for +confidential clients), composition over key-cape's new +`key-cape.oidc-client-registration` declaration (contract PASS, `maturity: +planned`), materialization and follow-through, verification and rollback, all +conditional on client type where it matters. One attended step, +`register-client`, justified by the live registry being a Kubernetes Secret. +README with threat model, alternatives, an honest fit table (portal and vergabe +fit; Forgejo is the first confidential case; openbao-admin, coulomb.social and +informed-decision-approver do not fit 1.0), and known limits. Examples: Forgejo +(checked against the accepted request) and a public portal. The validator gained +`when` / `required_when` conditions and provider discovery. 43 tests pass. +ops-mason's and ops-warden's sections are drafted in the type. Their owners +review them in MASON-WP-0010-T01 and WARDEN-WP-0044-T01 before acknowledgment +(T05). + +```task +id: NK-WP-0048-T05 +status: done +depends_on: [NK-WP-0048-T04, MASON-WP-0010-T01, WARDEN-WP-0044-T01] +priority: high +state_hub_task_id: "3d987d3d-0797-538b-86e4-51185a327ae2" +``` + +### T05 — Acknowledge the type through informed-decision + +Prepare the Decision Memo from the type's README and `type.yaml` and submit it +through informed-decision's production intake. On acknowledgment the type +moves to `acknowledged`. + +2026-10-08: Decision Memo prepared. Hub decision `e418263e-a71f-4f01-a970-c87c9e187c09`. +`security-case-types/oidc-browser-relying-party/decision/`: `packet.md` (question, +one act, what acknowledging does not do, the shape, alternatives, attended +steps, limits, evidence), `build_intake.py` and the built `intake.json` (memo +class `decision-record`, review group `net-kingdom-admins`, resolves the hub +decision on answer). The envelope passes informed-decision's own +`validate_submission`. Queuing it is an attended operator step, a write inside +the production dispatcher pod: + + kubectl -n state-hub exec -i deploy/state-hub-memo-dispatch -- \ + python -m api.services.memo_dispatch --queue /data/private enqueue --envelope /dev/stdin \ + < security-case-types/oidc-browser-relying-party/decision/intake.json + +The answer then appears for review in informed-decision. + +2026-10-08: queued by the founder (`pending`, `queued`) after one refusal: the +first envelope's idempotency key was not State Hub's stable +`state-hub..` form, and the builder now validates against both +sides. The dispatcher delivered it to informed-decision (ledger 7 → 8 +delivered). T05 waits for the founder's answer in informed-decision, which +resolves hub decision `e418263e`. + +2026-10-08: ops-mason's review (MASON-WP-0010-T01) folded into the type, still +1.0 `proposed`: `secret_ref` is no longer a parameter, the type derives +`platform/workloads/keycape/{client_id}`; policies are +`workload-kv-read-keycape-{client_id}-eso` and +`workload-kv-read-keycape-{client_id}-{consuming_service}`, each with a templated +body (`read` on that one data path); new `openbao_role` +`external-secrets-keycape-{client_id}` (Kubernetes auth, ESO service account, +15m, no default policy); catalog entry `{consuming_service}-keycape-client`; +the two rollback steps on the KV path collapse into one metadata delete. The +schema and validator now require a `body` on policy and role items. 105 tests +pass. Two consequences: the delivered memo describes the type before these +changes (a case-set custody reference, two differently named policies, no ESO +role), so the founder answers on a superseded shape unless the memo is +re-sent; and Forgejo's custody path moves from the accepted request's proposed +`platform/workloads/forgejo/netkingdom-oidc-client` to +`platform/workloads/keycape/forgejo`, which key-cape (KEY-WP-0038) and +railiance-apps (RAPPS-WP-0016-T02) need to adopt in T06. ops-warden's review +(WARDEN-WP-0044-T01) is still in progress. + +2026-10-08: revised memo `nk-wp-0048-t05-oidc-browser-relying-party-r2` built +(`packet.md`, `intake.json`, packet `sha256:af6435e2…`); it states that it +supersedes the first memo and passes informed-decision's and the dispatcher's +validation. It needs a new memo id because the dispatcher refuses new content +under a delivered idempotency key (`idempotency_conflict`). Queuing it is the +same attended step as above. Both memos serve hub decision `e418263e`; the +first answer resolves it, so the first memo must not stay open: an accept on it +would acknowledge the superseded shape, and a decline would resolve the +decision against r2. Close it with `return` (a reviewer verb that requests a +new revision and does not resolve the hub decision; only accept and decline +do), commenting that r2 supersedes it, then answer r2. `withdraw` exists in +the vocabulary but is not offered to reviewers, and lifecycle `revise` only +applies to lifecycle-registered memos, not intake-delivered ones. + +2026-10-09: the founder reports r2 enqueued and the first memo returned. At +close, hub decision `e418263e` still showed `open` (not verified whether r2 +was answered or the answer is still in transit; the dispatcher ledger was not +read). ops-warden's review (WARDEN-WP-0044-T01, ops-warden +`history/2026-10-08-oidc-rp-follow-through-review.md`, d470f2e) arrived at +21:25, before r2 was built, and is not in r2. Required before WARDEN-WP-0044-T02: +R1 a per-step `lane` (missing: identity-provisioner caller credential, +railiance-platform broker grant), R2 canon meaning of `warden_executes: true` +(caller proxy, caller identity, no standing ops-warden token; ADR-0002), R3 a +per-step `value_handling`. Refinements R4 (one type-level catalog lane instead +of a per-case entry, which conflicts with ops-mason's item 5), R5 (ESO Ready gate +before bind-env), R6 (exact accepted memberships), R7 (mfa-pinned and +secret-delivered checks). Next: if r2 was accepted, fold R1-R3 and R5-R7 in as +a follow-up and settle R4 between ops-warden and ops-mason before T06; if not, +hold r2, fold them in and build r3. + key-cape and railiance-apps were told about Forgejo's new custody +path. + +2026-10-09: r2 was not answered (hub decision `e418263e` open; dispatcher 9 +delivered), so r2 is held and r3 replaces it. ops-warden withdrew R4 for R4′ +(message a66a9204): one catalog entry per case with its body templated in the +type. Folded in: + +- Canon section 3: a narrow meaning of `warden_executes: true` (caller proxy, + caller identity, no standing credential; ops-warden ADR-0002), plus per-step + `lane`, `value_handling` and an optional `ready_check`. A catalog entry body is + now required, like policy and role bodies. +- Schema and validator enforce all of these. +- The type: lanes `identity-provisioner-groups` and + `keycape-client-secret-generate` (both new) and `key-cape-oidc-login`; + `mint-secret` is `generated_in_custody`; a Ready gate on `deliver-secret`; + `ensure-groups` adds exactly the accepted case's memberships; new checks + `secret-delivered` and `mfa-pinned`; the catalog-entry body follows R4′. +- 110 tests pass. + +Memo r3 (`nk-wp-0048-t05-oidc-browser-relying-party-r3`, packet +`sha256:726dfc01…`) passes informed-decision's and the dispatcher's validation. +The founder queues it with the same command, returns r2 with a comment that r3 +supersedes it, then answers r3. + +Prerequisites for T06, not for T05: +- the two new lanes, each through its owner; +- a founder-approved rollout of an identity-provisioner image containing + 630ebfd (the deployed one is main-6c4fcaf from 2026-09-24, without + `/v1/groups/ensure`). + +ops-mason's request to create LLDAP group `fluid-telegram-operators` (message +259ec26d) was answered: not possible from here by that path; the founder creates +it in the LLDAP admin UI. + +2026-10-09: ops-warden finished its side of WARDEN-WP-0044-T01 (message +`58d33f40`, commit 0e7ceb8): the revised type conforms to `warden case +check-type`, with no further changes asked of the type. This acknowledges that +review. Its later addendum R8 (message `af274783`, ops-warden +`history/2026-10-08-oidc-rp-follow-through-review.md`) showed that lane +`identity-provisioner-groups` cannot exist as declared: the provisioner's only +HTTP credential is user-engine's runtime token, and its NetworkPolicy admits +only user-engine pods, so a caller-proxy lane would act as user-engine. Taken +option 1 (ops-warden's recommendation): `ensure-groups` and its rollback are +attended, in the `register-client` session on `key-cape-oidc-login`, run inside +the provisioner pod; no token is read. Option 2 (an operator credential of its +own on the provisioner) stays open for a later version. `warden case +check-type` now reports only `keycape-client-secret-generate` +(railiance-platform) as missing. 89 tests pass. + +Hub decision `e418263e` was still open, so memo r4 +(`nk-wp-0048-t05-oidc-browser-relying-party-r4`, packet `sha256:452ea421…`) +replaces r3; it passes informed-decision's and the dispatcher's validation. The +founder queues it with the same command, returns r2/r3 if still open with a +comment that r4 supersedes them, then answers r4. + +Prerequisites for T06 are now: lane `keycape-client-secret-generate` +(railiance-platform), and a founder-approved rollout of an identity-provisioner +image containing 630ebfd. + +2026-10-09: done. The founder queued r4 (`pending`, `queued`), returned r3 as +superseded and accepted r4 in informed-decision. informed-decision's hub +write-back is off in production (`INFD_HUB_WRITEBACK` unset), so net-kingdom +resolved hub decision `e418263e` from the founder's report, naming memo r4 in +the rationale. The type is `acknowledged`; it becomes `implemented` with T06. + +```task +id: NK-WP-0048-T06 +status: wait +depends_on: [NK-WP-0048-T05, NK-WP-0048-T08, NK-WP-0048-T09, NK-WP-0048-T12, NK-WP-0048-T13, NK-WP-0048-T14, NK-WP-0048-T15, NK-WP-0048-T16, NK-WP-0048-T17, NK-WP-0048-T18, MASON-WP-0010-T03, WARDEN-WP-0044-T03] +blocking_reason: "Type1.2 architecture has a confirmed source-only founder acknowledgment; canonical1.1 remains unchanged/default and the1.2 reference case stays proposed. Needs explicit version selection/reference-source integration without normal admission bypass, closed owner adapters plus global/native receipt verifier (WARDEN-WP-0044-T03), independently verified six-object structural materialization and CCR42 claims/created-only provenance, exact broker/case/source reconciliation, then separately authorized native mint/delivery and attended registry/consumer verification. Source acknowledgment, a bundle or local tests do not prove implemented/materialized/live status." +priority: high +state_hub_task_id: "d9b26bcf-a7c6-59a2-92a0-f913d76a203c" +``` + +### T06 — First KnownSecurityCase: Forgejo + +Write `security-cases/forgejo-netkingdom-oidc.yaml` from the parameters the +founder accepted in KEY-WP-0038-T01 (`key-cape/docs/forgejo-oidc-client-request.yaml`), +so that acceptance carries over as the case acceptance. ops-mason materializes it; +ops-warden follows through and verifies it. The type becomes `implemented` once +this reference case is live. + +This replaces the hand-built path proposed for KEY-WP-0038-T02. + +2026-10-09: case written, `security-cases/forgejo-netkingdom-oidc.yaml`, status +`accepted`: the example's parameters plus the founder's 2026-10-08 acceptance of +the key-cape request (owner and founder), with the custody-path derivation +covered by the type's acknowledgment (memo r4). It validates and renders (plan +`execution.permitted: false`; three attended steps on `key-cape-oidc-login`); +it needs `--allow-unimplemented` until it is live, since it is the case that +makes the type `implemented`. A test keeps it equal to the type's example. +Still open before it runs: ops-mason renders and builds it (MASON-WP-0010-T02, +T03, unblocked by T05), ops-warden plans and runs it (WARDEN-WP-0044-T02, T03), +railiance-platform's `keycape-client-secret-generate` lane (asked by ops-mason +`789feb39` and ops-warden `d9f889b1`, both unread), and a founder-approved +identity-provisioner rollout containing 630ebfd. + +2026-10-09: identity-provisioner rolled out, founder-approved in session: +`main-630ebfd` (`sha256:bafb4860…`, built by the image workflow) replaces +`main-6c4fcaf` (`sha256:ffacd5d7…`; only change: 630ebfd, additive). Applied +with `kubectl set image` on the Deployment only (generation 17 → 18, 1/1 Ready); +the file's Service, NetworkPolicies and Ingress were not reapplied. Verified in +the pod: `ensure_service_groups` and `revoke_service_groups` present, `/readyz` +200; no token used, no group changed. Rollback: `kubectl -n sso set image +deploy/identity-provisioner provisioner=forgejo.coulomb.social/coulomb/identity-provisioner@sha256:ffacd5d790a5b7800e0535f3aa95b7de52c7a68e4f575b9192c3e1622fa672c2`. +The pin in `sso-mfa/k8s/identity-provisioner/deployment.yaml` follows. + +2026-10-09: ops-warden closed WARDEN-WP-0044-T01 and shipped `warden case plan` +(T02, message `06303d1d`); the accepted Forgejo case is refused until ops-mason +materializes it, and a materialized copy plans cleanly once +`keycape-client-secret-generate` exists. Its observation that `bind-env` carries +a literal `KEYCAPE__CLIENT_SECRET`: the validator now renders a +`{param:env}` format spec (upper case, `-` as `_`, as `keycape-client.py` +derives the name) and rejects unknown specs, and follow-through and rollback +text is now checked for unknown placeholders too (patch-level, canon section 6). +The type keeps the literal until ops-warden's planner (`text.format`) renders the +spec as well; asked of ops-warden. 53 tests pass. + +2026-10-10: ops-warden rendered `{param:env}` in `warden case plan` +(23ba584, message `c1df14b7`; other specs and `!conversion` refused). The type +adopts it as a 1.0 patch (canon section 6): `bind-env` and its rollback now read +`KEYCAPE_{client_id:env}_CLIENT_SECRET`. ops-mason reports the Forgejo case +rendered and built (message `1ebddae9`; CCR drafts handed to railiance-platform), +so T06 now waits on the platform apply, the mint-secret lane and the attended +register-client session. + +2026-10-10: key-cape adopted the type's custody path (key-cape ab5cccd, message +`88a4559c`): `docs/forgejo-oidc-client-request.yaml` and KEY-WP-0038-T02 name +`platform/workloads/keycape/forgejo`, field `CLIENT_SECRET`, delivered as +`sso/keycape-forgejo-client` through `external-secrets-keycape-forgejo`, and the +`forgejo/netkingdom-oidc-client` proposal is withdrawn. This matches the type and +the Forgejo example; no change needed here. + +2026-10-10: key-cape's type-composition review (key-cape 6038483, message +`8e5ed5f6`) found the composition sound with two blocking findings, taken as a +1.0 patch (stricter validation and documentation, canon section 6): +R1, redirect URIs now refuse `?` and `#` in the type pattern and in +`keycape-client.py`'s `REDIRECT` (KeyCape refuses `?` at startup and the single +replica would stop); R2, `allowed_scopes` always includes `groups`, the renderer +refuses `access_groups` without it, and `access_groups` now says the consumer +enforces the gate because KeyCape has none; R3, `mfa_required` "false" means the +per-user privacyIDEA policy; R4, register-client's justification records the +restart for every client, the quiet window, and that the rendered registry must +pass KeyCape's client validation before it. The accepted Forgejo case and both +examples still validate; 116 tests pass. +Open, needs a decision: R2's verification step that the consumer enforces the +group gate is a new verification step, hence 1.1 (minor), and the validator pins +cases to the exact type version, so the accepted 1.0 Forgejo case would need +re-pinning. Also open with key-cape: a KeyCape entry point that validates a +rendered client registry without starting the server; until then +`keycape-client.py` mirrors the redirect rule. + +2026-10-10: founder go for 1.1, so no 1.0 run overlaps. The type adds +verification `consumer-enforces-groups`, and hand-over now requires the consumer +to enforce `access_groups` (minor, canon section 6; risk class unchanged, no new +acknowledgment; history in the type README). The accepted Forgejo case and both +examples are re-pinned to `oidc-browser-relying-party@1.1` before any apply; the +acknowledgment intake keeps `@1.0` as the record of what was acknowledged. +ops-mason's build and ops-warden's case-ref digest predate this and need a +re-render. + +2026-10-10: ops-mason re-rendered the Forgejo case from `@1.1` (message +`22c6bba6`; plan digest `af1f4f81…`, no collisions) and rebuilt the bundle. The +rendered OpenBao objects are identical to 1.0, so the already applied +CCR-2026-0041 stands; 1.1 changes only consumer-side steps. key-cape shipped +`keycape check-config` (c528daa, message `a7b0aeff`): KeyCape's startup checks +without secret values, including the new duplicate-clientId rule. +`keycape-client.py --check-config ` now runs it on the live and the +proposed registry, fed on stdin, and apply refuses a proposed registry that +fails; register-client's entry point names the option (1.1 patch). The deployed +image predates c528daa, so the binary is built from key-cape source for now. +Before the next KeyCape rollout (Forgejo register-client or the CCR-2026-0018 +window), the attended operator runs `inspect --check-config` so the live +registry is checked too. + +```task +id: NK-WP-0048-T07 +status: wait +depends_on: [NK-WP-0048-T06] +blocking_reason: "The next types follow once the first one is proven end to end." +priority: medium +state_hub_task_id: "c3c9c3ae-4f23-5ba7-9e6f-e63bcf29b9f7" +``` + +### T07 — Queue the next types + +From the T02 ranking, open the next type drafts (likely the confidential service +client and the workload KV read lane), each to be acknowledged on its own. +Record which existing hand-built instances get adopted as cases, without +rewriting them in place. + +```task +id: NK-WP-0048-T08 +status: done +priority: high +state_hub_task_id: "8e519082-dc8b-5e46-b660-03d03e55dcf6" +``` + +### T08 — identity-provisioner: ensure service-access groups + +The type's `ensure-groups` step calls `identity-provisioner groups ensure --groups + --members `. The provisioner already has idempotent +`_ensure_group` and `_add_group`, but exposes only tenant-scoped operations. Add +the general operation (create if absent, add the listed memberships, never +remove others), its rollback (remove only this case's memberships), tests, and +the ops-warden route for its service identity. + +2026-10-08: done. `identity-provisioner/provisioner.py` gained +`ensure_service_groups` (`POST /v1/groups/ensure`) and `revoke_service_groups` +(`POST /v1/groups/revoke`), behind the existing service-token check. Names +matching `tenant*`, `netkingdom-*` and `lldap_*` are reserved; members must +already exist; everything is validated before any change; memberships are read +back before success; nothing else is ever removed; groups are deleted only when +asked and empty. 7 new tests (39 in the provisioner suite). The type's +`ensure-groups` step and its rollback now name these endpoints. The ops-warden +route for the provisioner's service token belongs to WARDEN-WP-0044-T01. + +```task +id: NK-WP-0048-T09 +status: done +priority: high +state_hub_task_id: "0ee12dfc-8260-5038-bd39-cc7fb4113be0" +``` + +### T09 — The `keycape-client` entry point and its declaration + +Planned first as key-cape KEY-WP-0038-T04. Moved here because the machinery a +live registration needs already lives in net-kingdom, where KeyCape is deployed +(`sso-mfa/k8s/keycape/`): the contained kubectl, the cluster-identity pin and the +issuer pin. A copy in key-cape would drift. + +2026-10-08: done. `sso-mfa/k8s/keycape/keycape-client.py` derives the KeyCape +client record from a case (`client-from-case`). It adds or removes exactly one +client with the existing safeguards (byte-preserving merge, exact match or +refuse, shared-reference refusal, compare-and-swap, read-back, contained errors), +requires the Deployment env binding first for confidential clients, and requires +the case acceptance record for `apply` instead of a per-client founder decision +id. 15 tests, including a regression showing that the vergabe case reproduces +its hand-built registration exactly; all 71 helper tests pass. Declaration +`capabilities/playbooks/net-kingdom.keycape-client-registration.yaml` (contract +PASS, `maturity: implemented`) replaces key-cape's draft. The type now pins it, +and for confidential clients adds `deliver-secret` (delivery from custody into +the sso namespace) and `bind-env` (Deployment env), attended in the same session +as `register-client`. The per-client helpers stay until their clients are +adopted as cases. + +2026-10-10 — NetKingdom owner review of the execution seam, source only: +ops-warden/docs/security-case-execution-proposal.md SHA256 78ab1439e4c5beb205aeccf15b1b2e706511be55ab195c5a3aa682f326c7c77c +was reviewed against INTENT, SecurityCase canon, acknowledged type1.1, current +case and owner source. docs/security-case-execution-owner-review.md supports +the closed adapter/receipt direction with explicit versioned schema, structural +materialization provenance, unchanged caller/session boundaries, safe CAS +reconciliation and exact-created-object rollback requirements. This is owner +review, not approval. Supporting file digests are in the adjacent sources JSON. + +Current source exposes a circular wait: ops-mason T03 waits on mint while Warden +and the broker refuse a non-materialized case. Structural acceptance therefore +needs its own native pinned receipt before mint, not a successful mint before +materialized. The platform broker additionally pins type1.0 while Forgejo is +accepted at1.1; platform-owner alignment and tests are required, not an unchecked +version bypass. T06 remains wait. No case/type lifecycle status, grant, authority +or live runtime changed by this review. A proposed informed-decision would +adopt the source contract only; native materialization and execution acceptance +stay distinct. + +Final proposal re-review: staged Warden proposal SHA256 78ab1439e4c5beb205aeccf15b1b2e706511be55ab195c5a3aa682f326c7c77c +adds only the explicitly local reducer appendix, not an accepted wire protocol +or executor. The exact-final review and sources JSON supersede the earlier +1035f049... proposal review. Platform staged the exact type1.1 broker correction +with ten fake-backend subprocess checks (script SHA256 +9576cd211cfc06526ec4cc9bd0a5b06f0f59287a68931bb2d6e3592a951ddc7d). +The next source work is the versioned closed adapter/receipt/materialization +contract, not another relaxation of the broker's status gate. Native work still +requires installed broker provenance, structural reader receipts, exact new +materialized-case bytes, caller grant/session revocation and verified execution. +No installed/native acceptance is inferred from that staged correction. + +2026-10-11 — new source-only type1.2 architecture acknowledgment: +Confirmed native founder ACCEPT of infd-20261011-forgejo-type12-source-r1@1 +at06:47:39Z, approval8e9332c8-754a-420c-b8b7-445cb4fe6765. Exact frozen packet +0b377d647ca748cd03c92f9792396412048055d4bde9f7bb06ff6d413f7acd68 and submission +585cee731de30d2b37883a57abfd9eb32f023ee694e8d911989109d5caf80569; native receipt +retained in docs/evidence/2026-10-11-forgejo-type12-source-acknowledgment/. +The complete acknowledged1.2 source package is isolated under +security-case-type-revisions/1.2/oidc-browser-relying-party; default loader and +current accepted1.1 case remain untouched because load_type is single-version. +The new proposed1.2 case is NOT accepted/materialized/live and has no acceptance +fields. Actual reference-migration design approval is a separate Apps owner +receipt; no1.1 approval is inherited. Source-recording review/test proof and exact +next gates are in docs/forgejo-type12-source-acknowledgment-followthrough.md. +T06 remains wait and T07 remains its existing dependent task; type is NOT +implemented and no first-case/native grant or broader workplan finish is asserted. + +2026-10-11 — bounded explicit version selection source preparation for T06: +The validator now offers optional exact version_roots/--version-root VERSION=PATH +selection with duplicate/mismatch/traversal/symlink refusal and no fallback. The +default1.1 source and historicalacceptedcase hashes/ordinaryrefusal/preparation +render stayunchanged; actual1.1 is acknowledged, not implemented. Acknowledged1.2 +ordinaryadmission also refuses; optionalpreparation remains nonexecuting.43local +validatorchecks pass and original1.1 CLIrenderbytes match exactly. Makerproof: +docs/evidence/2026-10-11-explicit-type-version-selection/source-integration-validation.json. +FormalNetownerreview/reference-source integration and all existing native adapter, +provenance/broker/materialization/attendedproof gates remain; T06 stayswait, no +new residual or firstcase/typeimplementation/native grant is inferred. + + +2026-10-16 — isolated source preparation for T06 (canonical integration pending): +Added `check-evidence` to the existing validator CLI. It reuses exact version +selection and ordinary type/case/render admission, checks actual Mason structural +request source/inventory/projection pins and reads actual Warden journal rows +read-only. Successful rows require exact type-declared descriptors and attempt/ +revocation fingerprints. Existing acknowledged types without machine descriptors +remain plan-only. The compatibility result never claims authenticated native +completion: Request and journal fingerprints are not portable native evidence. +Owner authentication/status binding and actual native gates remain unresolved; +T06/T07 statuses and both type/case lifecycles remain unchanged. No dispatch, +credential access or authority expansion is introduced. See tool README for the +existing owner protocol boundaries and source integration tests. + +Concrete next Warden owner integration: current `Executor.admit` refuses every +case parameter whose sensitivity is not public/operational, so browser +`redirect_uris`, `allowed_scopes`, `client_type`, `mfa_required` (and 1.2 +`consumer_namespace`) fail after otherwise prepared admission. The sensitive +classification is a security-change review level, not secret content. The +Warden owner must validate through NetKingdom's existing exact-version case/type +validator, then project only descriptor-declared safe target metadata; do not +remove the credential/value refusal or copy complete parameters into journals. +Actual owner descriptor acknowledgment and independent evidence admission remain +prerequisites, with normal admission still refusing unimplemented types. + + +## T10 — Source compatibility checker for Mason and Warden evidence + +```task +id: NK-WP-0048-T10 +status: done +priority: high +state_hub_task_id: "4b06ca97-c1e4-5680-bf22-ab75ab1646a4" +``` + +Completed 2026-10-16 in published source `c31d21ba`. The read-only +`check-evidence` command checks the actual Mason structural request and Warden +journal source protocols against an exact version/render. Independent source +review passed 103 tests at that checkpoint. It always reports native completion +false and execution prohibited. Fixtures and local journal fingerprints do not +provide authenticated owner/native evidence. This records completed source scope +previously described under T06; it does not duplicate native acceptance credit. + +## T11 — Validate the optional machine-descriptor pin contract + +```task +id: NK-WP-0048-T11 +status: done +priority: high +state_hub_task_id: "ef2a0119-f59e-5e45-9519-115da9df7b85" +``` + +Completed 2026-10-16. Reproduced the JSON schema rejecting a valid +`execution_descriptors` map while the Python validator accepted malformed maps. +The schema now supports the existing Warden optional exact-byte pin map; semantic +validation binds keys to declared steps, bounds the map and checks digest shape. +The direct evidence API refuses malformed maps before accessing them. Twenty-one +new checks and all 127 affected source checks passed. No current type/case bytes, +status, owner descriptors or native acceptance changed. Source evidence and +publication receipt: Custodian +`docs/changes/netkingdom-target-2026-10-16/browser-acceptance-next/`. + +## T12 — Prepare NetKingdom-owned attended step adapters + +```task +id: NK-WP-0048-T12 +status: todo +priority: high +state_hub_task_id: "7ce4eee3-cd47-55a9-bfa5-4f1835d4d6fd" +``` + +Owner: NetKingdom. Retained scope: prepare bounded attended Deployment env, +config verification and restart/readiness steps around the existing helpers. +Next action: identify exact current entry points and their revision/readback +contracts, then implement fixed source-pinned callbacks or receipt-only attended +steps. Completion requires positive/refusal checks, exact target/source pins and +uncertainty behavior without acquiring user-engine runtime identity or exposing +registry/client secrets. Existing caller/session, cluster/issuer, CAS/readback +and status-only output boundaries remain mandatory. + +2026-10-16 task refinement: the original broad adapter scope is split into T13 +(completed registration source candidate), T14 (group creation provenance), T15 +(durable registration recovery) and T16 (descriptor/artifact selection and owner +review). This preserves T12's task ID and Hub UUID while giving each remaining +outcome an owner and completion gate. T06 now names every source dependency; +its native/reference lifecycle acceptance scope remains unchanged. Completion +of any source task supplies no Warden, Platform or Apps assent and no native +execution authority. + +## T13 — Prepare the fixed registration adapter source candidate + +```task +id: NK-WP-0048-T13 +status: done +priority: high +state_hub_task_id: "b264c10b-565c-5d29-b788-03cf5fbd2883" +``` + +Completed 2026-10-16, source implementation and synthetic fixture proof only. +Added an injection-only adapter around actual `keycape-client.run`: fixed +case/principal/human/lane/source/acceptance binding, existing CAS/readback, +metadata-only owner evidence and per-attempt parent-session revocation boundary. +The descriptor source composition pins the exact checker executable digest +alongside adapter/helper source. Creation-only rollback authenticates its +creation receipt and refuses changed postcreation UID/resourceVersion; +pre-existing clients confer no rollback authority. Missing readback, evidence, +revocation or process-local creation provenance cannot become retry-safe success. + +Evidence: Custodian `docs/changes/netkingdom-target-2026-10-16/browser-adapter-next/` +R3 candidate, explicit patch and postimages, retained R1 rejected custody finding +and R2 revision, and 36 passing source checks (16 new, 20 existing). Three checks +use actual Warden kernel admission/dispatch/rollback with published Warden +`d97652a3bb5fa733b063d71a1b74068efcccf572` producer pins unchanged. Owner trust, +descriptor/type/case and Kubernetes objects remain synthetic fixtures. This +records a completed source deliverable, not independent review, publication, +type acknowledgment, exercised procedure, identity admission or native browser +acceptance. Independent root source review: Custodian browser-adapter-next/coordinator-review-r3.json. Source integration and normal publication/synchronization receipts: Custodian browser-adapter-next/canonical-integration.json. These receipts supply no other-owner assent or native authority. + +Source publication: `0775c2bfa41af236301155bb5a69bbf088bdf823`, independently +reviewed by the Custodian coordinator before integration. The required initial +projection and ordinary UUID writeback both completed with zero failures and +zero automation errors. Generated-only brief commits were refused by the normal +hook; exact pre-existing staged/working brief bytes were restored. The separate +canonical integration receipt retains those refusals and projection evidence. +Native admission and all other-owner acceptance remain open. + +## T14 — Preserve created-only provenance for group ensure and revoke + +```task +id: NK-WP-0048-T14 +status: todo +priority: high +state_hub_task_id: "fce95440-a25b-5f27-8ef3-a3d182099113" +``` + +Owner: NetKingdom. Next action: extend or wrap the existing identity-provisioner +ensure/revoke contract with independently retained prestate and exact newly +created groups/memberships before exposing reversible owner callbacks. Current +ensure returns no membership-creation inventory; current revoke may remove a +membership that predated this case. Completion requires actual helper-backed +positive/refusal fixtures, all validation before mutation, per-attempt evidence +and revocation, readback, ambiguity handling, and rollback that preserves every +pre-existing/shared group and membership. No existing endpoint or service token +is treated as new authorization. + +## T15 — Define authenticated recovery for lost registration process state + +```task +id: NK-WP-0048-T15 +status: todo +priority: high +state_hub_task_id: "1b825705-4848-5edc-832f-b1622cd9bdb7" +``` + +Owner: NetKingdom for registration state, with Warden owning kernel evidence +admission. Next action: select the exact existing owner evidence holder/readback +contract and define authenticated lookup for the original creation attempt. +Completion requires restart/lost-receipt fixtures that bind case/source/principal, +original created-only provenance and current native revision before any undo; +uncertain outcomes remain non-replayable. The T13 adapter deliberately refuses +post-restart rollback and installs no reconciliation callback. A new broker or +unverified JSON receipt is not a recovery contract. Actual evidence-holder +acceptance and attended recovery remain separately authorized native work. + +## T16 — Select and review the actual machine descriptor composition + +```task +id: NK-WP-0048-T16 +status: todo +priority: high +state_hub_task_id: "200bf500-5084-5ae7-be26-3033fe1f296e" +``` + +Owner: NetKingdom for descriptor/type source and exact checker artifact; Warden +independently owns global provenance/admission. Next action: obtain review of the +exact checker source/artifact proposed by T17 and identify native owner evidence/ +session callback contracts, then assemble the complete step descriptor set and +a proposed explicit type version selection. T13's historical descriptor retains +its synthetic checker; T17 supplies an artifact-bound proposal, not admission. +Completion requires separate owner source review of exact machine descriptors, +artifact/source/config pins, normal type/version admission and independent +evidence-verification boundaries. Warden d97652a refreshes producer validator +pins for NetKingdom 015694ee; it does not attest T13's new adapter. Pinning any +machine descriptor into an acknowledged type is a separately reviewed source +change. Platform owns structural/mint/broker/delivery evidence; Apps owns actual +consumer login/group enforcement. Neither their assent nor native acceptance is +supplied by this task. + +2026-10-16 source checkpoint: T19 below extracts the completed per-attempt +executable snapshot implementation and reviewed fixture proof. T16 retains +native executable storage/custody, exact owner evidence/session callbacks, +complete descriptor composition and ordinary type/version admission. The +successor descriptor/source pins are proposed only; the earlier descriptor +history is preserved. T14 group provenance and T15 durable restart/recovery +remain independent open requirements. No native acceptance is counted here. + +## T17 — Build and exercise the exact offline config-checker artifact + +```task +id: NK-WP-0048-T17 +status: done +priority: high +state_hub_task_id: "8c3e4baa-a218-5996-af9e-fe1153a46dac" +``` + +Completed 2026-10-16, isolated artifact/source fixture proof only. Built the +existing `key-cape/src/cmd/keycape` command from published clean source revision +`3c71226a1a7b8a2bdb710e9c497e1a1f7f8a3987`, source tree +`f0010b9cab17dbc3a787628a29657d98b900320f`, with cached Go 1.23.0, +CGO disabled, Linux/amd64 v1, trimmed paths, disabled VCS stamping and offline +read-only module selection. The full retained closure pins 108 source files, +9876 toolchain files and ten used module source directories. Three isolated +build locations produced identical 13633409-byte artifacts: +`sha256:be37738fe1c08259fb4f9ca0daf0f9301bfffc237e5bb38435525dfbf11ccff5`. + +Eight disposable checks use the actual executable and maintained helper/kernel, +with complete synthetic configs and a minimal environment: add/created-only +removal, normal pinned producer admission, rollback, lost-readback uncertainty, +stale revisions, substituted checker refusal, invalid startup config and missing +secret-env refusal. No checker stub or producer-pin override remains. The +artifact-bound descriptor proposal is +`sha256:76b1153d188062ece53fcd5e8d35290fdbfd02a69dd79d4b001fc42781afcdc3`. +Evidence: Custodian +`docs/changes/netkingdom-target-2026-10-16/browser-checker-artifact-next/`. +The first two build attempts succeeded but their closure collection failed; +both failures are retained, as is refusal of the older incomplete synthetic +fixture. No production config, credentials or cluster mutation was used. + +This extracts completed build/fixture scope from T16 without claiming KeyCape +owner endorsement, native checker selection/storage custody, acknowledged type +integration, native evidence/session callbacks or attended browser acceptance. +T16 retains exact owner review and composition/admission scope. The attended +registration procedure remains unexercised. + +## T18 — Make checker input and executable selection deterministic + +```task +id: NK-WP-0048-T18 +status: done +priority: high +state_hub_task_id: "9c45da71-fb4c-5f20-9065-90f448f37c64" +``` + +Completed 2026-10-16, isolated source correction and real-artifact fixture proof +only; independent integration/publication remain separately recorded. KeyCape +source review identified non-secret `KEYCAPE_ACCOUNT_PORTAL_URL` and +`KEYCAPE_BROWSER_LOGOUT_URL` overrides in config.Load. The original helper +inherited ambient environment, allowing valid supplied configuration to be +rejected or invalid supplied URL configuration to be admitted. A bare relative +checker filename could also be replaced by PATH lookup after the adapter had +hashed the selected local file. All three counterexamples are retained against +the unchanged predecessor. + +The existing helper now resolves the selected checker file to an absolute path +and passes an explicit empty subprocess environment. Three opt-in real-checker +regressions verify both URL-verdict directions and relative-file selection under +a non-checker PATH decoy. All 39 affected checks pass with the exact supplied +artifact, as do 11 real-artifact helper/kernel scenarios. Source evidence: +Custodian `docs/changes/netkingdom-target-2026-10-16/browser-checker-environment-r2/`. +Historical descriptors remain retained; the helper correction changes successor +source-composition pins and requires their separate review/admission. An absolute +path is not immutable custody and does not eliminate file replacement races; +T16 retains native executable storage/custody and callback/type admission. +No production configuration, credentials or cluster mutation was used. + +Source integration and publication: `916ba4b33e03179274d03b78bd898a4843e8653f`, following independent +Custodian source review in browser-checker-environment-r2/coordinator-review.json. +The ordinary protected projection pass assigned T17/T18 UUIDs with zero failures +and zero automation errors; generated-only brief commit refusal is retained. +The exact prior staged/working brief bytes were restored. Final source/publication/ +projection evidence is retained in Custodian browser-checker-environment-r2/ +canonical-integration.json. These source receipts do not admit native executable +custody, owner callbacks, acknowledged type changes or browser identities. + +## T19 — Bind checker execution to verified immutable attempt bytes + +```task +id: NK-WP-0048-T19 +status: done +priority: high +``` + +Completed 2026-10-16, independently reviewed source implementation and synthetic +fixture proof only. The registration adapter now opens a bounded regular checker +artifact without following its terminal symlink, verifies the reviewed digest, +and snapshots the exact bytes into a write/grow/shrink/seal-removal-sealed Linux +memfd before helper access. Live and proposed config checks execute that same fd +through procfs with the existing empty environment; the fd closes on completion +or failure. Unsupported snapshot/seal/exec policy refuses through the existing +parent-session revocation and uncertainty boundaries. The earlier unbounded +pre-snapshot path read was removed during independent review; actual adapter +symlink/FIFO/oversize refusal tests verify helper access never starts. + +Evidence: Custodian `docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/`, +including exact base/preimages/postimages/patch, proposed descriptor, coordinator +source review and canonical integration receipt. Independent Custodian checks +passed 49 affected source tests and 12 actual Go checker/helper/kernel scenarios; +the sets overlap and are not separate acceptance counts. Existing published +Warden d97652a producer pins remain unchanged. The exact KeyCape artifact remains +`sha256:be37738fe1c08259fb4f9ca0daf0f9301bfffc237e5bb38435525dfbf11ccff5`. +The predecessor path-replacement refusal and the first expanded reopened-fd test +failure are retained; the latter fixture was corrected to restore write mode +before independently proving seals block reopened-fd writes and shared mmap. + +This source task adds no native owner custody, callback, evidence trust, type +acknowledgment, credential access, consumer sign-in or rollback acceptance. +Executable memfds, Linux seals and usable procfs are explicit runtime dependencies +and still require native admission. This is an extracted source outcome from +T16, not completion of T16; T12/T14/T15 and all external-owner gates remain open. diff --git a/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/workplan-preimage.md b/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/workplan-preimage.md new file mode 100644 index 00000000..8d38b9a1 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/workplan-preimage.md @@ -0,0 +1,827 @@ +--- +id: NK-WP-0048 +type: workplan +title: "Establish SecurityCaseTypes so that known security cases are light" +domain: infotech +repo: net-kingdom +status: active +flavor: implementation +owner: claude-code +topic_slug: netkingdom +created: "2026-10-08" +updated: "2026-10-16" +depends_on: [] +related_repos: + - ops-mason + - ops-warden + - key-cape + - railiance-platform + - informed-decision +state_hub_workstream_id: "e31f3f9d-0748-5137-b0a3-7b3ca46a66eb" +--- + +# NK-WP-0048 - Establish SecurityCaseTypes so that known security cases are light + +## Why + +The founder's direction (2026-10-08): setting up a security integration of a +kind that has been done before should be easy, meaning accept the case, then +ops-mason arranges it and ops-warden follows through. Setting up a new kind +needs ample documentation and tests and an informed-decision, because its shape +is part of the security architecture. + +Today every integration is hand-built: per-client helper scripts, per-lane +custody, founder checklists. The trigger was KEY-WP-0038 (Forgejo as an OIDC +client), whose next step was going to be yet another hand-written rollout +script. + +This plan continues the pattern work of NK-WP-0008 (pattern infospace), +NK-WP-0009 (tutorials) and NK-WP-0030 (scenario composition). It turns patterns +into **SecurityCaseTypes** that ops-mason and ops-warden can execute. Canon: +`canon/standards/security-case-types_v0.1.md` (proposed). + +Companion plans: ops-mason `MASON-WP-0010` (materialization) and ops-warden +`WARDEN-WP-0044` (follow-through). + +## Tasks + +```task +id: NK-WP-0048-T01 +status: done +priority: high +state_hub_task_id: "930ac3e9-20ec-5473-98cd-3fe8fddb730f" +``` + +### T01 — Accept the terminology and the canon + +Founder review of `canon/standards/security-case-types_v0.1.md`, including its +section 10 open questions. On acceptance the canon status goes from `proposed` +to `accepted`. + +2026-10-08: accepted by the founder (in session). Canon status `accepted`; +section 10's open questions are carried forward to the first type (T04, T05). + +```task +id: NK-WP-0048-T02 +status: done +priority: high +state_hub_task_id: "19c25d96-a55c-58f0-9168-3fdbc4955fc2" +``` + +### T02 — Inventory the de-facto types already built by hand + +Read-only. List every security integration built so far and group them by shape. +Known starting points: + +- browser OIDC relying parties on key-cape: informed-decision-approver, + coulomb-social, user-engine portal, openbao-admin, and Forgejo (pending) +- confidential service clients with `client_credentials`: secrets-engine + approval, requester and openbao clients, approval-engine operator +- workload KV read lanes on OpenBao: the railiance-platform credential lane + designs and CCRs, for example `railiance-backup-offsite-lane` +- short-lived SSH certificates through ops-warden (`warden sign`) +- tunnelled operator access (ops-bridge) + +For each: the repositories involved, the hand-built steps, attended human steps, +custody paths, verification evidence, and rollback (if any). Output +`docs/security-case-type-inventory.md`, ranking candidate types by how often the +shape recurs and how much manual work each instance costs today. + +2026-10-08: done, `docs/security-case-type-inventory.md`. 36 of 37 CCRs are +already `request_type: workload-kv-read`, a de-facto type reviewed each time as +if it were new. Browser clients: 5 hand-built plus Forgejo, each with its own +rollout script. Confidential service clients need a registration plus two or +three KV-read CCRs each. A forge read identity is needed in bulk by +ORGREF-WP-0002-T06. Recommendation: oidc-browser-relying-party first (unblocks +the forge), workload-kv-read second (largest payoff), then forge-read-identity. + +```task +id: NK-WP-0048-T03 +status: done +depends_on: [NK-WP-0048-T01] +priority: high +state_hub_task_id: "e487975b-73b1-516c-971d-da4cbbb93063" +``` + +### T03 — Schema, template and validator + +- `canon/schemas/security-case-type_v0.1.schema.json` and + `security-case_v0.1.schema.json` +- `security-case-types/_template/` with `type.yaml`, `README.md` and tests to copy +- a validator, extending `tools/security-scenario-composer/`, that checks a type, + checks a case against its type, and renders the case into the composer's + deterministic plan (still `execution.permitted: false`) + +Acceptance: the validator rejects missing sections, secret-shaped parameter +values, unpinned providers and cases against non-`implemented` types, with tests +for each. + +2026-10-08: done. `canon/schemas/security-case-type_v0.1.schema.json` and +`security-case_v0.1.schema.json`; `tools/security-case-types/security_case_validator.py` +(`validate-type`, `validate-case`, `render`, reusing the scenario composer); +`security-case-types/_template/` (a valid, runnable example type over +`net-kingdom.local-identity`, with README sections, an example case and tests that +travel with every copy). 31 tests pass, including the composer's existing suite. + +```task +id: NK-WP-0048-T04 +status: done +depends_on: [NK-WP-0048-T02, NK-WP-0048-T03] +priority: high +state_hub_task_id: "2fffe112-7d55-5680-9952-78d74f44f87f" +``` + +### T04 — Define the first type: browser OIDC relying party + +`security-case-types/oidc-browser-relying-party/`, version 1.0: + +- parameters: client id, display name, exact redirect URIs, scopes, client type + (confidential or public), MFA pin, the groups the consuming service maps, + custody path for the secret (confidential only), consuming service and its + owner +- composition: key-cape client registration (key-cape declares it as a playbook + capability), secret minting into custody (railiance-platform), LLDAP group + existence and membership (net-kingdom identity-provisioner), handover to the + consuming service +- materialization (ops-mason) and follow-through (ops-warden) sections +- verification: discovery lists the redirect URI; a test sign-in returns the + expected claims for one human; anonymous access to the custody path is denied +- rollback: remove exactly this client and its custody path +- README: threat model, MFA rationale, why redirect URIs are exact, which steps + still need an attended login and how that is minimised +- reference instances: informed-decision-approver, coulomb-social, user-engine + portal, openbao-admin + +Acceptance: complete per canon section 3, tests green, status `proposed`. + +2026-10-08: done. `security-case-types/oidc-browser-relying-party/` 1.0 (proposed): +`type.yaml` with 12 parameters (the custody reference is required only for +confidential clients), composition over key-cape's new +`key-cape.oidc-client-registration` declaration (contract PASS, `maturity: +planned`), materialization and follow-through, verification and rollback, all +conditional on client type where it matters. One attended step, +`register-client`, justified by the live registry being a Kubernetes Secret. +README with threat model, alternatives, an honest fit table (portal and vergabe +fit; Forgejo is the first confidential case; openbao-admin, coulomb.social and +informed-decision-approver do not fit 1.0), and known limits. Examples: Forgejo +(checked against the accepted request) and a public portal. The validator gained +`when` / `required_when` conditions and provider discovery. 43 tests pass. +ops-mason's and ops-warden's sections are drafted in the type. Their owners +review them in MASON-WP-0010-T01 and WARDEN-WP-0044-T01 before acknowledgment +(T05). + +```task +id: NK-WP-0048-T05 +status: done +depends_on: [NK-WP-0048-T04, MASON-WP-0010-T01, WARDEN-WP-0044-T01] +priority: high +state_hub_task_id: "3d987d3d-0797-538b-86e4-51185a327ae2" +``` + +### T05 — Acknowledge the type through informed-decision + +Prepare the Decision Memo from the type's README and `type.yaml` and submit it +through informed-decision's production intake. On acknowledgment the type +moves to `acknowledged`. + +2026-10-08: Decision Memo prepared. Hub decision `e418263e-a71f-4f01-a970-c87c9e187c09`. +`security-case-types/oidc-browser-relying-party/decision/`: `packet.md` (question, +one act, what acknowledging does not do, the shape, alternatives, attended +steps, limits, evidence), `build_intake.py` and the built `intake.json` (memo +class `decision-record`, review group `net-kingdom-admins`, resolves the hub +decision on answer). The envelope passes informed-decision's own +`validate_submission`. Queuing it is an attended operator step, a write inside +the production dispatcher pod: + + kubectl -n state-hub exec -i deploy/state-hub-memo-dispatch -- \ + python -m api.services.memo_dispatch --queue /data/private enqueue --envelope /dev/stdin \ + < security-case-types/oidc-browser-relying-party/decision/intake.json + +The answer then appears for review in informed-decision. + +2026-10-08: queued by the founder (`pending`, `queued`) after one refusal: the +first envelope's idempotency key was not State Hub's stable +`state-hub..` form, and the builder now validates against both +sides. The dispatcher delivered it to informed-decision (ledger 7 → 8 +delivered). T05 waits for the founder's answer in informed-decision, which +resolves hub decision `e418263e`. + +2026-10-08: ops-mason's review (MASON-WP-0010-T01) folded into the type, still +1.0 `proposed`: `secret_ref` is no longer a parameter, the type derives +`platform/workloads/keycape/{client_id}`; policies are +`workload-kv-read-keycape-{client_id}-eso` and +`workload-kv-read-keycape-{client_id}-{consuming_service}`, each with a templated +body (`read` on that one data path); new `openbao_role` +`external-secrets-keycape-{client_id}` (Kubernetes auth, ESO service account, +15m, no default policy); catalog entry `{consuming_service}-keycape-client`; +the two rollback steps on the KV path collapse into one metadata delete. The +schema and validator now require a `body` on policy and role items. 105 tests +pass. Two consequences: the delivered memo describes the type before these +changes (a case-set custody reference, two differently named policies, no ESO +role), so the founder answers on a superseded shape unless the memo is +re-sent; and Forgejo's custody path moves from the accepted request's proposed +`platform/workloads/forgejo/netkingdom-oidc-client` to +`platform/workloads/keycape/forgejo`, which key-cape (KEY-WP-0038) and +railiance-apps (RAPPS-WP-0016-T02) need to adopt in T06. ops-warden's review +(WARDEN-WP-0044-T01) is still in progress. + +2026-10-08: revised memo `nk-wp-0048-t05-oidc-browser-relying-party-r2` built +(`packet.md`, `intake.json`, packet `sha256:af6435e2…`); it states that it +supersedes the first memo and passes informed-decision's and the dispatcher's +validation. It needs a new memo id because the dispatcher refuses new content +under a delivered idempotency key (`idempotency_conflict`). Queuing it is the +same attended step as above. Both memos serve hub decision `e418263e`; the +first answer resolves it, so the first memo must not stay open: an accept on it +would acknowledge the superseded shape, and a decline would resolve the +decision against r2. Close it with `return` (a reviewer verb that requests a +new revision and does not resolve the hub decision; only accept and decline +do), commenting that r2 supersedes it, then answer r2. `withdraw` exists in +the vocabulary but is not offered to reviewers, and lifecycle `revise` only +applies to lifecycle-registered memos, not intake-delivered ones. + +2026-10-09: the founder reports r2 enqueued and the first memo returned. At +close, hub decision `e418263e` still showed `open` (not verified whether r2 +was answered or the answer is still in transit; the dispatcher ledger was not +read). ops-warden's review (WARDEN-WP-0044-T01, ops-warden +`history/2026-10-08-oidc-rp-follow-through-review.md`, d470f2e) arrived at +21:25, before r2 was built, and is not in r2. Required before WARDEN-WP-0044-T02: +R1 a per-step `lane` (missing: identity-provisioner caller credential, +railiance-platform broker grant), R2 canon meaning of `warden_executes: true` +(caller proxy, caller identity, no standing ops-warden token; ADR-0002), R3 a +per-step `value_handling`. Refinements R4 (one type-level catalog lane instead +of a per-case entry, which conflicts with ops-mason's item 5), R5 (ESO Ready gate +before bind-env), R6 (exact accepted memberships), R7 (mfa-pinned and +secret-delivered checks). Next: if r2 was accepted, fold R1-R3 and R5-R7 in as +a follow-up and settle R4 between ops-warden and ops-mason before T06; if not, +hold r2, fold them in and build r3. + key-cape and railiance-apps were told about Forgejo's new custody +path. + +2026-10-09: r2 was not answered (hub decision `e418263e` open; dispatcher 9 +delivered), so r2 is held and r3 replaces it. ops-warden withdrew R4 for R4′ +(message a66a9204): one catalog entry per case with its body templated in the +type. Folded in: + +- Canon section 3: a narrow meaning of `warden_executes: true` (caller proxy, + caller identity, no standing credential; ops-warden ADR-0002), plus per-step + `lane`, `value_handling` and an optional `ready_check`. A catalog entry body is + now required, like policy and role bodies. +- Schema and validator enforce all of these. +- The type: lanes `identity-provisioner-groups` and + `keycape-client-secret-generate` (both new) and `key-cape-oidc-login`; + `mint-secret` is `generated_in_custody`; a Ready gate on `deliver-secret`; + `ensure-groups` adds exactly the accepted case's memberships; new checks + `secret-delivered` and `mfa-pinned`; the catalog-entry body follows R4′. +- 110 tests pass. + +Memo r3 (`nk-wp-0048-t05-oidc-browser-relying-party-r3`, packet +`sha256:726dfc01…`) passes informed-decision's and the dispatcher's validation. +The founder queues it with the same command, returns r2 with a comment that r3 +supersedes it, then answers r3. + +Prerequisites for T06, not for T05: +- the two new lanes, each through its owner; +- a founder-approved rollout of an identity-provisioner image containing + 630ebfd (the deployed one is main-6c4fcaf from 2026-09-24, without + `/v1/groups/ensure`). + +ops-mason's request to create LLDAP group `fluid-telegram-operators` (message +259ec26d) was answered: not possible from here by that path; the founder creates +it in the LLDAP admin UI. + +2026-10-09: ops-warden finished its side of WARDEN-WP-0044-T01 (message +`58d33f40`, commit 0e7ceb8): the revised type conforms to `warden case +check-type`, with no further changes asked of the type. This acknowledges that +review. Its later addendum R8 (message `af274783`, ops-warden +`history/2026-10-08-oidc-rp-follow-through-review.md`) showed that lane +`identity-provisioner-groups` cannot exist as declared: the provisioner's only +HTTP credential is user-engine's runtime token, and its NetworkPolicy admits +only user-engine pods, so a caller-proxy lane would act as user-engine. Taken +option 1 (ops-warden's recommendation): `ensure-groups` and its rollback are +attended, in the `register-client` session on `key-cape-oidc-login`, run inside +the provisioner pod; no token is read. Option 2 (an operator credential of its +own on the provisioner) stays open for a later version. `warden case +check-type` now reports only `keycape-client-secret-generate` +(railiance-platform) as missing. 89 tests pass. + +Hub decision `e418263e` was still open, so memo r4 +(`nk-wp-0048-t05-oidc-browser-relying-party-r4`, packet `sha256:452ea421…`) +replaces r3; it passes informed-decision's and the dispatcher's validation. The +founder queues it with the same command, returns r2/r3 if still open with a +comment that r4 supersedes them, then answers r4. + +Prerequisites for T06 are now: lane `keycape-client-secret-generate` +(railiance-platform), and a founder-approved rollout of an identity-provisioner +image containing 630ebfd. + +2026-10-09: done. The founder queued r4 (`pending`, `queued`), returned r3 as +superseded and accepted r4 in informed-decision. informed-decision's hub +write-back is off in production (`INFD_HUB_WRITEBACK` unset), so net-kingdom +resolved hub decision `e418263e` from the founder's report, naming memo r4 in +the rationale. The type is `acknowledged`; it becomes `implemented` with T06. + +```task +id: NK-WP-0048-T06 +status: wait +depends_on: [NK-WP-0048-T05, NK-WP-0048-T08, NK-WP-0048-T09, NK-WP-0048-T12, NK-WP-0048-T13, NK-WP-0048-T14, NK-WP-0048-T15, NK-WP-0048-T16, NK-WP-0048-T17, NK-WP-0048-T18, MASON-WP-0010-T03, WARDEN-WP-0044-T03] +blocking_reason: "Type1.2 architecture has a confirmed source-only founder acknowledgment; canonical1.1 remains unchanged/default and the1.2 reference case stays proposed. Needs explicit version selection/reference-source integration without normal admission bypass, closed owner adapters plus global/native receipt verifier (WARDEN-WP-0044-T03), independently verified six-object structural materialization and CCR42 claims/created-only provenance, exact broker/case/source reconciliation, then separately authorized native mint/delivery and attended registry/consumer verification. Source acknowledgment, a bundle or local tests do not prove implemented/materialized/live status." +priority: high +state_hub_task_id: "d9b26bcf-a7c6-59a2-92a0-f913d76a203c" +``` + +### T06 — First KnownSecurityCase: Forgejo + +Write `security-cases/forgejo-netkingdom-oidc.yaml` from the parameters the +founder accepted in KEY-WP-0038-T01 (`key-cape/docs/forgejo-oidc-client-request.yaml`), +so that acceptance carries over as the case acceptance. ops-mason materializes it; +ops-warden follows through and verifies it. The type becomes `implemented` once +this reference case is live. + +This replaces the hand-built path proposed for KEY-WP-0038-T02. + +2026-10-09: case written, `security-cases/forgejo-netkingdom-oidc.yaml`, status +`accepted`: the example's parameters plus the founder's 2026-10-08 acceptance of +the key-cape request (owner and founder), with the custody-path derivation +covered by the type's acknowledgment (memo r4). It validates and renders (plan +`execution.permitted: false`; three attended steps on `key-cape-oidc-login`); +it needs `--allow-unimplemented` until it is live, since it is the case that +makes the type `implemented`. A test keeps it equal to the type's example. +Still open before it runs: ops-mason renders and builds it (MASON-WP-0010-T02, +T03, unblocked by T05), ops-warden plans and runs it (WARDEN-WP-0044-T02, T03), +railiance-platform's `keycape-client-secret-generate` lane (asked by ops-mason +`789feb39` and ops-warden `d9f889b1`, both unread), and a founder-approved +identity-provisioner rollout containing 630ebfd. + +2026-10-09: identity-provisioner rolled out, founder-approved in session: +`main-630ebfd` (`sha256:bafb4860…`, built by the image workflow) replaces +`main-6c4fcaf` (`sha256:ffacd5d7…`; only change: 630ebfd, additive). Applied +with `kubectl set image` on the Deployment only (generation 17 → 18, 1/1 Ready); +the file's Service, NetworkPolicies and Ingress were not reapplied. Verified in +the pod: `ensure_service_groups` and `revoke_service_groups` present, `/readyz` +200; no token used, no group changed. Rollback: `kubectl -n sso set image +deploy/identity-provisioner provisioner=forgejo.coulomb.social/coulomb/identity-provisioner@sha256:ffacd5d790a5b7800e0535f3aa95b7de52c7a68e4f575b9192c3e1622fa672c2`. +The pin in `sso-mfa/k8s/identity-provisioner/deployment.yaml` follows. + +2026-10-09: ops-warden closed WARDEN-WP-0044-T01 and shipped `warden case plan` +(T02, message `06303d1d`); the accepted Forgejo case is refused until ops-mason +materializes it, and a materialized copy plans cleanly once +`keycape-client-secret-generate` exists. Its observation that `bind-env` carries +a literal `KEYCAPE__CLIENT_SECRET`: the validator now renders a +`{param:env}` format spec (upper case, `-` as `_`, as `keycape-client.py` +derives the name) and rejects unknown specs, and follow-through and rollback +text is now checked for unknown placeholders too (patch-level, canon section 6). +The type keeps the literal until ops-warden's planner (`text.format`) renders the +spec as well; asked of ops-warden. 53 tests pass. + +2026-10-10: ops-warden rendered `{param:env}` in `warden case plan` +(23ba584, message `c1df14b7`; other specs and `!conversion` refused). The type +adopts it as a 1.0 patch (canon section 6): `bind-env` and its rollback now read +`KEYCAPE_{client_id:env}_CLIENT_SECRET`. ops-mason reports the Forgejo case +rendered and built (message `1ebddae9`; CCR drafts handed to railiance-platform), +so T06 now waits on the platform apply, the mint-secret lane and the attended +register-client session. + +2026-10-10: key-cape adopted the type's custody path (key-cape ab5cccd, message +`88a4559c`): `docs/forgejo-oidc-client-request.yaml` and KEY-WP-0038-T02 name +`platform/workloads/keycape/forgejo`, field `CLIENT_SECRET`, delivered as +`sso/keycape-forgejo-client` through `external-secrets-keycape-forgejo`, and the +`forgejo/netkingdom-oidc-client` proposal is withdrawn. This matches the type and +the Forgejo example; no change needed here. + +2026-10-10: key-cape's type-composition review (key-cape 6038483, message +`8e5ed5f6`) found the composition sound with two blocking findings, taken as a +1.0 patch (stricter validation and documentation, canon section 6): +R1, redirect URIs now refuse `?` and `#` in the type pattern and in +`keycape-client.py`'s `REDIRECT` (KeyCape refuses `?` at startup and the single +replica would stop); R2, `allowed_scopes` always includes `groups`, the renderer +refuses `access_groups` without it, and `access_groups` now says the consumer +enforces the gate because KeyCape has none; R3, `mfa_required` "false" means the +per-user privacyIDEA policy; R4, register-client's justification records the +restart for every client, the quiet window, and that the rendered registry must +pass KeyCape's client validation before it. The accepted Forgejo case and both +examples still validate; 116 tests pass. +Open, needs a decision: R2's verification step that the consumer enforces the +group gate is a new verification step, hence 1.1 (minor), and the validator pins +cases to the exact type version, so the accepted 1.0 Forgejo case would need +re-pinning. Also open with key-cape: a KeyCape entry point that validates a +rendered client registry without starting the server; until then +`keycape-client.py` mirrors the redirect rule. + +2026-10-10: founder go for 1.1, so no 1.0 run overlaps. The type adds +verification `consumer-enforces-groups`, and hand-over now requires the consumer +to enforce `access_groups` (minor, canon section 6; risk class unchanged, no new +acknowledgment; history in the type README). The accepted Forgejo case and both +examples are re-pinned to `oidc-browser-relying-party@1.1` before any apply; the +acknowledgment intake keeps `@1.0` as the record of what was acknowledged. +ops-mason's build and ops-warden's case-ref digest predate this and need a +re-render. + +2026-10-10: ops-mason re-rendered the Forgejo case from `@1.1` (message +`22c6bba6`; plan digest `af1f4f81…`, no collisions) and rebuilt the bundle. The +rendered OpenBao objects are identical to 1.0, so the already applied +CCR-2026-0041 stands; 1.1 changes only consumer-side steps. key-cape shipped +`keycape check-config` (c528daa, message `a7b0aeff`): KeyCape's startup checks +without secret values, including the new duplicate-clientId rule. +`keycape-client.py --check-config ` now runs it on the live and the +proposed registry, fed on stdin, and apply refuses a proposed registry that +fails; register-client's entry point names the option (1.1 patch). The deployed +image predates c528daa, so the binary is built from key-cape source for now. +Before the next KeyCape rollout (Forgejo register-client or the CCR-2026-0018 +window), the attended operator runs `inspect --check-config` so the live +registry is checked too. + +```task +id: NK-WP-0048-T07 +status: wait +depends_on: [NK-WP-0048-T06] +blocking_reason: "The next types follow once the first one is proven end to end." +priority: medium +state_hub_task_id: "c3c9c3ae-4f23-5ba7-9e6f-e63bcf29b9f7" +``` + +### T07 — Queue the next types + +From the T02 ranking, open the next type drafts (likely the confidential service +client and the workload KV read lane), each to be acknowledged on its own. +Record which existing hand-built instances get adopted as cases, without +rewriting them in place. + +```task +id: NK-WP-0048-T08 +status: done +priority: high +state_hub_task_id: "8e519082-dc8b-5e46-b660-03d03e55dcf6" +``` + +### T08 — identity-provisioner: ensure service-access groups + +The type's `ensure-groups` step calls `identity-provisioner groups ensure --groups + --members `. The provisioner already has idempotent +`_ensure_group` and `_add_group`, but exposes only tenant-scoped operations. Add +the general operation (create if absent, add the listed memberships, never +remove others), its rollback (remove only this case's memberships), tests, and +the ops-warden route for its service identity. + +2026-10-08: done. `identity-provisioner/provisioner.py` gained +`ensure_service_groups` (`POST /v1/groups/ensure`) and `revoke_service_groups` +(`POST /v1/groups/revoke`), behind the existing service-token check. Names +matching `tenant*`, `netkingdom-*` and `lldap_*` are reserved; members must +already exist; everything is validated before any change; memberships are read +back before success; nothing else is ever removed; groups are deleted only when +asked and empty. 7 new tests (39 in the provisioner suite). The type's +`ensure-groups` step and its rollback now name these endpoints. The ops-warden +route for the provisioner's service token belongs to WARDEN-WP-0044-T01. + +```task +id: NK-WP-0048-T09 +status: done +priority: high +state_hub_task_id: "0ee12dfc-8260-5038-bd39-cc7fb4113be0" +``` + +### T09 — The `keycape-client` entry point and its declaration + +Planned first as key-cape KEY-WP-0038-T04. Moved here because the machinery a +live registration needs already lives in net-kingdom, where KeyCape is deployed +(`sso-mfa/k8s/keycape/`): the contained kubectl, the cluster-identity pin and the +issuer pin. A copy in key-cape would drift. + +2026-10-08: done. `sso-mfa/k8s/keycape/keycape-client.py` derives the KeyCape +client record from a case (`client-from-case`). It adds or removes exactly one +client with the existing safeguards (byte-preserving merge, exact match or +refuse, shared-reference refusal, compare-and-swap, read-back, contained errors), +requires the Deployment env binding first for confidential clients, and requires +the case acceptance record for `apply` instead of a per-client founder decision +id. 15 tests, including a regression showing that the vergabe case reproduces +its hand-built registration exactly; all 71 helper tests pass. Declaration +`capabilities/playbooks/net-kingdom.keycape-client-registration.yaml` (contract +PASS, `maturity: implemented`) replaces key-cape's draft. The type now pins it, +and for confidential clients adds `deliver-secret` (delivery from custody into +the sso namespace) and `bind-env` (Deployment env), attended in the same session +as `register-client`. The per-client helpers stay until their clients are +adopted as cases. + +2026-10-10 — NetKingdom owner review of the execution seam, source only: +ops-warden/docs/security-case-execution-proposal.md SHA256 78ab1439e4c5beb205aeccf15b1b2e706511be55ab195c5a3aa682f326c7c77c +was reviewed against INTENT, SecurityCase canon, acknowledged type1.1, current +case and owner source. docs/security-case-execution-owner-review.md supports +the closed adapter/receipt direction with explicit versioned schema, structural +materialization provenance, unchanged caller/session boundaries, safe CAS +reconciliation and exact-created-object rollback requirements. This is owner +review, not approval. Supporting file digests are in the adjacent sources JSON. + +Current source exposes a circular wait: ops-mason T03 waits on mint while Warden +and the broker refuse a non-materialized case. Structural acceptance therefore +needs its own native pinned receipt before mint, not a successful mint before +materialized. The platform broker additionally pins type1.0 while Forgejo is +accepted at1.1; platform-owner alignment and tests are required, not an unchecked +version bypass. T06 remains wait. No case/type lifecycle status, grant, authority +or live runtime changed by this review. A proposed informed-decision would +adopt the source contract only; native materialization and execution acceptance +stay distinct. + +Final proposal re-review: staged Warden proposal SHA256 78ab1439e4c5beb205aeccf15b1b2e706511be55ab195c5a3aa682f326c7c77c +adds only the explicitly local reducer appendix, not an accepted wire protocol +or executor. The exact-final review and sources JSON supersede the earlier +1035f049... proposal review. Platform staged the exact type1.1 broker correction +with ten fake-backend subprocess checks (script SHA256 +9576cd211cfc06526ec4cc9bd0a5b06f0f59287a68931bb2d6e3592a951ddc7d). +The next source work is the versioned closed adapter/receipt/materialization +contract, not another relaxation of the broker's status gate. Native work still +requires installed broker provenance, structural reader receipts, exact new +materialized-case bytes, caller grant/session revocation and verified execution. +No installed/native acceptance is inferred from that staged correction. + +2026-10-11 — new source-only type1.2 architecture acknowledgment: +Confirmed native founder ACCEPT of infd-20261011-forgejo-type12-source-r1@1 +at06:47:39Z, approval8e9332c8-754a-420c-b8b7-445cb4fe6765. Exact frozen packet +0b377d647ca748cd03c92f9792396412048055d4bde9f7bb06ff6d413f7acd68 and submission +585cee731de30d2b37883a57abfd9eb32f023ee694e8d911989109d5caf80569; native receipt +retained in docs/evidence/2026-10-11-forgejo-type12-source-acknowledgment/. +The complete acknowledged1.2 source package is isolated under +security-case-type-revisions/1.2/oidc-browser-relying-party; default loader and +current accepted1.1 case remain untouched because load_type is single-version. +The new proposed1.2 case is NOT accepted/materialized/live and has no acceptance +fields. Actual reference-migration design approval is a separate Apps owner +receipt; no1.1 approval is inherited. Source-recording review/test proof and exact +next gates are in docs/forgejo-type12-source-acknowledgment-followthrough.md. +T06 remains wait and T07 remains its existing dependent task; type is NOT +implemented and no first-case/native grant or broader workplan finish is asserted. + +2026-10-11 — bounded explicit version selection source preparation for T06: +The validator now offers optional exact version_roots/--version-root VERSION=PATH +selection with duplicate/mismatch/traversal/symlink refusal and no fallback. The +default1.1 source and historicalacceptedcase hashes/ordinaryrefusal/preparation +render stayunchanged; actual1.1 is acknowledged, not implemented. Acknowledged1.2 +ordinaryadmission also refuses; optionalpreparation remains nonexecuting.43local +validatorchecks pass and original1.1 CLIrenderbytes match exactly. Makerproof: +docs/evidence/2026-10-11-explicit-type-version-selection/source-integration-validation.json. +FormalNetownerreview/reference-source integration and all existing native adapter, +provenance/broker/materialization/attendedproof gates remain; T06 stayswait, no +new residual or firstcase/typeimplementation/native grant is inferred. + + +2026-10-16 — isolated source preparation for T06 (canonical integration pending): +Added `check-evidence` to the existing validator CLI. It reuses exact version +selection and ordinary type/case/render admission, checks actual Mason structural +request source/inventory/projection pins and reads actual Warden journal rows +read-only. Successful rows require exact type-declared descriptors and attempt/ +revocation fingerprints. Existing acknowledged types without machine descriptors +remain plan-only. The compatibility result never claims authenticated native +completion: Request and journal fingerprints are not portable native evidence. +Owner authentication/status binding and actual native gates remain unresolved; +T06/T07 statuses and both type/case lifecycles remain unchanged. No dispatch, +credential access or authority expansion is introduced. See tool README for the +existing owner protocol boundaries and source integration tests. + +Concrete next Warden owner integration: current `Executor.admit` refuses every +case parameter whose sensitivity is not public/operational, so browser +`redirect_uris`, `allowed_scopes`, `client_type`, `mfa_required` (and 1.2 +`consumer_namespace`) fail after otherwise prepared admission. The sensitive +classification is a security-change review level, not secret content. The +Warden owner must validate through NetKingdom's existing exact-version case/type +validator, then project only descriptor-declared safe target metadata; do not +remove the credential/value refusal or copy complete parameters into journals. +Actual owner descriptor acknowledgment and independent evidence admission remain +prerequisites, with normal admission still refusing unimplemented types. + + +## T10 — Source compatibility checker for Mason and Warden evidence + +```task +id: NK-WP-0048-T10 +status: done +priority: high +state_hub_task_id: "4b06ca97-c1e4-5680-bf22-ab75ab1646a4" +``` + +Completed 2026-10-16 in published source `c31d21ba`. The read-only +`check-evidence` command checks the actual Mason structural request and Warden +journal source protocols against an exact version/render. Independent source +review passed 103 tests at that checkpoint. It always reports native completion +false and execution prohibited. Fixtures and local journal fingerprints do not +provide authenticated owner/native evidence. This records completed source scope +previously described under T06; it does not duplicate native acceptance credit. + +## T11 — Validate the optional machine-descriptor pin contract + +```task +id: NK-WP-0048-T11 +status: done +priority: high +state_hub_task_id: "ef2a0119-f59e-5e45-9519-115da9df7b85" +``` + +Completed 2026-10-16. Reproduced the JSON schema rejecting a valid +`execution_descriptors` map while the Python validator accepted malformed maps. +The schema now supports the existing Warden optional exact-byte pin map; semantic +validation binds keys to declared steps, bounds the map and checks digest shape. +The direct evidence API refuses malformed maps before accessing them. Twenty-one +new checks and all 127 affected source checks passed. No current type/case bytes, +status, owner descriptors or native acceptance changed. Source evidence and +publication receipt: Custodian +`docs/changes/netkingdom-target-2026-10-16/browser-acceptance-next/`. + +## T12 — Prepare NetKingdom-owned attended step adapters + +```task +id: NK-WP-0048-T12 +status: todo +priority: high +state_hub_task_id: "7ce4eee3-cd47-55a9-bfa5-4f1835d4d6fd" +``` + +Owner: NetKingdom. Retained scope: prepare bounded attended Deployment env, +config verification and restart/readiness steps around the existing helpers. +Next action: identify exact current entry points and their revision/readback +contracts, then implement fixed source-pinned callbacks or receipt-only attended +steps. Completion requires positive/refusal checks, exact target/source pins and +uncertainty behavior without acquiring user-engine runtime identity or exposing +registry/client secrets. Existing caller/session, cluster/issuer, CAS/readback +and status-only output boundaries remain mandatory. + +2026-10-16 task refinement: the original broad adapter scope is split into T13 +(completed registration source candidate), T14 (group creation provenance), T15 +(durable registration recovery) and T16 (descriptor/artifact selection and owner +review). This preserves T12's task ID and Hub UUID while giving each remaining +outcome an owner and completion gate. T06 now names every source dependency; +its native/reference lifecycle acceptance scope remains unchanged. Completion +of any source task supplies no Warden, Platform or Apps assent and no native +execution authority. + +## T13 — Prepare the fixed registration adapter source candidate + +```task +id: NK-WP-0048-T13 +status: done +priority: high +state_hub_task_id: "b264c10b-565c-5d29-b788-03cf5fbd2883" +``` + +Completed 2026-10-16, source implementation and synthetic fixture proof only. +Added an injection-only adapter around actual `keycape-client.run`: fixed +case/principal/human/lane/source/acceptance binding, existing CAS/readback, +metadata-only owner evidence and per-attempt parent-session revocation boundary. +The descriptor source composition pins the exact checker executable digest +alongside adapter/helper source. Creation-only rollback authenticates its +creation receipt and refuses changed postcreation UID/resourceVersion; +pre-existing clients confer no rollback authority. Missing readback, evidence, +revocation or process-local creation provenance cannot become retry-safe success. + +Evidence: Custodian `docs/changes/netkingdom-target-2026-10-16/browser-adapter-next/` +R3 candidate, explicit patch and postimages, retained R1 rejected custody finding +and R2 revision, and 36 passing source checks (16 new, 20 existing). Three checks +use actual Warden kernel admission/dispatch/rollback with published Warden +`d97652a3bb5fa733b063d71a1b74068efcccf572` producer pins unchanged. Owner trust, +descriptor/type/case and Kubernetes objects remain synthetic fixtures. This +records a completed source deliverable, not independent review, publication, +type acknowledgment, exercised procedure, identity admission or native browser +acceptance. Independent root source review: Custodian browser-adapter-next/coordinator-review-r3.json. Source integration and normal publication/synchronization receipts: Custodian browser-adapter-next/canonical-integration.json. These receipts supply no other-owner assent or native authority. + +Source publication: `0775c2bfa41af236301155bb5a69bbf088bdf823`, independently +reviewed by the Custodian coordinator before integration. The required initial +projection and ordinary UUID writeback both completed with zero failures and +zero automation errors. Generated-only brief commits were refused by the normal +hook; exact pre-existing staged/working brief bytes were restored. The separate +canonical integration receipt retains those refusals and projection evidence. +Native admission and all other-owner acceptance remain open. + +## T14 — Preserve created-only provenance for group ensure and revoke + +```task +id: NK-WP-0048-T14 +status: todo +priority: high +state_hub_task_id: "fce95440-a25b-5f27-8ef3-a3d182099113" +``` + +Owner: NetKingdom. Next action: extend or wrap the existing identity-provisioner +ensure/revoke contract with independently retained prestate and exact newly +created groups/memberships before exposing reversible owner callbacks. Current +ensure returns no membership-creation inventory; current revoke may remove a +membership that predated this case. Completion requires actual helper-backed +positive/refusal fixtures, all validation before mutation, per-attempt evidence +and revocation, readback, ambiguity handling, and rollback that preserves every +pre-existing/shared group and membership. No existing endpoint or service token +is treated as new authorization. + +## T15 — Define authenticated recovery for lost registration process state + +```task +id: NK-WP-0048-T15 +status: todo +priority: high +state_hub_task_id: "1b825705-4848-5edc-832f-b1622cd9bdb7" +``` + +Owner: NetKingdom for registration state, with Warden owning kernel evidence +admission. Next action: select the exact existing owner evidence holder/readback +contract and define authenticated lookup for the original creation attempt. +Completion requires restart/lost-receipt fixtures that bind case/source/principal, +original created-only provenance and current native revision before any undo; +uncertain outcomes remain non-replayable. The T13 adapter deliberately refuses +post-restart rollback and installs no reconciliation callback. A new broker or +unverified JSON receipt is not a recovery contract. Actual evidence-holder +acceptance and attended recovery remain separately authorized native work. + +## T16 — Select and review the actual machine descriptor composition + +```task +id: NK-WP-0048-T16 +status: todo +priority: high +state_hub_task_id: "200bf500-5084-5ae7-be26-3033fe1f296e" +``` + +Owner: NetKingdom for descriptor/type source and exact checker artifact; Warden +independently owns global provenance/admission. Next action: obtain review of the +exact checker source/artifact proposed by T17 and identify native owner evidence/ +session callback contracts, then assemble the complete step descriptor set and +a proposed explicit type version selection. T13's historical descriptor retains +its synthetic checker; T17 supplies an artifact-bound proposal, not admission. +Completion requires separate owner source review of exact machine descriptors, +artifact/source/config pins, normal type/version admission and independent +evidence-verification boundaries. Warden d97652a refreshes producer validator +pins for NetKingdom 015694ee; it does not attest T13's new adapter. Pinning any +machine descriptor into an acknowledged type is a separately reviewed source +change. Platform owns structural/mint/broker/delivery evidence; Apps owns actual +consumer login/group enforcement. Neither their assent nor native acceptance is +supplied by this task. + +## T17 — Build and exercise the exact offline config-checker artifact + +```task +id: NK-WP-0048-T17 +status: done +priority: high +state_hub_task_id: "8c3e4baa-a218-5996-af9e-fe1153a46dac" +``` + +Completed 2026-10-16, isolated artifact/source fixture proof only. Built the +existing `key-cape/src/cmd/keycape` command from published clean source revision +`3c71226a1a7b8a2bdb710e9c497e1a1f7f8a3987`, source tree +`f0010b9cab17dbc3a787628a29657d98b900320f`, with cached Go 1.23.0, +CGO disabled, Linux/amd64 v1, trimmed paths, disabled VCS stamping and offline +read-only module selection. The full retained closure pins 108 source files, +9876 toolchain files and ten used module source directories. Three isolated +build locations produced identical 13633409-byte artifacts: +`sha256:be37738fe1c08259fb4f9ca0daf0f9301bfffc237e5bb38435525dfbf11ccff5`. + +Eight disposable checks use the actual executable and maintained helper/kernel, +with complete synthetic configs and a minimal environment: add/created-only +removal, normal pinned producer admission, rollback, lost-readback uncertainty, +stale revisions, substituted checker refusal, invalid startup config and missing +secret-env refusal. No checker stub or producer-pin override remains. The +artifact-bound descriptor proposal is +`sha256:76b1153d188062ece53fcd5e8d35290fdbfd02a69dd79d4b001fc42781afcdc3`. +Evidence: Custodian +`docs/changes/netkingdom-target-2026-10-16/browser-checker-artifact-next/`. +The first two build attempts succeeded but their closure collection failed; +both failures are retained, as is refusal of the older incomplete synthetic +fixture. No production config, credentials or cluster mutation was used. + +This extracts completed build/fixture scope from T16 without claiming KeyCape +owner endorsement, native checker selection/storage custody, acknowledged type +integration, native evidence/session callbacks or attended browser acceptance. +T16 retains exact owner review and composition/admission scope. The attended +registration procedure remains unexercised. + +## T18 — Make checker input and executable selection deterministic + +```task +id: NK-WP-0048-T18 +status: done +priority: high +state_hub_task_id: "9c45da71-fb4c-5f20-9065-90f448f37c64" +``` + +Completed 2026-10-16, isolated source correction and real-artifact fixture proof +only; independent integration/publication remain separately recorded. KeyCape +source review identified non-secret `KEYCAPE_ACCOUNT_PORTAL_URL` and +`KEYCAPE_BROWSER_LOGOUT_URL` overrides in config.Load. The original helper +inherited ambient environment, allowing valid supplied configuration to be +rejected or invalid supplied URL configuration to be admitted. A bare relative +checker filename could also be replaced by PATH lookup after the adapter had +hashed the selected local file. All three counterexamples are retained against +the unchanged predecessor. + +The existing helper now resolves the selected checker file to an absolute path +and passes an explicit empty subprocess environment. Three opt-in real-checker +regressions verify both URL-verdict directions and relative-file selection under +a non-checker PATH decoy. All 39 affected checks pass with the exact supplied +artifact, as do 11 real-artifact helper/kernel scenarios. Source evidence: +Custodian `docs/changes/netkingdom-target-2026-10-16/browser-checker-environment-r2/`. +Historical descriptors remain retained; the helper correction changes successor +source-composition pins and requires their separate review/admission. An absolute +path is not immutable custody and does not eliminate file replacement races; +T16 retains native executable storage/custody and callback/type admission. +No production configuration, credentials or cluster mutation was used. + +Source integration and publication: `916ba4b33e03179274d03b78bd898a4843e8653f`, following independent +Custodian source review in browser-checker-environment-r2/coordinator-review.json. +The ordinary protected projection pass assigned T17/T18 UUIDs with zero failures +and zero automation errors; generated-only brief commit refusal is retained. +The exact prior staged/working brief bytes were restored. Final source/publication/ +projection evidence is retained in Custodian browser-checker-environment-r2/ +canonical-integration.json. These source receipts do not admit native executable +custody, owner callbacks, acknowledged type changes or browser identities. diff --git a/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/workplan.patch b/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/workplan.patch new file mode 100644 index 00000000..15e9976f --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/workplan.patch @@ -0,0 +1,57 @@ +--- a/workplans/NK-WP-0048-security-case-types.md ++++ b/workplans/NK-WP-0048-security-case-types.md +@@ -748,6 +748,14 @@ + consumer login/group enforcement. Neither their assent nor native acceptance is + supplied by this task. + ++2026-10-16 source checkpoint: T19 below extracts the completed per-attempt ++executable snapshot implementation and reviewed fixture proof. T16 retains ++native executable storage/custody, exact owner evidence/session callbacks, ++complete descriptor composition and ordinary type/version admission. The ++successor descriptor/source pins are proposed only; the earlier descriptor ++history is preserved. T14 group provenance and T15 durable restart/recovery ++remain independent open requirements. No native acceptance is counted here. ++ + ## T17 — Build and exercise the exact offline config-checker artifact + + ```task +@@ -825,3 +833,39 @@ + projection evidence is retained in Custodian browser-checker-environment-r2/ + canonical-integration.json. These source receipts do not admit native executable + custody, owner callbacks, acknowledged type changes or browser identities. ++ ++## T19 — Bind checker execution to verified immutable attempt bytes ++ ++```task ++id: NK-WP-0048-T19 ++status: done ++priority: high ++``` ++ ++Completed 2026-10-16, independently reviewed source implementation and synthetic ++fixture proof only. The registration adapter now opens a bounded regular checker ++artifact without following its terminal symlink, verifies the reviewed digest, ++and snapshots the exact bytes into a write/grow/shrink/seal-removal-sealed Linux ++memfd before helper access. Live and proposed config checks execute that same fd ++through procfs with the existing empty environment; the fd closes on completion ++or failure. Unsupported snapshot/seal/exec policy refuses through the existing ++parent-session revocation and uncertainty boundaries. The earlier unbounded ++pre-snapshot path read was removed during independent review; actual adapter ++symlink/FIFO/oversize refusal tests verify helper access never starts. ++ ++Evidence: Custodian `docs/changes/netkingdom-target-2026-10-16/browser-resume-r1/`, ++including exact base/preimages/postimages/patch, proposed descriptor, coordinator ++source review and canonical integration receipt. Independent Custodian checks ++passed 49 affected source tests and 12 actual Go checker/helper/kernel scenarios; ++the sets overlap and are not separate acceptance counts. Existing published ++Warden d97652a producer pins remain unchanged. The exact KeyCape artifact remains ++`sha256:be37738fe1c08259fb4f9ca0daf0f9301bfffc237e5bb38435525dfbf11ccff5`. ++The predecessor path-replacement refusal and the first expanded reopened-fd test ++failure are retained; the latter fixture was corrected to restore write mode ++before independently proving seals block reopened-fd writes and shared mmap. ++ ++This source task adds no native owner custody, callback, evidence trust, type ++acknowledgment, credential access, consumer sign-in or rollback acceptance. ++Executable memfds, Linux seals and usable procfs are explicit runtime dependencies ++and still require native admission. This is an extracted source outcome from ++T16, not completion of T16; T12/T14/T15 and all external-owner gates remain open. diff --git a/docs/changes/netkingdom-target-2026-10-16/checkpoint-sync-3291a66.json b/docs/changes/netkingdom-target-2026-10-16/checkpoint-sync-3291a66.json new file mode 100644 index 00000000..19bee46f --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/checkpoint-sync-3291a66.json @@ -0,0 +1,16 @@ +{ + "session_id": 77536, + "exit_code": 0, + "source_revision": "3291a660ae5170d368bdc325b87d9bd5c551e7ff", + "summary": { + "fail": 0, + "automation_error": 0, + "warn": 7, + "info": 4 + }, + "bindings": "89/89", + "pushed": true, + "generated_only_commit_refused": true, + "unrelated_staging_preserved": true, + "native_acceptance": false +} diff --git a/docs/changes/netkingdom-target-2026-10-16/checkpoint-sync-bc86ef1.json b/docs/changes/netkingdom-target-2026-10-16/checkpoint-sync-bc86ef1.json new file mode 100644 index 00000000..4d1df3a9 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/checkpoint-sync-bc86ef1.json @@ -0,0 +1,16 @@ +{ + "commit": "bc86ef1cf45aae9e6d534de70ad44ba8b617bc69", + "published": true, + "verified_at": "2026-10-16T15:40:45.359016+00:00", + "command": "statehub fix-consistency --repo the-custodian --no-writeback --json", + "bindings_synced": 89, + "bindings_total": 89, + "fail": 0, + "automation_error": 0, + "warnings": 7, + "info": 4, + "task_description_synced": "CUST-WP-0080-T06", + "retained_caveat": "Generated-only working-memory commit refused; unrelated existing changes preserved. Deferred CUST73T05 rotation remains deferred despite generic C40 warning.", + "native_gates_open": 16, + "progress_id": "23a4bd92-3406-4c39-9267-a9bc9a1503c5" +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/actual-source-allocator-preview.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/actual-source-allocator-preview.json new file mode 100644 index 00000000..8b450cc0 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/actual-source-allocator-preview.json @@ -0,0 +1,210 @@ +{ + "status": "candidate_read_only_allocator_preview_not_admission", + "assignments": [ + { + "kind": "task", + "record_id": "RCLUSTER-WP-ADHOC-2026-07-27-T01", + "uuid": "3e3f39d3-5070-5ba3-97db-19f5f361f264", + "path": "workplans/ADHOC-2026-07-27.md" + }, + { + "kind": "workplan", + "record_id": "RCLUSTER-WP-0018", + "uuid": "e0a7813d-c916-5e49-84b2-5e5b30441891", + "path": "workplans/RCLUSTER-WP-0018-keycape-original-tls-recovery.md" + }, + { + "kind": "task", + "record_id": "RCLUSTER-WP-0018-T01", + "uuid": "7280193d-0d59-5b78-a2d3-96ebd127127f", + "path": "workplans/RCLUSTER-WP-0018-keycape-original-tls-recovery.md" + }, + { + "kind": "task", + "record_id": "RCLUSTER-WP-0018-T02", + "uuid": "90836435-917f-504c-a711-3f29ece348e3", + "path": "workplans/RCLUSTER-WP-0018-keycape-original-tls-recovery.md" + }, + { + "kind": "task", + "record_id": "RCLUSTER-WP-0018-T03", + "uuid": "5dfa9a3a-2639-5cf6-b0a4-13eef417d47a", + "path": "workplans/RCLUSTER-WP-0018-keycape-original-tls-recovery.md" + }, + { + "kind": "task", + "record_id": "RCLUSTER-WP-0018-T04", + "uuid": "021b07bb-65bc-5743-bac5-4294b4be74f6", + "path": "workplans/RCLUSTER-WP-0018-keycape-original-tls-recovery.md" + }, + { + "kind": "task", + "record_id": "RCLUSTER-WP-0018-T05", + "uuid": "5bc67a22-0991-5625-86a6-f749560d8a0e", + "path": "workplans/RCLUSTER-WP-0018-keycape-original-tls-recovery.md" + } + ], + "files_changed": [ + "workplans/ADHOC-2026-07-27.md", + "workplans/RCLUSTER-WP-0018-keycape-original-tls-recovery.md" + ], + "historical_invalid_identifiers": [ + { + "kind": "task", + "id": "T01", + "source": "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md#task-block-1" + }, + { + "kind": "task", + "id": "T02", + "source": "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md#task-block-2" + }, + { + "kind": "task", + "id": "T03", + "source": "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md#task-block-3" + }, + { + "kind": "task", + "id": "T04", + "source": "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md#task-block-4" + }, + { + "kind": "task", + "id": "T05", + "source": "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md#task-block-5" + }, + { + "kind": "task", + "id": "T01", + "source": "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md#task-block-1" + }, + { + "kind": "task", + "id": "T02", + "source": "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md#task-block-2" + }, + { + "kind": "task", + "id": "T03", + "source": "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md#task-block-3" + }, + { + "kind": "task", + "id": "T04", + "source": "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md#task-block-4" + }, + { + "kind": "task", + "id": "T01", + "source": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md#task-block-1" + }, + { + "kind": "task", + "id": "T02", + "source": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md#task-block-2" + }, + { + "kind": "task", + "id": "T03", + "source": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md#task-block-3" + }, + { + "kind": "task", + "id": "T04", + "source": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md#task-block-4" + }, + { + "kind": "task", + "id": "T05", + "source": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md#task-block-5" + }, + { + "kind": "task", + "id": "T06", + "source": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md#task-block-6" + } + ], + "historical_identity_collisions": [ + { + "kind": "task", + "id": "T01", + "uuid": null, + "sources": [ + "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md#task-block-1", + "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md#task-block-1", + "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md#task-block-1" + ], + "parent_id": "RAIL-BS-WP-0002", + "uuids": [ + "3f042630-eab0-4c6a-9167-e2b28ff20e40", + "4526a842-ea31-4874-9231-92ab556cfe7b", + "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc" + ] + }, + { + "kind": "task", + "id": "T02", + "uuid": null, + "sources": [ + "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md#task-block-2", + "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md#task-block-2", + "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md#task-block-2" + ], + "parent_id": "RAIL-BS-WP-0002", + "uuids": [ + "a6313e06-1976-46a7-8e31-df4eb2eca880", + "ab166073-30a7-4702-a037-4091e8706e20", + "e8510646-46ed-4697-a345-f3d3009eea78" + ] + }, + { + "kind": "task", + "id": "T03", + "uuid": null, + "sources": [ + "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md#task-block-3", + "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md#task-block-3", + "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md#task-block-3" + ], + "parent_id": "RAIL-BS-WP-0002", + "uuids": [ + "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3", + "140da396-8e30-4f4d-b88c-c42c0cd46c01", + "dab2c07f-8aa0-4635-8df6-857e87e93fc5" + ] + }, + { + "kind": "task", + "id": "T04", + "uuid": null, + "sources": [ + "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md#task-block-4", + "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md#task-block-4", + "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md#task-block-4" + ], + "parent_id": "RAIL-BS-WP-0002", + "uuids": [ + "08233868-d522-4117-bc4e-6c0f52545665", + "5c3d40e4-239b-488e-9519-6f7a38d2325f", + "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d" + ] + }, + { + "kind": "task", + "id": "T05", + "uuid": null, + "sources": [ + "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md#task-block-5", + "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md#task-block-5" + ], + "parent_id": "RAIL-BS-WP-0002", + "uuids": [ + "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8", + "7f9e0e58-a130-467a-a2d0-b3f2564e496f" + ] + } + ], + "strict_scan_ok": false, + "all_source_bytes_preserved": true +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/coordination-cases.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/coordination-cases.json new file mode 100644 index 00000000..0a82171b --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/coordination-cases.json @@ -0,0 +1,25 @@ +{ + "schema": "coordination-derived-cases.v1", + "status": "local_observed_waiting_not_hub_published", + "worker_injection": false, + "cases": [ + { + "scenario_type": "project_driver", + "case_id": "cluster-frozen-receiver-before-registrar-r1", + "driver_repo": "prj-state-hub-retirement", + "driver_workplan_id": "SHR-WP-0001", + "beneficiary_workplan_id": "RMGR-WP-0019", + "beneficiary_task_id": "RMGR-WP-0019-T05", + "obligor_workplan_id": "STATE-WP-0095", + "obligor_task_id": "STATE-WP-0095-T06", + "flavor": "implementation", + "state": "waiting", + "reason": "receiver-first reviewed rollout is a human gate; no worker wake", + "indirect_beneficiaries": [ + "RCLUSTER-WP-0018", + "NK-WP-0049-T35" + ], + "next_action": "independent source review, canonical integration and exact CI image proposal before scoped rollout review" + } + ] +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/current-deployment-metadata.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/current-deployment-metadata.json new file mode 100644 index 00000000..d0f6a1ec --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/current-deployment-metadata.json @@ -0,0 +1,49 @@ +{ + "schema": "cluster-historical-receiver-deployment-metadata.v1", + "status": "metadata_observed", + "mutations": false, + "target_node_ip_confirmed": "92.205.62.239", + "deployments": [ + { + "name": "state-hub", + "generation": 76, + "observed_generation": 76, + "ready_replicas": 1, + "images": [ + "forgejo.coulomb.social/coulomb/state-hub:main-7d99282" + ] + }, + { + "name": "state-hub-mcp", + "generation": 48, + "observed_generation": 48, + "ready_replicas": 1, + "images": [ + "forgejo.coulomb.social/coulomb/state-hub:main-7d99282" + ] + } + ], + "helm_history": [ + { + "revision": 73, + "status": "superseded", + "chart": "state-hub-0.1.0", + "app_version": "main-5dd3df9", + "updated": "2026-10-04T15:39:41.98613624+02:00" + }, + { + "revision": 74, + "status": "superseded", + "chart": "state-hub-0.1.0", + "app_version": "main-5dd3df9", + "updated": "2026-10-06T10:27:00.194272456+02:00" + }, + { + "revision": 75, + "status": "deployed", + "chart": "state-hub-0.1.0", + "app_version": "main-5dd3df9", + "updated": "2026-10-06T12:02:21.111126728+02:00" + } + ] +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/managed-adhoc-backfill.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/managed-adhoc-backfill.json new file mode 100644 index 00000000..b038457a --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/managed-adhoc-backfill.json @@ -0,0 +1,10 @@ +{ + "status": "preview_only_apply_via_normal_allocator_after_receiving_acceptance", + "path": "workplans/ADHOC-2026-07-27.md", + "record_id": "RCLUSTER-WP-ADHOC-2026-07-27-T01", + "uuid": "3e3f39d3-5070-5ba3-97db-19f5f361f264", + "before_sha256": "f480e729a820367109d0b58cbbf367f6ac4137408f5caadbc2708fa181a80c67", + "expected_after_sha256": "6d3b25ded22ae68835a2a2c97d517194b06fedf98a29dbcc252f99505fbe0ace", + "added_lines": 1, + "scope": "one existing task UUID field only; no ID rename or content changes" +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/managed-adhoc-backfill.preview.patch b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/managed-adhoc-backfill.preview.patch new file mode 100644 index 00000000..1a5b4d14 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/managed-adhoc-backfill.preview.patch @@ -0,0 +1,10 @@ +--- a/workplans/ADHOC-2026-07-27.md ++++ b/workplans/ADHOC-2026-07-27.md +@@ -20,6 +20,7 @@ + id: RCLUSTER-WP-ADHOC-2026-07-27-T01 + status: done + priority: high ++state_hub_task_id: "3e3f39d3-5070-5ba3-97db-19f5f361f264" + ``` + + Enable only `kubernetes.podspec-init-containers` in Knative diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/README.md b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/README.md new file mode 100644 index 00000000..01448a79 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/README.md @@ -0,0 +1,43 @@ +# Disposable inbox migration compatibility candidate + +Prepared, not executed or approved. This is a bounded source follow-up to the +State Hub receiver rollout proposal; it does not authorize deploying it. + +The offline builder used the exact deployed7d99282 ORM source for the two inbox +tables, compiled their real PostgreSQL DDL and old-source SQLAlchemy DML, then +invoked the actual c4d5e6f7a8b9 migration through Alembic offline Operations. +It imported no application config, database module or pytest conftest and opened +no database/network connection. Rebuilding requires PYTHONHASHSEED=0 to +provide a repeatable Python process baseline. SQLAlchemy also orders independent +FK/index objects by identity, so the builder canonicalizes only independent FK +clause order and emits sorted indexes after dependency-ordered compiled tables. +Two failed repeat-byte checks are retained before that normalization fix. Source and generated SQL hashes are bound by +manifest.json. No dependency install or workflow change was introduced. + +The proposed driver reuses the frozen current group fixture utility source and +canonical isolated-keycape containment. It pins the same cached PostgreSQL and +UserEngine helper images, fixed daemon ID/name/socket, nonroot users, readonly +rootfs, dropped capabilities, no privilege gain, none/shared loopback network, +no host ports/mounts/retained volumes, private bounded tmpfs, no logs/restart, +finite lifetimes/deadline and exact-name/label cleanup. PostgreSQL declared +volumes are explicitly masked. Generated password is private stdin/tmpfs only. +The helper must be exact /usr/local/bin/python3 with psycopg3.3.6. No image +pull/build/push, package download, real credential or production API is allowed. + +The phase would create only the old inbox table schema, apply exact generated +migration SQL, then submit old compiled insert/receipt/update/delete shapes. +It checks revision backfill/write triggers, transaction-shared revisions, no-op +stability and tombstones while leaving the additive schema in place. Receipt +contains closed booleans, synthetic timing, safe error class/SQLSTATE and exact +session/cleanup metadata; no raw Docker/PostgreSQL output. The driver defaults +to plan and has not been invoked with its execution flag. Root must review the +exact manifest and code before any disposable Docker call. + +This is a narrow SQL compatibility fixture. It does not prove complete old/new +HTTP/ORM behavior, concurrent production feed behavior, real schema state, +large-data lock/runtime, service ownership or backup/PITR/recovery authority. +The existing full feed tests must only run inside separately reviewed exact +disposable database containment, never their default destructive local URL. +Helm rollback leaves the migration installed; no automatic schema downgrade is +proposed. A hard parent-loss absence/reaper proof is outside this fixture; normal +success or failure must retain exact cleanup receipts before claiming absence. diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/build-offline-sql.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/build-offline-sql.py new file mode 100644 index 00000000..b9cef313 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/build-offline-sql.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Generate SQL from exact old ORM and new migration; no config/DB connection.""" +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import sys +import types +from sqlalchemy import create_mock_engine, insert, update, delete, func +from sqlalchemy.dialects import postgresql +from alembic.migration import MigrationContext +from alembic.operations import Operations +HERE=Path(__file__).resolve().parent + +def load(name,path): + spec=importlib.util.spec_from_file_location(name,path);mod=importlib.util.module_from_spec(spec) + sys.modules[name]=mod;spec.loader.exec_module(mod);return mod + +def build(): + if os.environ.get("PYTHONHASHSEED") != "0": + raise RuntimeError("offline_builder_requires_fixed_python_hash_seed_0") + # Isolated source namespace: never import api.config/database or conftest. + for name in ('api','api.models'): + module=types.ModuleType(name);module.__path__=[];sys.modules[name]=module + oldbase=load('api.models.base',HERE/'old-base.py') + old=load('api.models.agent_message',HERE/'old-agent-message.py') + ddl=[] + engine=create_mock_engine('postgresql://',lambda sql,*args,**kwargs:ddl.append(str(sql.compile(dialect=engine.dialect)))) + oldbase.Base.metadata.create_all(engine) + migration=load('reviewed_inbox_migration',HERE/'migration.py') + output=io.StringIO() + migration.op=Operations(MigrationContext.configure(dialect_name='postgresql',opts={'as_sql':True,'output_buffer':output})) + migration.upgrade() + # SQLAlchemy orders otherwise equivalent FK objects/index sets by identity. + # Normalize only independent FK clause order and place sorted indexes after + # the original dependency-ordered tables; every clause remains compiled ORM SQL. + tables=[];indexes=[] + for statement in ddl: + lines=statement.splitlines() + positions=[i for i,line in enumerate(lines) if line.lstrip().startswith('FOREIGN KEY(')] + values=sorted(lines[i].rstrip().rstrip(',') for i in positions) + for i,value in zip(positions,values): + ending=', ' if lines[i].endswith(', ') else ',' if lines[i].endswith(',') else '' + lines[i]=value+ending + normalized='\n'.join(lines) + (tables if statement.lstrip().startswith('CREATE TABLE') else indexes).append(normalized) + (HERE/'old-schema.sql').write_text(';\n'.join(tables+sorted(indexes))+';\n') + (HERE/'upgrade.sql').write_text(output.getvalue()) + # Compile real old-source SQLAlchemy write shapes; values are synthetic. + msg='11111111-1111-4111-8111-111111111111' + statements={ + 'insert_message':insert(old.AgentMessage.__table__).values(id=msg,from_agent='fixture-sender',to_agent='fixture-reader',subject='synthetic',body='synthetic',kind='message'), + 'insert_receipt':insert(old.MessageReceipt.__table__).values(message_id=msg,agent='fixture-reader',delivered_at=func.now()), + 'update_message':update(old.AgentMessage.__table__).where(old.AgentMessage.id==msg).values(read_at=func.now()), + 'delete_message':delete(old.AgentMessage.__table__).where(old.AgentMessage.id==msg), + } + writes={} + for name,statement in statements.items(): + compiled=statement.compile(dialect=postgresql.dialect()) + writes[name]={'sql':str(compiled),'params':compiled.params} + (HERE/'old-writes.json').write_text(json.dumps(writes,indent=2)+'\n') + return {name:'sha256:'+hashlib.sha256((HERE/name).read_bytes()).hexdigest() for name in ['old-schema.sql','upgrade.sql','old-writes.json']} +if __name__=='__main__':print(json.dumps(build(),indent=2)) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/diagnostic_protocol.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/diagnostic_protocol.py new file mode 100644 index 00000000..d6f8107a --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/diagnostic_protocol.py @@ -0,0 +1,86 @@ +"""Closed public fixture diagnostics. Never serialize exception text or SQL values.""" +import json +import re + +DRIVER_STAGES = ('session_registered', 'anchor_launch', 'anchor_containment', 'postgres_private_import', + 'postgres_initdb', 'postgres_launch', 'anchor_identity', 'helper_launch', 'full_containment', + 'helper_interpreter', 'candidate_import', 'bootstrap', 'restart', 'concurrency', 'fork', 'lost-ack', + 'restored', 'database_dump', 'database_restore', 'cleanup', 'complete') +PROBE_STAGES = ('probe_entry', 'loopback_check', 'phase_dispatch', 'bootstrap_connect_wait', + 'bootstrap_connected', 'bootstrap_schema', 'bootstrap_migration', 'bootstrap_epoch_insert', + 'bootstrap_role_create', 'bootstrap_runtime_grants', 'bootstrap_checkpoint_init', + 'bootstrap_directory_init', 'bootstrap_ensure', 'bootstrap_ensure_readback', + 'bootstrap_runtime_ddl_refusal', 'bootstrap_complete', 'phase_complete') +ERROR_CLASSES = ('GroupProvenanceUnavailable', 'RuntimeError', 'ValueError', 'TypeError', 'KeyError', + 'FileNotFoundError', 'PermissionError', 'OperationalError', 'ProgrammingError', 'IntegrityError', + 'InsufficientPrivilege', 'InvalidPassword', 'UndefinedTable', 'UndefinedColumn', 'SyntaxError', + 'DuplicateTable', 'DuplicateObject', 'InvalidSchemaName', 'DeadlockDetected', 'ConnectionException', + 'InvalidAuthorizationSpecification', 'OSError', 'TimeoutExpired', 'ImportError', + 'ModuleNotFoundError', 'JSONDecodeError', 'OtherError') + + +def error_metadata(error, stage): + if stage not in DRIVER_STAGES + PROBE_STAGES: + stage = 'phase_dispatch' + name = type(error).__name__ + name = name if name in ERROR_CLASSES else 'OtherError' + state = None + current = error + for _ in range(8): + if current is None: + break + try: + candidate = getattr(current, 'sqlstate', None) + if isinstance(candidate, str) and re.fullmatch('[0-9A-Z]{5}', candidate): + state = candidate + break + current = current.__cause__ or current.__context__ + except Exception: + break + return {'stage': stage, 'exception_class': name, 'sqlstate': state} + + +def emit_stage(stage): + if stage not in PROBE_STAGES: + raise RuntimeError('invalid_public_fixture_stage') + print(json.dumps({'event': 'stage', 'stage': stage}), flush=True) + + +def parse_frames(raw, *, require_final=True): + if isinstance(raw, bytes): + raw = raw.decode('utf-8') + if not isinstance(raw, str) or len(raw) > 32768: + raise RuntimeError('invalid_public_fixture_diagnostics') + stages, final = [], None + lines = raw.splitlines() + if len(lines) > 96: + raise RuntimeError('invalid_public_fixture_diagnostics') + for line in lines: + value = json.loads(line) + if not isinstance(value, dict) or final is not None: + raise RuntimeError('invalid_public_fixture_diagnostics') + if value.get('event') == 'stage': + if set(value) != {'event', 'stage'} or value['stage'] not in PROBE_STAGES: + raise RuntimeError('invalid_public_fixture_diagnostics') + stages.append(value['stage']) + elif value.get('status') in {'passed', 'failed'} and final is None: + if value['status'] == 'failed': + if set(value) != {'status', 'failure', 'diagnostic'} or value['failure'] != 'synthetic_pg_oracle_failed': + raise RuntimeError('invalid_public_fixture_diagnostics') + diagnostic = value['diagnostic'] + if set(diagnostic) != {'stage', 'exception_class', 'sqlstate'} or diagnostic['stage'] not in PROBE_STAGES or diagnostic['exception_class'] not in ERROR_CLASSES: + raise RuntimeError('invalid_public_fixture_diagnostics') + if diagnostic['sqlstate'] is not None and not re.fullmatch('[0-9A-Z]{5}', str(diagnostic['sqlstate'])): + raise RuntimeError('invalid_public_fixture_diagnostics') + elif (set(value) != {'status', 'checks', 'psycopg_version', 'active_database'} + or not isinstance(value['checks'], dict) + or not isinstance(value['psycopg_version'], str) + or not re.fullmatch(r'3\.[0-9]+\.[0-9]+(?:[a-z]+[0-9]+)?', value['psycopg_version']) + or value['active_database'] not in {'postgres', 'fixture_fork'}): + raise RuntimeError('invalid_public_fixture_diagnostics') + final = value + else: + raise RuntimeError('invalid_public_fixture_diagnostics') + if require_final and final is None: + raise RuntimeError('missing_public_fixture_result') + return stages, final diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/driver.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/driver.py new file mode 100644 index 00000000..1731b546 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/driver.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Default plan only; exact independent review required before disposable Docker.""" +import argparse +import base64 +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import re +import secrets +import tempfile +import time +import uuid +HERE=Path(__file__).resolve().parent +MANIFEST=HERE/'manifest.json' +spec=importlib.util.spec_from_file_location('reviewed_pg_containment_driver',HERE/'reviewed-driver-base.py') +base=importlib.util.module_from_spec(spec);spec.loader.exec_module(base) +require=base.require +CHECKS={'only_loopback_no_external_routes','actual_old_model_schema_created','actual_new_migration_sql_applied','legacy_rows_backfilled_one_revision', + 'old_compiled_insert_accepted_on_new_schema','old_compiled_receipt_same_transaction_revision', + 'old_compiled_update_stamped_revision','old_noop_update_revision_unchanged', + 'old_compiled_delete_writes_tombstone','no_application_schema_downgrade'} +IMPORT=""" +import base64,json,os,sys +from pathlib import Path +os.umask(0o077) +root=Path('/fixture') +allowed={'probe.py','old-schema.sql','upgrade.sql','old-writes.json','private.json'} +files=json.load(sys.stdin) +if set(files)!=allowed:raise ValueError() +for name,encoded in files.items(): + fd=os.open(root/name,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600) + with os.fdopen(fd,'wb') as target:target.write(base64.b64decode(encoded,validate=True)) +""" + +def execute(manifest,adapter,result,checkpoint): + def publish(): + try:checkpoint() + except OSError:result['receipt_checkpoint_failed']=True + session=base.session_plan(manifest,uuid.uuid4().hex) + result['session']={'run_id':session['run_id'],'containers':session['containers'],'label':'net-kingdom.recovery-run='+session['run_id']} + original=adapter.command;deadline=time.monotonic()+180 + def run(argv,**kwargs): + remaining=deadline-time.monotonic();require(remaining>0) + try:return original(argv,timeout=min(40,remaining),**kwargs) + except BaseException: + result['daemon_request_uncertain']=True;publish();raise + def stage(name):result['stage']=name;publish() + adapter.command=run + try: + anchor=session['worker'];helper=session['issuer'];python=base.helper_python(manifest) + stage('postgres_launch') + args=base.base_args(session,anchor,'none',user='999:999')+[ + '--tmpfs','/var/lib/postgresql:rw,nosuid,nodev,noexec,size=1048576,uid=999,gid=999,mode=0700', + '--tmpfs','/var/lib/postgresql/data:rw,nosuid,nodev,noexec,size=134217728,uid=999,gid=999,mode=0700', + '--tmpfs','/run/fixture:rw,nosuid,nodev,noexec,size=1048576,uid=999,gid=999,mode=0700', + '--env','PGDATA=/var/lib/postgresql/data','--entrypoint','/bin/sh',manifest['postgres_image'],'-c','sleep 240'] + run(args);adapter.inspect_session({**session,'containers':[anchor],'issuer':anchor},worker_only=True) + password=secrets.token_hex(24) + run(['docker','exec','-i',anchor,'tar','-xf','-','-C','/run/fixture'],input=base.tar_bytes({'password':password.encode()})) + stage('initdb') + raw=run(['docker','exec',anchor,'/bin/sh','-c',"if initdb -D /var/lib/postgresql/data --username=fixture_admin --auth-local=trust --auth-host=scram-sha-256 --pwfile=/run/fixture/password > /dev/null 2>&1; then echo '{\"ready\":true}'; else echo '{\"ready\":false}'; fi"],max_output=256) + require(json.loads(raw)=={'ready':True}) + run(['docker','exec','--detach',anchor,'/bin/sh','-c','exec postgres -D /var/lib/postgresql/data -c listen_addresses=127.0.0.1 -c unix_socket_directories=/run/fixture > /dev/null 2>&1']) + ready=False + for _ in range(15): + raw=run(['docker','exec',anchor,'/bin/sh','-c',"if pg_isready --host /run/fixture -U fixture_admin > /dev/null 2>&1; then echo '{\"ready\":true}'; else echo '{\"ready\":false}'; fi"],max_output=256) + if json.loads(raw)=={'ready':True}:ready=True;break + time.sleep(0.2) + require(ready) + anchor_id=run(['docker','container','inspect',anchor,'--format','{{.Id}}']).decode().strip();require(bool(re.fullmatch('[a-f0-9]{64}',anchor_id))) + stage('helper_launch') + args=base.base_args(session,helper,'container:'+anchor_id)+['--tmpfs','/fixture:rw,nosuid,nodev,noexec,size=16777216,uid=1000,gid=1000,mode=0700','--entrypoint',python,manifest['python_image'],'-c','import time;time.sleep(240)'] + run(args);adapter.inspect_session(session,worker_only=True) + interpreter=json.loads(run(['docker','exec',helper,python,'-c',base.INTERPRETER_PROBE],max_output=4096)) + require(interpreter.get('helper_ready') is True and interpreter.get('executable')==python and interpreter.get('psycopg_version')=='3.3.6') + result['containment_verified_before_private_import']=True;publish() + files={name:(HERE/name).read_bytes() for name in ('probe.py','old-schema.sql','upgrade.sql','old-writes.json')} + files['private.json']=json.dumps({'admin_password':password}).encode() + stage('private_import') + run(['docker','exec','-i',helper,python,'-c',IMPORT],input=json.dumps({k:base64.b64encode(v).decode() for k,v in files.items()}).encode()) + stage('migration_compatibility') + value=json.loads(run(['docker','exec',helper,python,'/fixture/probe.py'],max_output=4096)) + if value.get('status')!='passed': + allowed_stages={'network_boundary','old_schema','new_migration','old_insert_receipt','old_update','old_delete'} + allowed_errors={'AssertionError','OperationalError','ProgrammingError','UniqueViolation','NotNullViolation','UndefinedColumn','UndefinedTable','SyntaxError','InsufficientPrivilege','unclassified'} + code=value.get('sqlstate') + result['diagnostic']={'stage':value.get('stage') if value.get('stage') in allowed_stages else 'unclassified', + 'error_class':value.get('error_class') if value.get('error_class') in allowed_errors else 'unclassified', + 'sqlstate':code if isinstance(code,str) and re.fullmatch('[0-9A-Z]{5}',code) else None} + publish();require(False) + require(set(value.get('checks',{}))==CHECKS and all(v is True for v in value['checks'].values())) + result['checks']=value['checks'];result['migration_seconds_synthetic_only']=value['migration_seconds_synthetic_only'];result['status']='synthetic_migration_old_write_shapes_passed' + finally: + stage('cleanup');adapter.command=lambda argv,**kwargs:original(argv,timeout=15,**kwargs) + try:adapter.cleanup(session,result);result['container_cleanup_verified']=True + except BaseException:result['container_cleanup_verified']=False;result['status']='failed' + adapter.command=original + if result.get('daemon_request_uncertain') or result.get('receipt_checkpoint_failed'):result['status']='failed' + publish() + +def main(): + ap=argparse.ArgumentParser();ap.add_argument('--execute-reviewed-fixture',action='store_true');ap.add_argument('--expected-manifest-sha256');args=ap.parse_args() + if not args.execute_reviewed_fixture:print(MANIFEST.read_text());return 0 + manifest=base.reviewed_manifest(args.expected_manifest_sha256) + adapter=base.load_containment(manifest);base.preflight(adapter,manifest) + os.umask(0o077);output=Path(tempfile.mkdtemp(prefix='nk-statehub-migration-compat-',dir='/tmp'))/'result.json' + result={'schema':'statehub.synthetic-migration-compat.v1','status':'failed','production_contacted':False,'actual_credentials_used':False,'native_acceptance_claimed':False,'manifest_sha256':args.expected_manifest_sha256} + def publish():output.write_text(json.dumps(result,indent=2)+'\n') + try:execute(manifest,adapter,result,publish) + except BaseException:result['status']='failed' + finally:publish() + print(json.dumps({'status':result['status'],'safe_result':str(output)}));return int(result['status']!='synthetic_migration_old_write_shapes_passed') +if __name__=='__main__':raise SystemExit(main()) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/manifest.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/manifest.json new file mode 100644 index 00000000..e5e9c750 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/manifest.json @@ -0,0 +1,64 @@ +{ + "schema": "statehub.synthetic-migration-compat-proposal.v1", + "stable_proposal_id": "STATE-WP-0095-T06-cluster-frozen-receiving-r1", + "revision": "migration-compatibility-1", + "status": "prepared_not_executed_not_approved", + "containment_helper": "/home/worsch/net-kingdom/tools/bootstrap-recovery/isolated-keycape.py", + "fixed_endpoint": "unix:///var/run/docker.sock", + "engine_id": "74f93783-6cef-43a8-a3c4-f62586cd3bc1", + "engine_name": "bnt-lap001", + "postgres_image": "postgres@sha256:ff23cdce56cac62ada6f66013e1a50864c0abbe79d132d40b6e05bd80f378a70", + "python_image": "forgejo.coulomb.social/coulomb/user-engine@sha256:f239bbf332b8eb87320c86390fddca1dcb4f7c628a7af75e79dee8058d446a7b", + "helper_interpreter": "/usr/local/bin/python3", + "source_sha256": { + "/home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/README.md": "sha256:159c12b98fd58a0d066e39f36ff2a100c5f0fc3596a700207957d6ff6d0f00c1", + "/home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/build-offline-sql.py": "sha256:cb49052b6836e7b67807aaade1b1f26f7471d6e1bd8be2333281ab8a9d0cafc9", + "/home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/diagnostic_protocol.py": "sha256:d141fc00e18906bcdbf6cf5542e219580668daaaced6c57cbab179132f7c6a61", + "/home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/driver.py": "sha256:9e53c74e0c4bc74378113f97fca8a190026100418ea6cf89335a5df5b2170970", + "/home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/migration.py": "sha256:48f1730037be99b5623e878d7f20a06bec243ce27317fde81c82e16006d7b3ce", + "/home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-agent-message.py": "sha256:4bbbed06d4ae17bcbf86c34d02a684bd94b98d15bd9c2205700a524923e76e08", + "/home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-base.py": "sha256:7a9bf19021a2ea53abe40adc06b478ebbba0249e73e202f2c7a30b2d0e5c73d7", + "/home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-schema.sql": "sha256:4f95abdbe3942e7dbb8ad056cd6717de86f7e03d2d08a912f0a460b188727259", + "/home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-writes.json": "sha256:67910bf4341fd98cfcddf0cf84f1d93663daf5d04f054140e53f21326b293a88", + "/home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/probe.py": "sha256:d830a0cea7cafb66d7be97f83e7adca18b8f2b0694b572b11182cc3950f576f1", + "/home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/reviewed-driver-base.py": "sha256:04237939df1887df8a201181d87ebfacfa25855401b64f4998a3921111bd91eb", + "/home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/test_driver.py": "sha256:399326de64527b0006668ea841f8f41bd1d747e027cea3adec2581ebf7e7d994", + "/home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/upgrade.sql": "sha256:8336c253ec951fb93ed03db39b8da7f958daecdfaa83afa2454f8be7645fe6c1", + "/home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/validation.json": "sha256:0e0f061c68be85fb50589d0815a5da5ca64ea372ba380115560e8198adf693a8", + "/home/worsch/net-kingdom/tools/bootstrap-recovery/isolated-keycape.py": "sha256:f67627b7d1647d75187dd899874efe22383bc91de5d97dabb9801076668ab3c2" + }, + "limits": { + "containers": 2, + "host_ports": 0, + "host_mounts": 0, + "named_or_anonymous_volumes": 0, + "memory_each_bytes": 402653184, + "cpus_each": 0.5, + "pids_each": 64, + "postgres_data_tmpfs_bytes": 134217728, + "helper_tmpfs_bytes": 16777216, + "container_lifetime_seconds": 240, + "driver_deadline_seconds": 180, + "rpc_timeout_seconds": 40, + "cleanup_rpc_timeout_seconds": 15, + "image_pull": "never", + "cleanup_total_bound_seconds": 120 + }, + "scope": "Actual new migration SQL from Alembic offline generation on actual old-model DDL and compiled old-source SQLAlchemy DML in disposable PostgreSQL. No full old/new app HTTP/ORM flush or production timing proof.", + "not_claimed": [ + "actual production schema/data compatibility", + "full old/new application or inbox-feed contract acceptance", + "real production migration runtime/lock budget", + "durable backup/PITR/rollback authority", + "native original TLS/KeyCape recovery acceptance", + "hard-parent-loss cleanup proof" + ], + "execution_command": [ + "python3", + "/home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/driver.py", + "--execute-reviewed-fixture", + "--expected-manifest-sha256", + "" + ], + "offline_builder_precondition": "PYTHONHASHSEED=0; exact generated SQL/writes bound; no regeneration during Docker execution" +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/migration.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/migration.py new file mode 100644 index 00000000..79965918 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/migration.py @@ -0,0 +1,147 @@ +"""inbox change-feed revisions and tombstones (STATE-WP-0098-T05) + +Implements §2 of docs/inbox-change-feed-contract.md (accepted 2026-10-13, D4): + +- sequence ``inbox_revision_seq``; +- ``revision bigint NOT NULL`` on ``agent_messages`` and ``message_receipts``, + backfilled with ONE sequence value (one snapshot = one revision), indexed; +- ``message_tombstones``; +- a BEFORE INSERT OR UPDATE trigger on both tables that skips no-op updates, + takes a transaction-scoped advisory lock (commit order = revision order) and + assigns one revision per transaction (cached per txid in a txn-local GUC); +- a BEFORE DELETE trigger on ``agent_messages`` that writes a tombstone. + +The SQL is a frozen copy of ``api.models.agent_message.INBOX_FEED_FUNCTIONS_SQL``; +tests/test_message_feed.py checks the two agree. + +Revision ID: c4d5e6f7a8b9 +Revises: b3c4d5e6f7a8 +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import UUID + +revision = "c4d5e6f7a8b9" +down_revision = "b3c4d5e6f7a8" +branch_labels = None +depends_on = None + +INBOX_ADVISORY_LOCK_KEY = 5_348_490_098 + +FUNCTIONS_SQL = f""" +CREATE OR REPLACE FUNCTION inbox_feed_revision() RETURNS bigint +LANGUAGE plpgsql AS $fn$ +DECLARE + tx text := txid_current()::text; + cached text := coalesce(current_setting('statehub.inbox_rev', true), ''); + rev bigint; +BEGIN + PERFORM pg_advisory_xact_lock({INBOX_ADVISORY_LOCK_KEY}); + IF split_part(cached, ':', 1) = tx THEN + RETURN split_part(cached, ':', 2)::bigint; + END IF; + rev := nextval('inbox_revision_seq'); + PERFORM set_config('statehub.inbox_rev', tx || ':' || rev::text, true); + RETURN rev; +END +$fn$; + +CREATE OR REPLACE FUNCTION inbox_feed_stamp_revision() RETURNS trigger +LANGUAGE plpgsql AS $fn$ +BEGIN + IF TG_OP = 'UPDATE' THEN + IF NEW IS NOT DISTINCT FROM OLD THEN + RETURN NEW; + END IF; + END IF; + NEW.revision := inbox_feed_revision(); + RETURN NEW; +END +$fn$; + +CREATE OR REPLACE FUNCTION inbox_feed_tombstone() RETURNS trigger +LANGUAGE plpgsql AS $fn$ +BEGIN + INSERT INTO message_tombstones (message_id, revision, deleted_at, reason) + VALUES ( + OLD.id, + inbox_feed_revision(), + now(), + nullif(current_setting('statehub.delete_reason', true), '') + ) + ON CONFLICT (message_id) DO UPDATE + SET revision = EXCLUDED.revision, + deleted_at = EXCLUDED.deleted_at, + reason = EXCLUDED.reason; + RETURN OLD; +END +$fn$; +""" + +TRIGGERS_SQL = """ +CREATE TRIGGER agent_messages_inbox_revision + BEFORE INSERT OR UPDATE ON agent_messages + FOR EACH ROW EXECUTE FUNCTION inbox_feed_stamp_revision(); +CREATE TRIGGER agent_messages_inbox_tombstone + BEFORE DELETE ON agent_messages + FOR EACH ROW EXECUTE FUNCTION inbox_feed_tombstone(); +CREATE TRIGGER message_receipts_inbox_revision + BEFORE INSERT OR UPDATE ON message_receipts + FOR EACH ROW EXECUTE FUNCTION inbox_feed_stamp_revision(); +""" + +_TABLES = ("agent_messages", "message_receipts") + + +def upgrade() -> None: + op.execute(sa.text("CREATE SEQUENCE inbox_revision_seq AS bigint")) + for table in _TABLES: + op.add_column(table, sa.Column("revision", sa.BigInteger(), nullable=True)) + # One sequence value for all existing rows: the pre-feed state is one revision. + op.execute( + sa.text( + """ + DO $$ + DECLARE base bigint := nextval('inbox_revision_seq'); + BEGIN + UPDATE agent_messages SET revision = base; + UPDATE message_receipts SET revision = base; + END + $$ + """ + ) + ) + for table in _TABLES: + op.alter_column(table, "revision", nullable=False) + op.create_index(f"ix_{table}_revision", table, ["revision"]) + op.create_table( + "message_tombstones", + sa.Column("message_id", UUID(as_uuid=True), primary_key=True), + sa.Column("revision", sa.BigInteger(), nullable=False), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("reason", sa.Text(), nullable=True), + ) + op.create_index("ix_message_tombstones_revision", "message_tombstones", ["revision"]) + op.execute(sa.text(FUNCTIONS_SQL)) + op.execute(sa.text(TRIGGERS_SQL)) + + +def downgrade() -> None: + op.execute( + sa.text( + """ + DROP TRIGGER IF EXISTS message_receipts_inbox_revision ON message_receipts; + DROP TRIGGER IF EXISTS agent_messages_inbox_tombstone ON agent_messages; + DROP TRIGGER IF EXISTS agent_messages_inbox_revision ON agent_messages; + DROP FUNCTION IF EXISTS inbox_feed_tombstone(); + DROP FUNCTION IF EXISTS inbox_feed_stamp_revision(); + DROP FUNCTION IF EXISTS inbox_feed_revision(); + """ + ) + ) + op.drop_index("ix_message_tombstones_revision", table_name="message_tombstones") + op.drop_table("message_tombstones") + for table in reversed(_TABLES): + op.drop_index(f"ix_{table}_revision", table_name=table) + op.drop_column(table, "revision") + op.execute(sa.text("DROP SEQUENCE IF EXISTS inbox_revision_seq")) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-agent-message.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-agent-message.py new file mode 100644 index 00000000..2ef53909 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-agent-message.py @@ -0,0 +1,90 @@ +import uuid +from datetime import datetime + +from sqlalchemy import DateTime, ForeignKey, Index, String, Text, text +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from api.models.base import Base, new_uuid + + +BROADCAST = "broadcast" +MESSAGE_KINDS = ("message", "news", "standing") + + +class AgentMessage(Base): + __tablename__ = "agent_messages" + __table_args__ = ( + Index("ix_agent_messages_to_kind_archived", "to_agent", "kind", "archived_at"), + ) + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=new_uuid + ) + from_agent: Mapped[str] = mapped_column(String(100), nullable=False) + to_agent: Mapped[str] = mapped_column(String(100), nullable=False, index=True) + subject: Mapped[str] = mapped_column(String(500), nullable=False) + body: Mapped[str] = mapped_column(Text, nullable=False) + thread_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), + ForeignKey("agent_messages.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + read_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + archived_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + # STATE-WP-0093: kind is only meaningful for broadcasts + # (message | news | standing); direct messages stay "message". + kind: Mapped[str] = mapped_column( + String(20), nullable=False, default="message", server_default="message" + ) + expires_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + supersedes_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), + ForeignKey("agent_messages.id", ondelete="SET NULL"), + nullable=True, + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=text("now()"), + nullable=False, + ) + + thread_root: Mapped["AgentMessage | None"] = relationship( + "AgentMessage", + remote_side="AgentMessage.id", + foreign_keys=[thread_id], + lazy="select", + ) + + +class MessageReceipt(Base): + """Per-reader state for broadcast messages (STATE-WP-0093). + + Written only for broadcasts; direct messages keep the global + ``AgentMessage.read_at``. + """ + + __tablename__ = "message_receipts" + + message_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("agent_messages.id", ondelete="CASCADE"), + primary_key=True, + ) + agent: Mapped[str] = mapped_column(String(100), primary_key=True, index=True) + delivered_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + read_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + acknowledged_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-base.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-base.py new file mode 100644 index 00000000..578c4d50 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-base.py @@ -0,0 +1,53 @@ +import os +import time +import uuid +from datetime import datetime + +from sqlalchemy import DateTime, func +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + + +class Base(DeclarativeBase): + pass + + +class TimestampMixin: + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now(), + onupdate=func.now(), + nullable=False, + ) + + +def new_uuid() -> uuid.UUID: + return uuid.uuid4() + + +def new_uuid7() -> uuid.UUID: + """Generate a UUIDv7 (RFC 9562): 48-bit big-endian ms timestamp, version + and variant bits, remaining bits random. Time-sortable, so primary keys + generated with this helper order chronologically without a separate + created_at index lookup — the identity layering canon + (work-record-types_v0.1.md) calls this out as the primary internal key + for new work-record entities. + + Dependency-free (no uuid7 in stdlib before Python 3.14, no third-party + lib added for a ~15-line, non-cryptographic layout). + """ + unix_ts_ms = int(time.time() * 1000) + rand = int.from_bytes(os.urandom(10), "big") + rand_a = (rand >> 62) & 0x0FFF # top 12 bits of the 80 random bits + rand_b = rand & 0x3FFFFFFFFFFFFFFF # bottom 62 bits + value = ( + (unix_ts_ms << 80) + | (0x7 << 76) # version 7 + | (rand_a << 64) + | (0x2 << 62) # variant 10 + | rand_b + ) + return uuid.UUID(int=value) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-schema.sql b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-schema.sql new file mode 100644 index 00000000..b365cfdb --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-schema.sql @@ -0,0 +1,34 @@ + +CREATE TABLE agent_messages ( + id UUID NOT NULL, + from_agent VARCHAR(100) NOT NULL, + to_agent VARCHAR(100) NOT NULL, + subject VARCHAR(500) NOT NULL, + body TEXT NOT NULL, + thread_id UUID, + read_at TIMESTAMP WITH TIME ZONE, + archived_at TIMESTAMP WITH TIME ZONE, + kind VARCHAR(20) DEFAULT 'message' NOT NULL, + expires_at TIMESTAMP WITH TIME ZONE, + supersedes_id UUID, + created_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL, + PRIMARY KEY (id), + FOREIGN KEY(supersedes_id) REFERENCES agent_messages (id) ON DELETE SET NULL, + FOREIGN KEY(thread_id) REFERENCES agent_messages (id) ON DELETE SET NULL +) +; + +CREATE TABLE message_receipts ( + message_id UUID NOT NULL, + agent VARCHAR(100) NOT NULL, + delivered_at TIMESTAMP WITH TIME ZONE, + read_at TIMESTAMP WITH TIME ZONE, + acknowledged_at TIMESTAMP WITH TIME ZONE, + PRIMARY KEY (message_id, agent), + FOREIGN KEY(message_id) REFERENCES agent_messages (id) ON DELETE CASCADE +) +; +CREATE INDEX ix_agent_messages_thread_id ON agent_messages (thread_id); +CREATE INDEX ix_agent_messages_to_agent ON agent_messages (to_agent); +CREATE INDEX ix_agent_messages_to_kind_archived ON agent_messages (to_agent, kind, archived_at); +CREATE INDEX ix_message_receipts_agent ON message_receipts (agent); diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-writes.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-writes.json new file mode 100644 index 00000000..2f1b232a --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/old-writes.json @@ -0,0 +1,32 @@ +{ + "insert_message": { + "sql": "INSERT INTO agent_messages (id, from_agent, to_agent, subject, body, kind) VALUES (%(id)s::UUID, %(from_agent)s, %(to_agent)s, %(subject)s, %(body)s, %(kind)s)", + "params": { + "id": "11111111-1111-4111-8111-111111111111", + "from_agent": "fixture-sender", + "to_agent": "fixture-reader", + "subject": "synthetic", + "body": "synthetic", + "kind": "message" + } + }, + "insert_receipt": { + "sql": "INSERT INTO message_receipts (message_id, agent, delivered_at) VALUES (%(message_id)s::UUID, %(agent)s, now())", + "params": { + "message_id": "11111111-1111-4111-8111-111111111111", + "agent": "fixture-reader" + } + }, + "update_message": { + "sql": "UPDATE agent_messages SET read_at=now() WHERE agent_messages.id = %(id_1)s::UUID", + "params": { + "id_1": "11111111-1111-4111-8111-111111111111" + } + }, + "delete_message": { + "sql": "DELETE FROM agent_messages WHERE agent_messages.id = %(id_1)s::UUID", + "params": { + "id_1": "11111111-1111-4111-8111-111111111111" + } + } +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/probe.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/probe.py new file mode 100644 index 00000000..b4b12446 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/probe.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Only disposable fixture DSN generated by contained driver; no env fallback.""" +import json +from pathlib import Path +import re +import time +import psycopg +ROOT=Path('/fixture') +CHECKS=('only_loopback_no_external_routes','actual_old_model_schema_created','actual_new_migration_sql_applied','legacy_rows_backfilled_one_revision', + 'old_compiled_insert_accepted_on_new_schema','old_compiled_receipt_same_transaction_revision', + 'old_compiled_update_stamped_revision','old_noop_update_revision_unchanged', + 'old_compiled_delete_writes_tombstone','no_application_schema_downgrade') + +def main(): + stage='network_boundary';checks={} + try: + routes=Path('/proc/net/route').read_text().splitlines()[1:] + assert all(line.split()[0]=='lo' for line in routes) + checks[CHECKS[0]]=True + private=json.loads((ROOT/'private.json').read_text()) + assert set(private)=={'admin_password'} and re.fullmatch('[0-9a-f]{48}',private['admin_password']) + writes=json.loads((ROOT/'old-writes.json').read_text()) + with psycopg.connect(host='127.0.0.1',port=5432,dbname='postgres',user='fixture_admin',password=private['admin_password'],connect_timeout=10) as conn: + stage='old_schema' + conn.execute((ROOT/'old-schema.sql').read_text());conn.commit() + checks[CHECKS[1]]=True + legacy='22222222-2222-4222-8222-222222222222' + current_id='11111111-1111-4111-8111-111111111111' + for name,key in (('insert_message','id'),('insert_receipt','message_id')): + value=writes[name];params=dict(value['params']);params[key]=legacy;conn.execute(value['sql'],params) + conn.commit() + stage='new_migration';start=time.monotonic() + conn.execute((ROOT/'upgrade.sql').read_text());conn.commit() + elapsed=round(time.monotonic()-start,6);checks[CHECKS[2]]=True + backfill=conn.execute('SELECT revision FROM agent_messages UNION ALL SELECT revision FROM message_receipts').fetchall() + assert len(backfill)==2 and backfill[0][0]==backfill[1][0] and backfill[0][0]>0 + checks[CHECKS[3]]=True;conn.commit() + stage='old_insert_receipt' + for name in ('insert_message','insert_receipt'): + value=writes[name];conn.execute(value['sql'],value['params']) + revisions=conn.execute('SELECT revision FROM agent_messages WHERE id=%s UNION ALL SELECT revision FROM message_receipts WHERE message_id=%s',(current_id,current_id)).fetchall() + assert len(revisions)==2 and revisions[0][0]>0 + checks[CHECKS[4]]=True;assert revisions[0][0]==revisions[1][0];checks[CHECKS[5]]=True + conn.commit();prior=revisions[0][0] + stage='old_update';value=writes['update_message'];conn.execute(value['sql'],value['params']) + current=conn.execute('SELECT revision FROM agent_messages WHERE id=%s',(current_id,)).fetchone()[0] + assert current>prior;conn.commit();checks[CHECKS[6]]=True + conn.execute('UPDATE agent_messages SET subject=subject') + assert conn.execute('SELECT revision FROM agent_messages WHERE id=%s',(current_id,)).fetchone()[0]==current + conn.commit();checks[CHECKS[7]]=True + stage='old_delete';value=writes['delete_message'];conn.execute(value['sql'],value['params']) + assert conn.execute('SELECT count(*) FROM agent_messages WHERE id=%s',(current_id,)).fetchone()[0]==0 + tombstone=conn.execute('SELECT revision FROM message_tombstones WHERE message_id=%s',(current_id,)).fetchone()[0] + assert tombstone>current;conn.commit();checks[CHECKS[8]]=True + assert conn.execute("SELECT count(*) FROM information_schema.columns WHERE table_name='agent_messages' AND column_name='revision'").fetchone()[0]==1 + checks[CHECKS[9]]=True + print(json.dumps({'status':'passed','checks':checks,'migration_seconds_synthetic_only':elapsed,'psycopg_version':psycopg.__version__})) + return 0 + except BaseException as error: + allowed={'AssertionError','OperationalError','ProgrammingError','UniqueViolation','NotNullViolation','UndefinedColumn','UndefinedTable','SyntaxError','InsufficientPrivilege'} + code=getattr(error,'sqlstate',None) + print(json.dumps({'status':'failed','stage':stage,'error_class':type(error).__name__ if type(error).__name__ in allowed else 'unclassified','sqlstate':code if isinstance(code,str) and re.fullmatch('[0-9A-Z]{5}',code) else None})) + return 0 # transport succeeds; closed status still refuses acceptance +if __name__=='__main__':raise SystemExit(main()) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/reviewed-driver-base.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/reviewed-driver-base.py new file mode 100644 index 00000000..23467e01 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/reviewed-driver-base.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +"""Default is a read-only plan. Docker execution awaits independent exact review.""" +import argparse +import base64 +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import re +import secrets +import tarfile +import tempfile +import time +import uuid + +HERE = Path(__file__).resolve().parent +MANIFEST = HERE / 'manifest.json' + +# Fixed local diagnostic parser; no dependency or credential discovery. +_spec = importlib.util.spec_from_file_location('fixture_diagnostic_protocol', HERE / 'diagnostic_protocol.py') +diagnostics = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(diagnostics) + + +def require(condition): + if not condition: + raise RuntimeError('bounded_synthetic_fixture_refused') + + +def reviewed_manifest(expected): + raw = MANIFEST.read_bytes() + require(expected == hashlib.sha256(raw).hexdigest()) + manifest = json.loads(raw) + for path, digest in manifest['source_sha256'].items(): + require(bool(re.fullmatch(r'sha256:[0-9a-f]{64}', digest))) + require('sha256:' + hashlib.sha256(Path(path).read_bytes()).hexdigest() == digest) + return manifest + + +def load_containment(manifest): + spec = importlib.util.spec_from_file_location('reviewed_existing_containment', manifest['containment_helper']) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def session_plan(manifest, run_id): + require(bool(re.fullmatch(r'[a-f0-9]{32}', run_id))) + anchor = 'nk49-kc-' + run_id + '-journal-postgres' + probe = 'nk49-kc-' + run_id + '-journal-probe' + return {'run_id': run_id, 'containers': [anchor, probe], 'worker': anchor, 'issuer': probe, + 'images': {anchor: manifest['postgres_image'], probe: manifest['python_image']}, 'volumes': {}} + + +def base_args(session, name, network, user="1000:1000"): + uid, gid = user.split(":") + return ['docker', 'run', '--detach', '--pull=never', '--name', name, '--network', network, + '--label', 'net-kingdom.recovery-run=' + session['run_id'], '--read-only', '--cap-drop', 'ALL', + '--security-opt', 'no-new-privileges', '--user', user, '--memory', '384m', '--cpus', '0.5', + '--pids-limit', '64', '--log-driver', 'none', '--restart', 'no', + '--tmpfs', '/tmp:rw,nosuid,nodev,noexec,size=16777216,uid=' + uid + ',gid=' + gid + ',mode=0700'] + + +def tar_bytes(files): + out = io.BytesIO() + with tarfile.open(fileobj=out, mode='w') as archive: + for name, value in files.items(): + require(bool(re.fullmatch(r'[a-zA-Z0-9._/-]+', name)) and not name.startswith('/') and '..' not in Path(name).parts) + entry = tarfile.TarInfo(name) + entry.size = len(value) + entry.mode = 0o600 + entry.uid = entry.gid = 1000 + archive.addfile(entry, io.BytesIO(value)) + return out.getvalue() + + +IMPORT = r''' +import base64,json,os,sys +from pathlib import Path +os.umask(0o077) +root=Path('/fixture') +(root/'source').mkdir(mode=0o700) +(root/'source/migrations').mkdir(mode=0o700) +(root/'source/tests').mkdir(mode=0o700) +for name,encoded in json.load(sys.stdin).items(): + if name not in ('source/provisioner.py','source/group_provenance.py','source/group_provenance_postgres.py','source/migrations/group_journal_postgres_v1.sql','source/tests/test_service_groups.py','probe.py','private.json','diagnostic_protocol.py'):raise ValueError() + fd=os.open(root/name,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600) + with os.fdopen(fd,'wb') as output:output.write(base64.b64decode(encoded,validate=True)) +''' + + +def helper_python(manifest): + # Exact producer source installs into official Python's system prefix. + # Never fall back to PATH or another interpreter/virtual environment. + require(manifest.get('helper_interpreter') == '/usr/local/bin/python3') + return manifest['helper_interpreter'] + + +INTERPRETER_PROBE = r""" +import json,sys +try: + import psycopg + valid=sys.executable=='/usr/local/bin/python3' and psycopg.__version__.split('.')[0]=='3' + print(json.dumps({'helper_ready':valid,'executable':sys.executable,'psycopg_version':psycopg.__version__})) +except Exception: + print(json.dumps({'helper_ready':False})) +""" + + +def preflight(adapter, manifest): + helper_python(manifest) + adapter.docker_environment() # Refuse ambient alternate routing before calls. + contexts = json.loads(adapter.command(['docker', 'context', 'inspect'], timeout=15)) + require(len(contexts) == 1 and contexts[0]['Endpoints']['docker']['Host'] == manifest['fixed_endpoint']) + facts = json.loads(adapter.command(['docker', 'info', '--format', + '{"ID":{{json .ID}},"Name":{{json .Name}},"OSType":{{json .OSType}}}'], timeout=15)) + require(facts == {'ID': manifest['engine_id'], 'Name': manifest['engine_name'], 'OSType': 'linux'}) + for image in (manifest['postgres_image'], manifest['python_image']): + value = adapter.command(['docker', 'image', 'inspect', '--format', '{{.Id}}', image], timeout=15).decode().strip() + require(bool(re.fullmatch(r'sha256:[a-f0-9]{64}', value))) + + +def retain_helper_result(result, phase, raw, publish): + stages, value = diagnostics.parse_frames(raw) + safe = {'status': value['status'], 'stages': stages} + if value['status'] == 'failed': + safe['diagnostic'] = value['diagnostic'] + result.setdefault('helper_diagnostics', {})[phase] = safe + publish() # Persist failure evidence before the phase oracle can raise. + return value + + +def execute(manifest, adapter, result, checkpoint=None): + session = session_plan(manifest, uuid.uuid4().hex) + result['session'] = {'run_id': session['run_id'], 'containers': session['containers'], + 'label': 'net-kingdom.recovery-run=' + session['run_id']} + python = helper_python(manifest) + deadline = time.monotonic() + 210 + original_command = adapter.command + def publish(): + if checkpoint is not None: + try: checkpoint() + except OSError: result['receipt_checkpoint_failed'] = True + def mark(stage): + require(stage in diagnostics.DRIVER_STAGES) + result['driver_stage'] = stage + publish() + def event(argv): + require(len(result.setdefault('rpc_trace', [])) < 192) + operation = argv[1] if len(argv) > 1 and argv[1] in {'run','exec','rm','container','context','image','info','volume'} else 'other' + row = {'stage': result['driver_stage'], 'operation': operation, 'status': 'started'} + result['rpc_trace'].append(row) + publish() + return row + def run(argv, **kwargs): + row = event(argv) + remaining = deadline - time.monotonic() + require(remaining > 0) + try: + value = original_command(argv, timeout=min(50, remaining), **kwargs) + row['status'] = 'completed' + publish() + return value + except BaseException as error: + row['status'] = 'failed' + result['daemon_request_uncertain'] = True + result.setdefault('driver_failure', diagnostics.error_metadata(error, result['driver_stage'])) + # Timeout output can contain complete public progress frames. Never + # retain raw stdout/stderr or command/exception text. + if result['driver_stage'] in manifest['phase_oracles']: + partial = getattr(error, 'stdout', None) + if partial: + try: + stages, _ = diagnostics.parse_frames(partial, require_final=False) + result.setdefault('helper_diagnostics', {})[result['driver_stage']] = {'status': 'interrupted', 'stages': stages} + except Exception: + pass + publish() + raise + mark('session_registered') + adapter.command = run + try: + anchor = session['worker'] + args = base_args(session, anchor, 'none', user='999:999') + [ + '--tmpfs', '/var/lib/postgresql:rw,nosuid,nodev,noexec,size=1048576,uid=999,gid=999,mode=0700', + '--tmpfs', '/var/lib/postgresql/data:rw,nosuid,nodev,noexec,size=134217728,uid=999,gid=999,mode=0700', + '--tmpfs', '/run/fixture:rw,nosuid,nodev,noexec,size=1048576,uid=999,gid=999,mode=0700', + '--env', 'PGDATA=/var/lib/postgresql/data', '--entrypoint', '/bin/sh', + manifest['postgres_image'], '-c', 'sleep 240'] + mark('anchor_launch') + run(args) + mark('anchor_containment') + adapter.inspect_session({**session, 'containers': [anchor], 'issuer': anchor}, worker_only=True) + admin_password, runtime_password = secrets.token_hex(24), secrets.token_hex(24) + mark('postgres_private_import') + run(['docker', 'exec', '-i', anchor, 'tar', '-xf', '-', '-C', '/run/fixture'], + input=tar_bytes({'password': admin_password.encode()})) + mark('postgres_initdb') + initialized = json.loads(run(['docker', 'exec', anchor, '/bin/sh', '-c', + "if initdb -D /var/lib/postgresql/data --username=fixture_admin --auth-local=trust --auth-host=scram-sha-256 --pwfile=/run/fixture/password > /dev/null 2>&1; then echo '{\"postgres_initdb\":true}'; else echo '{\"postgres_initdb\":false}'; fi"], max_output=256)) + result['postgres_initdb_complete'] = initialized == {'postgres_initdb': True} + publish() + require(result['postgres_initdb_complete']) + mark('postgres_launch') + run(['docker', 'exec', '--detach', anchor, '/bin/sh', '-c', + 'exec postgres -D /var/lib/postgresql/data -c listen_addresses=127.0.0.1 -c unix_socket_directories=/run/fixture > /dev/null 2>&1']) + mark('anchor_identity') + anchor_id = run(['docker', 'container', 'inspect', anchor, '--format', '{{.Id}}']).decode().strip() + require(bool(re.fullmatch(r'[a-f0-9]{64}', anchor_id))) + args = base_args(session, session['issuer'], 'container:' + anchor_id) + [ + '--tmpfs', '/fixture:rw,nosuid,nodev,noexec,size=16777216,uid=1000,gid=1000,mode=0700', + '--entrypoint', python, manifest['python_image'], '-c', 'import time;time.sleep(240)'] + mark('helper_launch') + run(args) + mark('full_containment') + adapter.inspect_session(session, worker_only=True) + result['containment_verified_before_generated_private_import'] = True + mark('helper_interpreter') + interpreter = json.loads(run(['docker', 'exec', session['issuer'], python, '-c', INTERPRETER_PROBE], max_output=4096)) + require(interpreter.get('helper_ready') is True and interpreter.get('executable') == python) + require(isinstance(interpreter.get('psycopg_version'), str) and re.fullmatch(r'3\.[0-9]+\.[0-9]+(?:[a-z]+[0-9]+)?', interpreter['psycopg_version'])) + result['helper_interpreter_verified_before_candidate_import'] = True + result['helper_psycopg_version'] = interpreter['psycopg_version'] + files = {'source/' + name: (HERE.parent / 'postimage/identity-provisioner' / name).read_bytes() + for name in ('provisioner.py', 'group_provenance.py', 'group_provenance_postgres.py', + 'migrations/group_journal_postgres_v1.sql')} + files['source/tests/test_service_groups.py'] = (HERE / 'test_service_groups.py').read_bytes() + files.update({'probe.py': (HERE / 'probe.py').read_bytes(), 'diagnostic_protocol.py': (HERE / 'diagnostic_protocol.py').read_bytes(), 'private.json': json.dumps({ + 'admin_password': admin_password, 'runtime_password': runtime_password}).encode()}) + mark('candidate_import') + run(['docker', 'exec', '-i', session['issuer'], python, '-c', IMPORT], + input=json.dumps({k: base64.b64encode(v).decode() for k, v in files.items()}).encode()) + result['checks'] = {} + backup = None + active_database = 'postgres' + for phase in ('bootstrap', 'restart', 'concurrency', 'fork', 'lost-ack', 'restored'): + if phase == 'restored': + require(backup is not None) + mark('database_restore') + run(['docker', 'exec', '-i', anchor, 'pg_restore', '--host', '/run/fixture', '-U', 'fixture_admin', + '--dbname', active_database, '--clean', '--if-exists', '--single-transaction', '--exit-on-error', '--no-owner'], input=backup) + mark(phase) + raw = run(['docker', 'exec', session['issuer'], python, '/fixture/probe.py', phase], max_output=32768) + value = retain_helper_result(result, phase, raw, publish) + expected = set(manifest['phase_oracles'][phase]) + require(value.get('status') == 'passed' and set(value.get('checks', {})) == expected) + require(all(v is True for v in value['checks'].values())) + result['checks'].update(value['checks']) + result['psycopg_version'] = value['psycopg_version'] + require(value.get('active_database') in {'postgres', 'fixture_fork'}) + active_database = value['active_database'] + if phase == 'bootstrap': + mark('database_dump') + backup = run(['docker', 'exec', anchor, 'pg_dump', '--host', '/run/fixture', '-U', 'fixture_admin', + '--dbname', 'postgres', '--format=custom', '--no-owner'], max_output=4194304) + result['status'] = 'synthetic_postgres_journal_compatibility_passed' + mark('complete') + except BaseException as error: + result.setdefault('driver_failure', diagnostics.error_metadata(error, result['driver_stage'])) + publish() + raise + finally: + mark('cleanup') + def cleanup_command(argv, **kwargs): + row = event(argv) + try: + value = original_command(argv, timeout=15, **kwargs) + row['status'] = 'completed' + publish() + return value + except BaseException: + result['daemon_request_uncertain'] = True + row['status'] = 'failed' + publish() + raise + adapter.command = cleanup_command + try: + adapter.cleanup(session, result) + result['container_cleanup_verified'] = True + except BaseException: + result['status'] = 'failed' + result['container_cleanup_verified'] = False + adapter.command = original_command + publish() + if result.get('daemon_request_uncertain') or result.get('receipt_checkpoint_failed'): + result['status'] = 'failed' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--plan', action='store_true') + parser.add_argument('--execute-reviewed-fixture', action='store_true') + parser.add_argument('--expected-manifest-sha256') + args = parser.parse_args() + if not args.execute_reviewed_fixture: + print(MANIFEST.read_text()) + return 0 + require(not args.plan) + manifest = reviewed_manifest(args.expected_manifest_sha256) + adapter = load_containment(manifest) + preflight(adapter, manifest) + os.umask(0o077) + output = Path(tempfile.mkdtemp(prefix='nk48-postgres-native-review-', dir='/tmp')) / 'result.json' + result = {'schema': 'nk48.synthetic-postgres-journal-result.v1', 'status': 'failed', + 'manifest_sha256': args.expected_manifest_sha256, 'production_contacted': False, + 'actual_credentials_used': False, 'native_acceptance_claimed': False} + try: + execute(manifest, adapter, result, checkpoint=lambda: output.write_text(json.dumps(result, indent=2) + '\n')) + except BaseException: + result['status'] = 'failed' + finally: + output.write_text(json.dumps(result, indent=2) + '\n') + print(json.dumps({'status': result['status'], 'safe_result': str(output)})) + return int(result['status'] != 'synthetic_postgres_journal_compatibility_passed') + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/test_driver.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/test_driver.py new file mode 100644 index 00000000..cef6b342 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/test_driver.py @@ -0,0 +1,86 @@ +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import pytest +HERE=Path(__file__).parent +spec=importlib.util.spec_from_file_location('migration_compat_driver',HERE/'driver.py') +driver=importlib.util.module_from_spec(spec);spec.loader.exec_module(driver) + +class FakeAdapter: + def __init__(self,refuse_containment=False,probe_failure=False): + self.calls=[];self.refuse_containment=refuse_containment;self.probe_failure=probe_failure;self.cleaned=False + def command(self,argv,**kwargs): + self.calls.append((argv,kwargs)) + if argv[1:3]==['container','inspect']:return ('a'*64).encode() + if 'INTERPRETER_PROBE' in argv:raise AssertionError('literal placeholder') + if driver.base.INTERPRETER_PROBE in argv:return json.dumps({'helper_ready':True,'executable':'/usr/local/bin/python3','psycopg_version':'3.3.6'}).encode() + if '/fixture/probe.py' in argv: + value={'status':'failed','stage':'new_migration','error_class':'NotNullViolation','sqlstate':'23502'} if self.probe_failure else {'status':'passed','checks':dict.fromkeys(driver.CHECKS,True),'migration_seconds_synthetic_only':0.01} + return json.dumps(value).encode() + if any('initdb -D' in a or 'pg_isready ' in a for a in argv):return b'{"ready":true}' + return b'fixture-command-ok' + def inspect_session(self,*args,**kwargs): + if self.refuse_containment:raise RuntimeError('synthetic containment refusal') + def cleanup(self,session,result): + self.cleaned=True;result['cleanup']={n:'absent' for n in session['containers']} + +@pytest.fixture +def manifest():return json.loads((HERE/'manifest.json').read_text()) + +def test_exact_bounded_argv_and_cleanup(manifest): + adapter=FakeAdapter();original=adapter.command;result={'status':'failed'} + driver.execute(manifest,adapter,result,lambda:None) + assert result['status']=='synthetic_migration_old_write_shapes_passed' + assert adapter.cleaned and result['container_cleanup_verified'] and adapter.command==original + runs=[a for a,_ in adapter.calls if a[:2]==['docker','run']] + assert len(runs)==2 + for args in runs: + assert '--pull=never' in args and '--read-only' in args + assert '--privileged' not in args and '--volume' not in args and '-v' not in args and '-p' not in args + assert args[args.index('--log-driver')+1]=='none' + assert runs[0][runs[0].index('--network')+1]=='none' + assert runs[1][runs[1].index('--network')+1]=='container:'+'a'*64 + assert any(a.startswith('/var/lib/postgresql/data:rw,nosuid,nodev,noexec,') for a in runs[0]) + assert any(a.startswith('/var/lib/postgresql:rw,nosuid,nodev,noexec,') for a in runs[0]) + imports=[a for a,k in adapter.calls if 'input' in k] + assert len(imports)==2 + +def test_containment_refusal_cleans_before_any_private_import(manifest): + adapter=FakeAdapter(refuse_containment=True);result={'status':'failed'} + with pytest.raises(RuntimeError):driver.execute(manifest,adapter,result,lambda:None) + assert adapter.cleaned and result['container_cleanup_verified'] + assert not any('input' in kwargs for _,kwargs in adapter.calls) + +def test_closed_probe_failure_is_retained_and_never_accepts(manifest): + adapter=FakeAdapter(probe_failure=True);result={'status':'failed'} + with pytest.raises(RuntimeError):driver.execute(manifest,adapter,result,lambda:None) + assert result['status']=='failed' and result['diagnostic']['sqlstate']=='23502' + assert adapter.cleaned and result['container_cleanup_verified'] + +def test_manifest_hash_mismatch_refuses_before_adapter_loading(monkeypatch): + with pytest.raises(RuntimeError):driver.base.reviewed_manifest('0'*64) + +def test_offline_real_orm_and_migration_compilation_without_network(tmp_path): + for name in ['build-offline-sql.py','old-base.py','old-agent-message.py','migration.py']: + shutil.copyfile(HERE/name,tmp_path/name) + script="import socket,runpy; socket.socket.connect=lambda *a: (_ for _ in ()).throw(AssertionError('network forbidden')); runpy.run_path('build-offline-sql.py',run_name='__main__')" + env=dict(os.environ);env['PYTHONHASHSEED']='0' + p=subprocess.run([sys.executable,'-c',script],cwd=tmp_path,capture_output=True,env=env) + assert p.returncode==0 + for name in ['old-schema.sql','upgrade.sql','old-writes.json']: + assert (tmp_path/name).read_bytes()==(HERE/name).read_bytes() + assert b'CREATE TRIGGER agent_messages_inbox_revision' in (tmp_path/'upgrade.sql').read_bytes() + assert b'revision' not in (tmp_path/'old-schema.sql').read_bytes() + + +def test_receipt_failure_still_cleans_and_cannot_report_success(manifest): + adapter=FakeAdapter();result={'status':'failed'} + def unavailable():raise OSError('synthetic private receipt unavailable') + driver.execute(manifest,adapter,result,unavailable) + assert adapter.cleaned and result['container_cleanup_verified'] + assert result['receipt_checkpoint_failed'] and result['status']=='failed' diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/upgrade.sql b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/upgrade.sql new file mode 100644 index 00000000..6e4b137d --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/upgrade.sql @@ -0,0 +1,90 @@ +CREATE SEQUENCE inbox_revision_seq AS bigint; + +ALTER TABLE agent_messages ADD COLUMN revision BIGINT; + +ALTER TABLE message_receipts ADD COLUMN revision BIGINT; + +DO $$ + DECLARE base bigint := nextval('inbox_revision_seq'); + BEGIN + UPDATE agent_messages SET revision = base; + UPDATE message_receipts SET revision = base; + END + $$; + +ALTER TABLE agent_messages ALTER COLUMN revision SET NOT NULL; + +CREATE INDEX ix_agent_messages_revision ON agent_messages (revision); + +ALTER TABLE message_receipts ALTER COLUMN revision SET NOT NULL; + +CREATE INDEX ix_message_receipts_revision ON message_receipts (revision); + +CREATE TABLE message_tombstones ( + message_id UUID NOT NULL, + revision BIGINT NOT NULL, + deleted_at TIMESTAMP WITH TIME ZONE NOT NULL, + reason TEXT, + PRIMARY KEY (message_id) +); + +CREATE INDEX ix_message_tombstones_revision ON message_tombstones (revision); + +CREATE OR REPLACE FUNCTION inbox_feed_revision() RETURNS bigint +LANGUAGE plpgsql AS $fn$ +DECLARE + tx text := txid_current()::text; + cached text := coalesce(current_setting('statehub.inbox_rev', true), ''); + rev bigint; +BEGIN + PERFORM pg_advisory_xact_lock(5348490098); + IF split_part(cached, ':', 1) = tx THEN + RETURN split_part(cached, ':', 2)::bigint; + END IF; + rev := nextval('inbox_revision_seq'); + PERFORM set_config('statehub.inbox_rev', tx || ':' || rev::text, true); + RETURN rev; +END +$fn$; + +CREATE OR REPLACE FUNCTION inbox_feed_stamp_revision() RETURNS trigger +LANGUAGE plpgsql AS $fn$ +BEGIN + IF TG_OP = 'UPDATE' THEN + IF NEW IS NOT DISTINCT FROM OLD THEN + RETURN NEW; + END IF; + END IF; + NEW.revision := inbox_feed_revision(); + RETURN NEW; +END +$fn$; + +CREATE OR REPLACE FUNCTION inbox_feed_tombstone() RETURNS trigger +LANGUAGE plpgsql AS $fn$ +BEGIN + INSERT INTO message_tombstones (message_id, revision, deleted_at, reason) + VALUES ( + OLD.id, + inbox_feed_revision(), + now(), + nullif(current_setting('statehub.delete_reason', true), '') + ) + ON CONFLICT (message_id) DO UPDATE + SET revision = EXCLUDED.revision, + deleted_at = EXCLUDED.deleted_at, + reason = EXCLUDED.reason; + RETURN OLD; +END +$fn$;; + +CREATE TRIGGER agent_messages_inbox_revision + BEFORE INSERT OR UPDATE ON agent_messages + FOR EACH ROW EXECUTE FUNCTION inbox_feed_stamp_revision(); +CREATE TRIGGER agent_messages_inbox_tombstone + BEFORE DELETE ON agent_messages + FOR EACH ROW EXECUTE FUNCTION inbox_feed_tombstone(); +CREATE TRIGGER message_receipts_inbox_revision + BEFORE INSERT OR UPDATE ON message_receipts + FOR EACH ROW EXECUTE FUNCTION inbox_feed_stamp_revision();; + diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/validation.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/validation.json new file mode 100644 index 00000000..5640a1c8 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/migration-compat-r1/validation.json @@ -0,0 +1,23 @@ +{ + "schema": "statehub-migration-compat-source-validation.v1", + "status": "source_prepared_not_docker_executed", + "production_mutations": false, + "actual_database_tests_executed": false, + "tests": { + "passed": 6, + "command": "/home/worsch/state-hub/.venv/bin/python -m pytest -q --tb=short test_driver.py", + "duration_seconds": 0.64 + }, + "retained_failure": { + "test": "test_offline_real_orm_and_migration_compilation_without_network", + "result": "failed_on_repeat_generation", + "cause": "SQLAlchemy independent foreign-key constraint/index object iteration order varied; fixed Python hash seed alone was insufficient", + "fix": "Require PYTHONHASHSEED=0 and normalize only independent compiled FK clause order plus sorted compiled indexes after dependency-ordered compiled tables; isolated child compilation with socket.connect disabled now byte-compares all three artifacts", + "attempts": 2 + }, + "scope_limits": [ + "Fake adapter tests verify containment argv/private import order/cleanup and refusal only", + "Real PostgreSQL compatibility fixture is prepared, not executed", + "No full application or production schema acceptance claim" + ] +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r1/state-hub/Makefile b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r1/state-hub/Makefile new file mode 100644 index 00000000..cec684ec --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r1/state-hub/Makefile @@ -0,0 +1,517 @@ +.PHONY: start install install-cli dashboard-install dashboard-check db db-tools migrate seed api dashboard dashboard-local sync check-primary primary-port-clear check check-local test test-python clean register-project register-codex-project register-mcp configure-codex bootstrap-env dev-hub edge-relay mcp-profile validate-adr add-domain rename-domain add-repo list-repos register-path register-from-classification register-from-classification-all cleanup-stale tunnels-up tunnels-status tunnels-check bridges install-hooks install-hooks-all gitea-inventory token-reconcile railiance-state-hub-render railiance-state-hub-client-dry-run railiance-state-hub-server-dry-run railiance-state-hub-headroom + +COMPOSE = docker compose -f infra/docker-compose.yml --env-file .env +PYTHON ?= python3 +HELM ?= $(shell command -v helm 2>/dev/null || if [ -x "$$HOME/.local/bin/helm" ]; then printf "%s" "$$HOME/.local/bin/helm"; else printf "%s" "helm"; fi) +KUBECTL ?= $(shell command -v kubectl 2>/dev/null || if [ -x "$$HOME/.local/bin/kubectl" ]; then printf "%s" "$$HOME/.local/bin/kubectl"; else printf "%s" "kubectl"; fi) + +RAILIANCE_STATE_HUB_RELEASE ?= state-hub +RAILIANCE_STATE_HUB_NAMESPACE ?= state-hub +RAILIANCE_STATE_HUB_CHART ?= deploy/railiance/apps/charts/state-hub +RAILIANCE_STATE_HUB_VALUES ?= deploy/railiance/apps/helm/state-hub-values.yaml +RAILIANCE_STATE_HUB_IMAGE_TAG ?= main-51bc70d@sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015 +RAILIANCE_STATE_HUB_PLATFORM_DIR ?= deploy/railiance/platform +RAILIANCE_STATE_HUB_APP_MANIFESTS ?= deploy/railiance/apps/manifests +# Codex/WSL non-login shells may not source ~/.profile; keep uv discoverable. +UV ?= $(shell command -v uv 2>/dev/null || if [ -x "$$HOME/.local/bin/uv" ]; then printf "%s" "$$HOME/.local/bin/uv"; else printf "%s" "uv"; fi) +RMGR ?= $(shell command -v rmgr 2>/dev/null || if [ -x "$$HOME/repo-manager/.venv/bin/rmgr" ]; then printf "%s" "$$HOME/repo-manager/.venv/bin/rmgr"; else printf "%s" "rmgr"; fi) +STATE_HUB_API_BASE ?= http://127.0.0.1:8000 +SYNC_PATH ?= . +SYNC_PUSH ?= 1 + +start: + @echo "# Normal production UI access (run in order)" + @echo "make bridges # connect local :8000 to the railiance01 primary" + @echo "make dashboard # verify the primary, then serve the UI on :3000" + @echo "make sync # push file-backed records and reconcile the exact forge commit" + @echo "" + @echo "# Optional local MCP adapter" + @echo "make mcp-http # local SSE adapter on :8001; not needed by the UI" + @echo "" + @echo "# Local fallback/development only" + @echo "make api # local Postgres + API; conflicts with the production :8000 tunnel" + @echo "make dashboard-local # dashboard against the deliberate local API" + +install: + $(UV) sync + +dashboard/node_modules/.bin/observable: dashboard/package.json dashboard/package-lock.json + cd dashboard && npm ci + +dashboard-install: dashboard/node_modules/.bin/observable + +dashboard-check: dashboard-install + cd dashboard && npm run build + +## Symlink the custodian CLI into ~/.local/bin so it's on PATH system-wide +install-cli: install + mkdir -p ~/.local/bin + ln -sf "$(shell pwd)/.venv/bin/custodian" ~/.local/bin/custodian + ln -sf "$(shell pwd)/.venv/bin/statehub" ~/.local/bin/statehub + @echo "Installed: custodian → $$(readlink -f ~/.local/bin/custodian)" + @echo "Installed: statehub → $$(readlink -f ~/.local/bin/statehub)" + @echo "Make sure ~/.local/bin is on your PATH:" + @echo " echo 'export PATH=\"\$$HOME/.local/bin:\$$PATH\"' >> ~/.bashrc && source ~/.bashrc" + +db: + $(COMPOSE) up -d postgres + +db-tools: + $(COMPOSE) --profile tools up -d + +migrate: + $(UV) run alembic upgrade head + +seed: + $(UV) run python scripts/seed.py + +## Start (or restart) the MCP SSE server on :8001 — primary transport for Claude Code. +## Remote clients (e.g. COULOMBCORE) connect via the ops-bridge tunnel (port 18001). +## Registration: claude mcp add-json -s user state-hub '{"type":"sse","url":"http://127.0.0.1:8001/sse"}' +mcp-http: + @fuser -k 8001/tcp 2>/dev/null && echo "Stopped running MCP server" || true + MCP_TRANSPORT=sse MCP_PORT=8001 $(UV) run python mcp_server/server.py + +## Require the production identity before serving the normal dashboard. This +## prevents an accidental local `make api` from presenting an empty fallback DB +## as the live State Hub. +check-primary: + @health="$$(curl -fsS --max-time 5 http://127.0.0.1:8000/state/health 2>/dev/null)" || { \ + echo "ERROR: State Hub primary is not reachable on 127.0.0.1:8000." >&2; \ + echo "Run 'make bridges' first." >&2; \ + exit 1; \ + }; \ + identity="$$(printf '%s' "$$health" | $(PYTHON) -c 'import json, sys; d=json.load(sys.stdin); print("{}/{}".format(d.get("instance_role", ""), d.get("instance_label", "")))')"; \ + if [ "$$identity" != "primary/railiance01" ]; then \ + echo "ERROR: 127.0.0.1:8000 is '$$identity', not the primary/railiance01 State Hub." >&2; \ + echo "Stop the local API, then run 'make bridges'. Use 'make dashboard-local' only for intentional local development." >&2; \ + exit 1; \ + fi + +## Refuse to start the production tunnel when a non-primary API already owns +## port 8000. With no listener, bridge is free to establish the tunnel. +primary-port-clear: + @if health="$$(curl -fsS --max-time 2 http://127.0.0.1:8000/state/health 2>/dev/null)"; then \ + identity="$$(printf '%s' "$$health" | $(PYTHON) -c 'import json, sys; d=json.load(sys.stdin); print("{}/{}".format(d.get("instance_role", ""), d.get("instance_label", "")))')"; \ + if [ "$$identity" != "primary/railiance01" ]; then \ + echo "ERROR: port 8000 is occupied by '$$identity'." >&2; \ + echo "Stop the local 'make api' process before starting the production tunnel." >&2; \ + exit 1; \ + fi; \ + fi + +dashboard: check-primary + @fuser -k 3000/tcp 2>/dev/null && echo "Stopped running dashboard" || true + $(MAKE) dashboard-install + cd dashboard && npm run dev + +## Deliberate local-development dashboard; bypasses the production identity gate. +dashboard-local: + @echo "WARNING: serving the dashboard against the local/fallback API on :8000." + @fuser -k 3000/tcp 2>/dev/null && echo "Stopped running dashboard" || true + $(MAKE) dashboard-install + cd dashboard && npm run dev + +## Fast work-record path. Repo Manager assigns only missing deterministic IDs, +## verifies the primary/railliance01 bridge, and asks central to derive the +## exact pushed Forgejo commit in one transactional request. +sync: + $(RMGR) sync --path "$(SYNC_PATH)" --api-base "$(STATE_HUB_API_BASE)" $(if $(filter 1 true yes,$(SYNC_PUSH)),--push,) + +check: check-primary + @echo "State Hub primary/railiance01 is healthy." + +check-local: + curl -sf http://127.0.0.1:8000/state/health | python3 -m json.tool + +# CUST-WP-0067-T09. The chart ships a copy of the-custodian canon allowed-values +# because a container has no checkout to read it from. Regenerate after any +# canon vocabulary change; the -check target fails on drift so a release cannot +# validate classification against a stale vocabulary. +sync-classification-allowed: + python3 scripts/sync_classification_allowed.py + +check-classification-allowed: + python3 scripts/sync_classification_allowed.py --check + +# STATE-WP-0091: read-only surge-headroom check. Does not start Helm. +# Live: make railiance-state-hub-headroom +# Fixtures: make railiance-state-hub-headroom HEADROOM_FIXTURE=tests/fixtures/release_headroom/rev63-105m.json +HEADROOM_FIXTURE ?= +railiance-state-hub-headroom: + @if [ -n "$(HEADROOM_FIXTURE)" ]; then \ + $(PYTHON) scripts/release_headroom_preflight.py --fixture "$(HEADROOM_FIXTURE)"; \ + else \ + $(PYTHON) scripts/release_headroom_preflight.py --kubectl "$(KUBECTL)"; \ + fi + +railiance-state-hub-render: check-classification-allowed + $(HELM) template $(RAILIANCE_STATE_HUB_RELEASE) $(RAILIANCE_STATE_HUB_CHART) \ + --namespace $(RAILIANCE_STATE_HUB_NAMESPACE) \ + -f $(RAILIANCE_STATE_HUB_VALUES) \ + --set image.tag=$(RAILIANCE_STATE_HUB_IMAGE_TAG) + +railiance-state-hub-client-dry-run: + @set -e; \ + tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + $(HELM) template $(RAILIANCE_STATE_HUB_RELEASE) $(RAILIANCE_STATE_HUB_CHART) \ + --namespace $(RAILIANCE_STATE_HUB_NAMESPACE) \ + -f $(RAILIANCE_STATE_HUB_VALUES) \ + --set image.tag=$(RAILIANCE_STATE_HUB_IMAGE_TAG) > "$$tmpdir/state-hub.yaml"; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-credentials.sops.yaml.template; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-cluster.yaml; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-networkpolicies.yaml; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-namespace.yaml; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-env.secret.sops.yaml.template; \ + $(KUBECTL) apply --dry-run=client -n $(RAILIANCE_STATE_HUB_NAMESPACE) -f "$$tmpdir/state-hub.yaml" + +railiance-state-hub-server-dry-run: + @set -e; \ + tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + $(HELM) template $(RAILIANCE_STATE_HUB_RELEASE) $(RAILIANCE_STATE_HUB_CHART) \ + --namespace $(RAILIANCE_STATE_HUB_NAMESPACE) \ + -f $(RAILIANCE_STATE_HUB_VALUES) \ + --set image.tag=$(RAILIANCE_STATE_HUB_IMAGE_TAG) > "$$tmpdir/state-hub.yaml"; \ + $(KUBECTL) apply --server-side --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-credentials.sops.yaml.template; \ + $(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-cluster.yaml; \ + $(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-networkpolicies.yaml; \ + $(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-namespace.yaml; \ + if $(KUBECTL) get namespace $(RAILIANCE_STATE_HUB_NAMESPACE) >/dev/null 2>&1; then \ + $(KUBECTL) apply --server-side --dry-run=server -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-env.secret.sops.yaml.template; \ + $(KUBECTL) apply --dry-run=server -n $(RAILIANCE_STATE_HUB_NAMESPACE) -f "$$tmpdir/state-hub.yaml"; \ + else \ + echo "Namespace $(RAILIANCE_STATE_HUB_NAMESPACE) does not exist; validating namespaced app manifests with client dry-run."; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-namespace.yaml; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-env.secret.sops.yaml.template; \ + $(KUBECTL) apply --dry-run=client -n $(RAILIANCE_STATE_HUB_NAMESPACE) -f "$$tmpdir/state-hub.yaml"; \ + fi + +test: test-python dashboard-check + +test-python: + TEST_DATABASE_URL=postgresql+asyncpg://custodian:changeme@127.0.0.1:5432/custodian_test \ + $(UV) run pytest -x -q + +## Benchmark /state/summary revision cache (API must be running on :8000) +benchmark-summary-cache: + $(UV) run python scripts/benchmark_summary_cache.py + +## ops-bridge managed tunnels +## Requires ops-bridge: bridge is at /home/worsch/.local/bin/bridge +tunnels-up: primary-port-clear + bridge up + +tunnels-status: + bridge status + +## End-to-end check: verifies SSH process alive + remote port listening on COULOMBCORE. +## Exits non-zero if any tunnel is not fully operational. +tunnels-check: + bridge check + +## Ensure all ops-bridge tunnels are up and healthy. +## Brings up any stopped/stale tunnels, shows final status, exits non-zero if anything is still down. +bridges: primary-port-clear + @echo "==> Bringing up all tunnels..." + bridge up + @echo "" + @echo "==> Tunnel status:" + bridge status + @echo "" + @echo "==> Checking tunnel health..." + bridge check + +## Start (or restart) the LOCAL FALLBACK backend — db + migrate + uvicorn. +## This replaces anything on :8000, including the production State Hub tunnel. +api: db + @echo "WARNING: starting the local fallback API; this is not the railiance01 primary." + @echo "Waiting for postgres..."; \ + for i in 1 2 3 4 5 6 7 8 9 10; do \ + nc -z 127.0.0.1 5432 2>/dev/null && break; \ + sleep 1; \ + done + $(MAKE) migrate + @fuser -k 8000/tcp 2>/dev/null && echo "Stopped running API" || true + $(UV) run uvicorn api.main:app --reload --reload-dir api --reload-dir mcp_server --reload-dir task_flow_engine --host 127.0.0.1 --port 8000 + +## Register a project (Claude Code): make register-project DOMAIN=railiance PROJECT_PATH=/home/worsch/railiance +register-project: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required. Usage: make register-project DOMAIN= PROJECT_PATH="; exit 1) + @test -n "$(PROJECT_PATH)" || (echo "ERROR: PROJECT_PATH is required."; exit 1) + scripts/register_project.sh "$(DOMAIN)" "$(PROJECT_PATH)" + +## Register a Codex project (AGENTS.md + HTTP API): make register-codex-project DOMAIN=capabilities PROJECT_PATH=/home/worsch/my-repo +register-codex-project: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required. Usage: make register-codex-project DOMAIN= PROJECT_PATH="; exit 1) + @test -n "$(PROJECT_PATH)" || (echo "ERROR: PROJECT_PATH is required."; exit 1) + scripts/register_project.sh "$(DOMAIN)" "$(PROJECT_PATH)" --codex + +## Register State Hub MCP for Claude Code. Optional: make register-mcp MCP_URL=http://127.0.0.1:18001/sse +register-mcp: + scripts/register-mcp.sh \ + $(if $(MCP_URL),--url "$(MCP_URL)",) \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(DRY_RUN),--dry-run,) + +## Configure Codex workspace-write networking for local State Hub access. +configure-codex: + scripts/configure-codex.sh \ + $(if $(CODEX_HOME),--codex-home "$(CODEX_HOME)",) \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(WITH_MCP),--with-mcp,) \ + $(if $(DRY_RUN),--dry-run,) + +## Bootstrap a new operator/collaborator environment. Optional: make bootstrap-env ARGS="--install-missing" +bootstrap-env: + scripts/bootstrap-env.sh $(ARGS) + +## Files-first local dev hub (CUST-WP-0054-T07): postgres + migrate + seed + API + repo registration. +## Optional: WITH_EDGE=1 WITH_MCP=1 make dev-hub +## Fleet tunnel when reachable: make dev-hub PROFILE=fleet +dev-hub: + bash scripts/dev_hub_up.sh $(if $(PROFILE),$(PROFILE),dev) + +## Local edge relay for offline write buffering (STATE-WP-0068). +edge-relay: + @fuser -k 18080/tcp 2>/dev/null && echo "Stopped running edge relay" || true + STATEHUB_UPSTREAM_URL=$${STATEHUB_UPSTREAM_URL:-http://127.0.0.1:8000} \ + $(UV) run uvicorn api.edge.relay:app --host 127.0.0.1 --port 18080 + +## Switch MCP/API profile: make mcp-profile PROFILE=dev|fleet|status +mcp-profile: + @test -n "$(PROFILE)" || (echo "ERROR: PROFILE is required (dev|fleet|status)."; exit 1) + bash scripts/mcp_hub_profile.sh "$(PROFILE)" + +## Add a second repo to an existing domain: make add-repo DOMAIN=railiance REPO_PATH=/home/worsch/railiance-infra +add-repo: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required."; exit 1) + @test -n "$(REPO_PATH)" || (echo "ERROR: REPO_PATH is required."; exit 1) + scripts/register_project.sh "$(DOMAIN)" "$(REPO_PATH)" --additional + +## Create a new domain: make add-domain DOMAIN=my_domain NAME="My Domain" +add-domain: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required (slug)."; exit 1) + @test -n "$(NAME)" || (echo "ERROR: NAME is required (display name)."; exit 1) + curl -sf -X POST http://127.0.0.1:8000/domains/ \ + -H "Content-Type: application/json" \ + -d "{\"slug\": \"$(DOMAIN)\", \"name\": \"$(NAME)\"}" | python3 -m json.tool + +## Rename a domain: make rename-domain DOMAIN=old_slug NEW_SLUG=new_slug NEW_NAME="New Name" +rename-domain: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN (old slug) is required."; exit 1) + @test -n "$(NEW_SLUG)" || (echo "ERROR: NEW_SLUG is required."; exit 1) + @test -n "$(NEW_NAME)" || (echo "ERROR: NEW_NAME is required."; exit 1) + curl -sf -X PATCH http://127.0.0.1:8000/domains/$(DOMAIN)/rename \ + -H "Content-Type: application/json" \ + -d "{\"new_slug\": \"$(NEW_SLUG)\", \"new_name\": \"$(NEW_NAME)\"}" | python3 -m json.tool + +## Register this machine's local path for a repo: make register-path REPO=marki-docx PATH=/home/tegwick/marki-docx +register-path: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make register-path REPO= PATH="; exit 1) + @test -n "$(PATH)" || (echo "ERROR: PATH is required. Usage: make register-path REPO= PATH="; exit 1) + curl -sf -X POST "http://127.0.0.1:8000/repos/$(REPO)/paths" \ + -H "Content-Type: application/json" \ + -d "{\"host\": \"$$(hostname)\", \"path\": \"$(PATH)\"}" | python3 -m json.tool + +## List repos for a domain: make list-repos DOMAIN=railiance +list-repos: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required."; exit 1) + curl -sf "http://127.0.0.1:8000/repos/?domain=$(DOMAIN)" | python3 -m json.tool + +## Ingest SBOM data for a repo (all mechanisms: lockfiles + ansible + sbom-tools.yaml). +## Auto-detect all sources: make ingest-sbom REPO=the-custodian REPO_PATH=/home/worsch/the-custodian +## Single lockfile (explicit): make ingest-sbom REPO=the-custodian LOCKFILE=/path/to/uv.lock +## Dry-run (no submit): make ingest-sbom REPO=the-custodian REPO_PATH=... DRY_RUN=1 +## Tip: run capture-tools first for repos with system-level tool dependencies. +ingest-sbom: + @test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1) + $(UV) run python scripts/ingest_sbom.py --repo "$(REPO)" \ + $(if $(LOCKFILE),--lockfile "$(LOCKFILE)") \ + $(if $(REPO_PATH),--repo-path "$(REPO_PATH)") \ + $(if $(DRY_RUN),--dry-run) + +## Ingest capability declarations from SCOPE.md into the catalog. +## Usage: make ingest-capabilities REPO=the-custodian [REPO_PATH=/home/worsch/the-custodian] +## Or: make ingest-capabilities-all +## Add DRY_RUN=1 to preview without writing. +ingest-capabilities: + @test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1) + $(UV) run python scripts/ingest_capabilities.py --repo "$(REPO)" \ + $(if $(REPO_PATH),--repo-path "$(REPO_PATH)") \ + $(if $(DRY_RUN),--dry-run) + +ingest-capabilities-all: + $(UV) run python scripts/ingest_capabilities.py --all \ + $(if $(DRY_RUN),--dry-run) + +## Check Repository Definition of Integrated (DoI) criteria for a repo. +## Usage: make check-doi REPO=llm-connect +## Or: make check-doi-all +## Add JSON=1 for machine-readable output. +check-doi: + @test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1) + $(UV) run python scripts/check_doi.py --repo "$(REPO)" $(if $(JSON),--json) + +check-doi-all: + $(UV) run python scripts/check_doi.py --all $(if $(JSON),--json) + +## Ingest tpsc.yaml service declarations from a repo into the TPSC catalog. +## Usage: make ingest-tpsc REPO=llm-connect +## Or: make ingest-tpsc-all +## Add DRY_RUN=1 to preview without writing. +ingest-tpsc: + @test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1) + $(UV) run python scripts/ingest_tpsc.py --repo "$(REPO)" \ + $(if $(DRY_RUN),--dry-run) + +ingest-tpsc-all: + $(UV) run python scripts/ingest_tpsc.py --all \ + $(if $(DRY_RUN),--dry-run) + +## Run SBOM capture agent for a repo — generates/updates sbom-tools.yaml. +## Usage: make capture-tools REPO=railiance-infra [REPO_PATH=/home/worsch/railiance-infra] +## Add DRY_RUN=1 to preview without writing. +capture-tools: + @test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1) + $(UV) run python scripts/capture_sbom_tools.py --repo "$(REPO)" \ + $(if $(REPO_PATH),--repo-path "$(REPO_PATH)") \ + $(if $(DRY_RUN),--dry-run) + +## Check a repo for ADR-001 compliance: make validate-adr REPO=/path/to/repo [DOMAIN=custodian] +validate-adr: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make validate-adr REPO= [DOMAIN=]"; exit 1) + $(UV) run python scripts/validate_repo_adr.py "$(REPO)" $(if $(DOMAIN),--domain "$(DOMAIN)",) + +## Consistency exit contract: +## - Direct scripts/consistency_check.py: 0 clean, 2 warnings-only, 1 failures. +## - Agent/operator Make wrappers below normalize warning-only 2 to shell success +## while preserving visible WARN output and keeping real failures non-zero. +## Check a single repo for ADR-001 consistency: make check-consistency REPO=the-custodian [REPO_PATH=/override] +## Exit 0 = clean or warnings-only (warnings stay visible), exit 1 = failures +check-consistency: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make check-consistency REPO="; exit 1) + $(UV) run python scripts/consistency_check.py --repo "$(REPO)" \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(REPO_PATH),--repo-path "$(REPO_PATH)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Check and auto-fix a single repo: make fix-consistency REPO=the-custodian [REPO_PATH=/override] +## Exit 0 = clean or warnings-only (warnings stay visible), exit 1 = failures +fix-consistency: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make fix-consistency REPO="; exit 1) + $(UV) run python scripts/consistency_check.py --repo "$(REPO)" --fix \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(REPO_PATH),--repo-path "$(REPO_PATH)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Normalize workplan frontmatter and task status literals in attached repos. +## Usage: make normalize-attached-workplans REPO=artifact-store +## make normalize-attached-workplans DIRTY=1 +normalize-attached-workplans: + $(UV) run python scripts/normalize_attached_repo_workplans.py \ + $(if $(REPO),--repo "$(REPO)",) \ + $(if $(DIRTY),--dirty,) \ + $(if $(DRY_RUN),--dry-run,) + @test -n "$(REPO)$(DIRTY)" || (echo "ERROR: set REPO= or DIRTY=1"; exit 1) + +## Regenerate AGENTS.md / CLAUDE.md / .claude/rules from templates. +## Usage: make update-agent-instructions REPO=artifact-store +## make update-agent-instructions DIRTY=1 +update-agent-instructions: + $(UV) run python scripts/update_agent_instruction_files.py \ + $(if $(REPO),--repo "$(REPO)",) \ + $(if $(DIRTY),--dirty,) + @test -n "$(REPO)$(DIRTY)" || (echo "ERROR: set REPO= or DIRTY=1"; exit 1) + +## Reconcile measured token sources against State Hub. +## Usage: make token-reconcile [SINCE=2026-05-19] [APPLY=1] [ZERO_FALLBACKS=1] +token-reconcile: + $(PYTHON) scripts/token_reconcile.py \ + $(if $(SINCE),--since "$(SINCE)",) \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(CODEX_HOME),--codex-home "$(CODEX_HOME)",) \ + $(if $(CLAUDE_HOME),--claude-home "$(CLAUDE_HOME)",) \ + $(if $(APPLY),--apply,) \ + $(if $(ZERO_FALLBACKS),--zero-superseded-fallbacks,) + +## Pull then fix: single repo or all repos if REPO omitted +## make fix-consistency-remote — smart pull+fix all repos that need it +## make fix-consistency-remote REPO=slug — pull+fix one repo +fix-consistency-remote: + $(UV) run python scripts/consistency_check.py \ + $(if $(REPO),--repo "$(REPO)",--all) \ + --remote \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(NO_WRITEBACK),--no-writeback,); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Infer repo slug from git remote URL and check: make check-consistency-here [REPO_PATH=/path/to/repo] +## Omit REPO_PATH to use the Python script's CWD (i.e. pass an empty --here flag). +check-consistency-here: + $(UV) run python scripts/consistency_check.py \ + --here $(if $(REPO_PATH),"$(REPO_PATH)",) \ + $(if $(API_BASE),--api-base "$(API_BASE)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Infer repo slug from git remote URL and fix: make fix-consistency-here [REPO_PATH=/path/to/repo] +fix-consistency-here: + $(UV) run python scripts/consistency_check.py \ + --here $(if $(REPO_PATH),"$(REPO_PATH)",) \ + --fix \ + $(if $(API_BASE),--api-base "$(API_BASE)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Check all registered repos for ADR-001 consistency +check-consistency-all: + $(UV) run python scripts/consistency_check.py --all $(if $(API_BASE),--api-base "$(API_BASE)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Check and auto-fix all registered repos +fix-consistency-all: + $(UV) run python scripts/consistency_check.py --all --fix $(if $(API_BASE),--api-base "$(API_BASE)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Cancel open tasks belonging to completed/archived workstreams. +## Safe to run at any time; also suitable for a daily cron job. +## Cron example: 0 3 * * * cd ~/state-hub && make cleanup-stale +cleanup-stale: + $(UV) run python scripts/cleanup_stale_tasks.py + +## Install custodian post-commit sync hook into one repo: make install-hooks REPO=marki-docx +install-hooks: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make install-hooks REPO="; exit 1) + bash scripts/install_hooks.sh --repo "$(REPO)" + +## Install custodian post-commit sync hook into all active registered repos +install-hooks-all: + bash scripts/install_hooks.sh --all + +## Remove custodian post-commit sync hook from one repo: make remove-hooks REPO=marki-docx +remove-hooks: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make remove-hooks REPO="; exit 1) + bash scripts/install_hooks.sh --repo "$(REPO)" --remove + +## Compare Gitea coulomb org repos against state-hub registered repos +## Requires GITEA_TOKEN in env or .env: make gitea-inventory GITEA_TOKEN= +gitea-inventory: + $(UV) run python scripts/gitea_inventory.py $(if $(JSON),--json) + +## Register/update one repo from .repo-classification.yaml: +## make register-from-classification REPO=state-hub +## make register-from-classification PATH=/path/to/repo +## Optional: DRY_RUN=1 +register-from-classification: + @test -n "$(REPO)" -o -n "$(PATH)" || (echo "ERROR: REPO or PATH is required."; exit 1) + $(UV) run python scripts/register_from_classification.py \ + $(if $(PATH),--repo-path "$(PATH)",--slug "$(REPO)") \ + $(if $(DRY_RUN),--dry-run,) + +## Bulk register/update all active repos with accessible local paths +register-from-classification-all: + $(UV) run python scripts/register_from_classification.py --bulk \ + $(if $(DRY_RUN),--dry-run,) + +clean: + $(COMPOSE) down -v diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r1/state-hub/deploy/railiance/apps/charts/state-hub/Chart.yaml b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r1/state-hub/deploy/railiance/apps/charts/state-hub/Chart.yaml new file mode 100644 index 00000000..1cb3040f --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r1/state-hub/deploy/railiance/apps/charts/state-hub/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: state-hub +description: State Hub API service for private Railiance operation +type: application +version: 0.1.0 +appVersion: "main-51bc70d" diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r1/state-hub/deploy/railiance/apps/helm/state-hub-values.yaml b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r1/state-hub/deploy/railiance/apps/helm/state-hub-values.yaml new file mode 100644 index 00000000..02a4bd62 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r1/state-hub/deploy/railiance/apps/helm/state-hub-values.yaml @@ -0,0 +1,69 @@ +# Production values for the State Hub Railiance chart handoff. +# Non-secret values only. DATABASE_URL comes from the Secret `state-hub-env`. + +namespace: + create: false + +image: + repository: forgejo.coulomb.social/coulomb/state-hub + tag: "main-51bc70d@sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015" + +ingress: + enabled: false + +# MCP layer on central (CUST-WP-0067-T08). Enabled together with the image tag +# above: MCP_HOST landed in main-6c1262e, and enabling this on an older image +# would deploy a pod that binds loopback and never becomes reachable. +# ClusterIP only — no ingress, same reason the API's is disabled. +mcp: + enabled: true + resources: + requests: + # 2026-09-10: 1,078 observed 5-minute samples, p99 2.35m, peak 30.45m. + # Preserve the 500m limit for bursts; free 40m for the API rollout surge. + cpu: 10m + +config: + # This deployment is the authoritative hub. Callers verify this rather than + # trusting that whatever answered on a port is central (CUST-WP-0067-T03). + instanceRole: primary + instanceLabel: railiance01 + sbomNexusUrl: "http://sbom-nexus.sbom-nexus.svc.cluster.local:8010" + # Reversible T04 read cutover; set back to `legacy` to roll back. + sbomNexusReadMode: nexus + # Reversible T04 write cutover; set back to `legacy` to roll back future writes. + sbomNexusWriteMode: nexus + +resources: + # Restored to the chart's 250m on 2026-09-21. The earlier 100m was a workaround + # for a node with <250m unallocated; ~830m of Knative/Kourier requests were + # released that day (the-custodian docs/kubernetes-change-gate-decision.md). + # State Hub idles near 5m but bursts past 400m under concurrent agent load. + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 2Gi + +sweep: + # RMGR-WP-0005-T11: disabled while railiance01 checkouts still target the + # stale gitea-remote lineage. Re-enable only after the governed remote + # reconciliation and registrar preflight are complete. + enabled: false + # Present in the live release but previously missing here, so deploying from + # this file alone silently dropped it. Inert while enabled is false. + hostname: 239.62.205.92.host.secureserver.net + hostPath: /home/tegwick + sshHostPath: /home/tegwick/.ssh + +# Forge read credential (STATE-WP-0084-T02). Coordinates only — the token lives +# in OpenBao at platform/workloads/state-hub/forge-derivation and reaches the +# pod through Kubernetes auth, never through this file or a Secret. +forgeRead: + enabled: true + +# RPF-WP-0035-T04: dedicated ESO signing key, API-only explicit reference. +renamePreflight: + enabled: true + secretName: state-hub-rename-preflight diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r1/state-hub/docs/evidence/2026-10-17-cluster-historical-receiving-rollout.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r1/state-hub/docs/evidence/2026-10-17-cluster-historical-receiving-rollout.json new file mode 100644 index 00000000..8e952e6b --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r1/state-hub/docs/evidence/2026-10-17-cluster-historical-receiving-rollout.json @@ -0,0 +1,102 @@ +{ + "schema": "statehub.cluster-historical-receiving-delivered.v1", + "status": "verified", + "proposal_id": "STATE-WP-0095-T06-cluster-frozen-receiving-r1", + "approved_revision": 4, + "approved_digest_sha256": "sha256:63a02e8296d126c0158b7eb2c4ce79c7eeb6f8d6f61298d473c1065c7557ba32", + "approval_decision_id": "e3798419-8aa9-5425-8c08-20df6bd82a19", + "source_revision": "51bc70d1ff8f25450a8650d8e50caf053093a02e", + "readiness_source_revision": "086d79a2736e34b916bd7de3c8a65e2ea38a3d64", + "helm_revision": 76, + "image_reference": "forgejo.coulomb.social/coulomb/state-hub:main-51bc70d@sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015", + "expected_schema": "c4d5e6f7a8b9", + "health": { + "status": "ok", + "instance_role": "primary", + "instance_label": "railiance01", + "schema": { + "status": "ok", + "applied": "c4d5e6f7a8b9", + "expected": "c4d5e6f7a8b9" + } + }, + "pods_ready": true, + "historical_module_sha256": "sha256:e3c4bf965764b32a6df5d61c71272ee22569df9a9ddae2a3fc3df97ec554e0b2", + "historical_preservation_verified": true, + "parents": 3, + "tasks": 15, + "full_historical_rows_equal": true, + "native_rollback_probes": { + "foreign-holder": { + "full_historical_rows_restored": true, + "historical_binding_refused": true, + "native_outcome_status": "refused", + "rolled_back": true, + "status": "passed" + }, + "missing-task": { + "full_historical_rows_restored": true, + "historical_binding_refused": true, + "native_outcome_status": "refused", + "rolled_back": true, + "status": "passed" + }, + "nonterminal-parent": { + "full_historical_rows_restored": true, + "historical_binding_refused": true, + "native_outcome_status": "refused", + "rolled_back": true, + "status": "passed" + }, + "nonterminal-task": { + "full_historical_rows_restored": true, + "historical_binding_refused": true, + "native_outcome_status": "refused", + "rolled_back": true, + "status": "passed" + }, + "preserve": { + "full_historical_rows_equal": true, + "full_historical_rows_restored": true, + "native_outcome_status": "applied", + "rolled_back": true, + "status": "passed" + } + }, + "native_probe_commits": 0, + "fresh_transaction_full_rows_equal": true, + "source_receiving_gate_delivered": true, + "normal_cluster_publication_verified": false, + "registrar_admission_or_cluster_publication_not_inferred": true, + "authority_limits": [ + "Rollback probe SQL effects rolled back; foreign-holder case is historical-binding refusal, not separate global holder branch.", + "Actual historical native receiving is verified; owner-native TLS/private admission remains unresolved.", + "Migration schema remains additive; Helm rollback does not downgrade schema or stop an active hook Job." + ], + "evidence": [ + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/approval-decision.json", + "sha256": "sha256:b51554c004daa882f3709f67bdae7f2e939d611120383e61be04ec6969878134" + }, + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/helm-return.json", + "sha256": "sha256:61727deacf22d37da2d28cbaf04f5a3ddee0f9ff7571c77f1e85dba97963b572" + }, + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/delivered-runtime.json", + "sha256": "sha256:89681066e8fe139456163ebfafa3efe77ffae8534ea167219230b0a74367f129" + }, + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/native-refusal-probes.json", + "sha256": "sha256:673ccef965c820704dc310a1bf7832b583c30c3edfeb814528cf892555199172" + }, + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/before-full-row-hashes.json", + "sha256": "sha256:75cdd87eef79059ff3f484ed24ca69e285899cdf8d851cc4e472653c417eb0ee" + }, + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/after-full-row-hashes.json", + "sha256": "sha256:2156aad18f190384838c27e792706a156a7423391408afbc6107e6c28c5872d8" + } + ] +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r1/state-hub/workplans/STATE-WP-0095-historical-source-receiving.md b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r1/state-hub/workplans/STATE-WP-0095-historical-source-receiving.md new file mode 100644 index 00000000..926b0fd8 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r1/state-hub/workplans/STATE-WP-0095-historical-source-receiving.md @@ -0,0 +1,271 @@ +--- +id: STATE-WP-0095 +type: workplan +title: "Preserve exact closed historical source bindings in Forge receiving" +domain: infotech +repo: state-hub +status: active +owner: codex +created: "2026-10-04" +updated: "2026-10-17" +flavor: implementation +depends_on: [] +state_hub_workstream_id: "79579572-8120-5d3b-aca5-dd4aff60d058" +--- + +## Goal + +Normal Forge receiving must preserve the seven bound Activity legacy source +views and the Custodian retired ADHOC source without new identities, source +rewrites, task movement, deletion, relabeling or unretirement. This implements +receiving support before RMGR-WP-0019 registrar admission. CUST-WP-0077 owns +cross-owner coordination; deployment and replay remain explicit owner returns. + +## Verify source bindings and shared views + +```task +id: STATE-WP-0095-T01 +status: done +priority: high +state_hub_task_id: "6a5f87bb-e63d-5e20-b9fc-c1b5cd403ff7" +``` + +Reviewed seven exact Activity files: four existing workplans and 64 UUID-bound +closed tasks. The archived custodian-WP-0003 parent and three split_from views +share one existing parent UUID; every repeated task binding/content agrees. +Custodian ADHOC-2026-08-25-retired retains parent +1c53d3db-6633-5e2d-987f-10706d73be9e and task +d8af3e6b-26ae-580c-bbeb-caa8ad419bec. No identity migration is authorized. + +## Implement preservation receiving and global collision refusal + +```task +id: STATE-WP-0095-T02 +status: done +priority: high +state_hub_task_id: "ebb24194-e4cc-5130-aaa6-663f2ea2c822" +``` + +Prepared exact source digest manifest, existing-parent/task verification, +immutable receiving and UUID-based historical diff. Ordinary canonical sources +continue deriving their identities. Missing/changed/symlinked source, absent +bound rows or nonterminal hub histories refuse. Check task UUID holders globally, +including retired and acknowledged-stale parent workplans, before INSERT. + +Offline preview tests and snapshot proof are evidence of source behavior only; +finish after applying reviewed source and its focused tests to the owning repo. + +## Deliver receiving through normal deployment and verify owner return + +```task +id: STATE-WP-0095-T03 +status: done +priority: high +depends_on: [STATE-WP-0095-T02] +state_hub_task_id: "e8566341-cf6e-5b9c-ad80-8564d14cd52a" +decision_id: "36772026-f43d-4331-b466-5f4b1afe4eca" +``` + +No direct live code patch. This task is a deployment return, not policy authority +or permission to resolve identity decisions. Exact receiving fixture acceptance +and global collision rejection must hold in the delivered version. + +## Verify normal historical replay preserves existing identities + +```task +id: STATE-WP-0095-T04 +status: done +priority: high +depends_on: [STATE-WP-0095-T03, RMGR-WP-0019-T03] +state_hub_task_id: "3bbb311f-194e-5862-8aed-358a580f22b4" +``` + +Verify Activity four historical parent UUIDs/64 task UUIDs and Custodian retired +parent/task unchanged; no historical row creation, movement, deletion, +relabeling or unretirement. Unrelated stale projections remain separate decisions. + +2026-10-05: Source exact-set retirement support now exists. Repository projection +reconcile accepts `expected_retirement_uuids`; that set must equal the stale +projections, and any task or dependency edge on a stale row refuses the call. +Live apply still needs a reviewed rollout. On 2026-10-05 the founder accepted +review `review-1ece4f3e-d0e3-4fd6-abc8-272f1b1876f4`, which names only the three +empty projections and preserves parent `b4eb45a9-69e3-4ab0-b00c-67a53c3117c5`. +That acceptance is not a live retirement and does not resolve +`91b7c633-a66f-5d43-9f4b-6565aefb9356`. + +## Offline evidence, 2026-10-04 + +Prepared receiver suite: Forge projection regression plus historical preservation, +missing/changed source, terminality, immutable UUID diff, and global holder query. +Real-source offline snapshot proof preserves four Activity parents/64 tasks and +one Custodian parent/task without mutations; no API writes or deployment occurred. +Exact original bindings are maintained in api/services/historical_projection.py. + +2026-10-04 actual owning-source validation: 155 focused receiver/qualifier/planner +cases passed; the independent offline current-source/current-hub-snapshot proof +retained five existing parents and 65 task UUIDs, no mutations, clean historical +diff. T02 is source-complete; T03/T04 remain wait for reviewed deployment and +normal registrar/replay receiving, respectively. Evidence: docs/evidence/2026-10-04-historical-receiving-source.json. + + +## Verified receiving rollout and remaining replay gate, 2026-10-04 + +T03 is done on actual founder approval of proposal +STATE-WP-0095-T03-historical-receiving revision 1, SHA256 +`d6789359b3fe7988a6201fff6f367cb46948aa197e1cc0ca66c31f45e1ea9981`, +approval receipt reference `user_reply:call_o87uEfFs3AtXfxO5YqUnSpcg:0`. +The normal owner Helm path delivered `main-5dd3df9`, commit +`5dd3df9c69eb564af10ea5ea01e349786212501a`, revision 72; API and MCP are Ready, +primary/railiance01 health and schema a0b1c2d3e4f5 verified, running source +hashes match. Rollback remains revision 71; no receipt-table downgrade is needed. +Actual historical preservation and global task-holder refusal were verified in +the delivered version. Durable receipt: +[verified historical receiving rollout](../docs/evidence/2026-10-04-historical-receiving-rollout.json). + +Custodian immutable replay applied while preserving the retired historical +parent/task UUIDs. Activity immutable replay retained all four historical parent +and 64 task UUIDs and historical row contents, but refused transactionally on +three unrelated stale projections: custodian-wp-0003 (archived), +custodian-wp-0003a (done), custodian-wp-0003b (done). No retirement acknowledgement +was issued and that refusal is not represented as normal replay success. + +Registrar source `c41d36c2` has 38 actual passing tests and Repo Manager's own +normal sync succeeded; registrar admission does not authorize retiring Activity +projections. T04 therefore remains wait, retaining RMGR-WP-0019-T03 and awaiting +Activity Core's precise reviewed disposition of those three stale rows. No +rename, identity migration, new paid native trial or further live action is +included in this source status update. + +## Activity Core return, 2026-10-06 + +Activity Core returned the exact disposition (message +`cf2d5847-8b3f-4653-9d9f-d5fcfbe3f1e4`, ACTIVITY-WP-0045, activity-core commit +`941fac0`). `expected_retirement_uuids`: +`683724d9-4249-5e0e-88e0-7baff4330d58` (custodian-wp-0003), +`644017c7-c5f2-5a64-81c3-d8ee806f8251` (custodian-wp-0003a), +`a5806df9-5b9a-50da-81c1-7e606076bed9` (custodian-wp-0003b). Each has zero +tasks and zero dependency edges. None is preserved parent +`b4eb45a9-69e3-4ab0-b00c-67a53c3117c5`. Repo Manager independently verified the +set (message `916841f6`, repo-manager commit `498682c`). Nothing was retired. +T04 now waits on decision `91b7c633` and a reviewed live rollout. After that, +Repo Manager re-runs the Activity owner sync and closes RMGR-WP-0019-T03. + +## Retirement rollout and T04 verification, 2026-10-06 + +The founder resolved decision `91b7c633` (retire exactly the three projections) +and approved the rollout. The founder ran the normal Helm upgrade to +`main-968a811` (revision 74; rollback revision 73, `main-fad9b3f`). API and MCP +are Ready, health is ok on primary/railiance01, and schema `b3c4d5e6f7a8` is +unchanged; no migrations. The activity-core reconcile at commit +`67a92d2ff8e033baf2c36998d24187fe83edec84` with `expected_retirement_uuids` set +to the three UUIDs applied: 0 created, 3 retired (tombstoned with an +`@retired-20261006` slug; UUIDs and statuses kept), 0 refused, 0 tasks +cancelled. All 7 bound Activity source files still match their digests. The four +historical parents and 64 task UUIDs are present under their bound parents with +unchanged fields. The Custodian retired parent and task are unchanged. Activity +had 54 workplans and 288 tasks before and after. Evidence: +[activity stale retirement rollout](../docs/evidence/2026-10-06-activity-stale-retirement-rollout.json). +Repo Manager's re-run of the Activity owner sync (RMGR-WP-0019-T03) and +ACTIVITY-WP-0045-T02 remain with their owners. + + +## Cluster exact historical receiving successor, 2026-10-17 + +This scoped extension retains the completed Activity/Custodian outcomes above. +Three frozen cluster archives contain three existing parent UUIDs and 15 existing +closed task UUIDs. Extend the existing exact historical receiving path before +registrar admission; never derive replacement identities or modify those +sources. Cross-owner observation attaches to existing SHR-WP-0001, with +RMGR-WP-0019 as obligor/beneficiary and NK-WP-0049-T35 benefiting indirectly. +The local derived case is waiting; no live worker injection is authorized. + +## Prepare exact cluster source preservation and refusal cases + +```task +id: STATE-WP-0095-T05 +status: done +priority: high +state_hub_task_id: "339b9a37-181c-5599-9a89-0393e199bd4b" +``` + +Prepared the same full-file SHA256 and 15 task binding manifest as Repo Manager. +Candidate receiving tests pass 26: existing rows remain entirely unchanged; +missing/foreign/nonterminal rows and unknown, changed, missing or symlinked +source refuse. Read-only public metadata confirms all 18 existing rows have the +expected ownership and terminal status. These are source and metadata checks, +not delivered receiving or a committed native trial. + +## Deliver reviewed cluster receiving successor before registrar admission + +```task +id: STATE-WP-0095-T06 +status: done +priority: high +depends_on: [STATE-WP-0095-T05] +needs_human: false +intervention_note: "Exact revision 4 approved and verified; no pending rollout intervention." +state_hub_task_id: "6af973b4-1964-5d4d-a2f8-8dbd4c01fda4" +``` + +Completed 2026-10-17 on the founder's approval of exact revision 4 of stable +proposal STATE-WP-0095-T06-cluster-frozen-receiving-r1 (decision + e3798419-8aa9-5425-8c08-20df6bd82a19; approved digest + sha256:63a02e8296d126c0158b7eb2c4ce79c7eeb6f8d6f61298d473c1065c7557ba32). +The ordinary owner Helm path delivered revision 76, runtime source 51bc70d and +exact image manifest sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015. +API/MCP Ready and primary/railiance01 health report expected schema +c4d5e6f7a8b9; delivered historical module sha256:e3c4bf965764b32a6df5d61c71272ee22569df9a9ddae2a3fc3df97ec554e0b2 matches. +All three parent and 15 task full DB row hashes/columns/UUIDs remained unchanged. +Five actual native probes passed: preserve, actual foreign-parent holder move, +nonterminal task, nonterminal parent and missing task. Every probe rolled back; +a fresh transaction confirmed complete row restoration. The foreign-holder +case precisely verified historical binding refusal, not the separate global +UUID-holder creation branch. No committed probe writes were made. + +Evidence: [verified receiving rollout](../docs/evidence/2026-10-17-cluster-historical-receiving-rollout.json). +This actual receiving return satisfies the prerequisite of RMGR-WP-0019-T05. +Normal cluster publication and its immutable-source/row return remain T07; +source readiness and this deployment do not count that later acceptance twice. +The additive inbox migration is retained; Helm rollback does not downgrade +schema or stop an active migration Job. Original failures and exact-UID recovery +limits remain in the referenced approved packet. No TLS recovery/native private +admission or new autonomy/cost grant is inferred. + +## Verify ordinary cluster receiving and immutable historical rows + +```task +id: STATE-WP-0095-T07 +status: wait +priority: high +depends_on: [STATE-WP-0095-T06, RMGR-WP-0019-T06] +state_hub_task_id: "cfd1ea99-e75b-542e-a434-acefb62f3c7d" +``` + +Capture normal cluster owner publication/receiving receipts and compare entire +existing row snapshots and exact source bytes. No creation, movement, relabel, +unretirement or retirement is authorized for these histories. A rollback first +fences cluster sync and restores registrar refusal before removing receiving +support; it never rewrites archived IDs or UUIDs. This work does not establish +original TLS recovery or owner-native admission. + +## Fix policy-compatible Secret-template server dry-run preparation + +```task +id: STATE-WP-0095-T08 +status: done +priority: high +state_hub_task_id: "2dd9b04f-2250-5a8a-b5bc-c8a009872858" +``` + +Prepared the narrow Makefile successor after the existing readiness command +refused its first Secret-template phase. Ordinary client-side apply adds the +last-applied annotation, correctly denied by the existing Platform +reject-secret-last-applied policy. Only the two Secret-template server +dry-run requests now use server-side apply with dry-run; no force-conflicts, +Secret value change, policy relaxation or actual Secret write is introduced. +Command-fixture tests verify both requests and preserve other dry-run phases. +Evidence and original refusal remain in the Custodian cluster historical +publication packet, revisions/r2 and r3-readiness. This is completed source +preparation; T06 separately records the verified actual rollout. The runtime image +continues to be main-51bc70d; Makefile/workplan changes do not create a new +application image or native acceptance. diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/Makefile b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/Makefile new file mode 100644 index 00000000..cec684ec --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/Makefile @@ -0,0 +1,517 @@ +.PHONY: start install install-cli dashboard-install dashboard-check db db-tools migrate seed api dashboard dashboard-local sync check-primary primary-port-clear check check-local test test-python clean register-project register-codex-project register-mcp configure-codex bootstrap-env dev-hub edge-relay mcp-profile validate-adr add-domain rename-domain add-repo list-repos register-path register-from-classification register-from-classification-all cleanup-stale tunnels-up tunnels-status tunnels-check bridges install-hooks install-hooks-all gitea-inventory token-reconcile railiance-state-hub-render railiance-state-hub-client-dry-run railiance-state-hub-server-dry-run railiance-state-hub-headroom + +COMPOSE = docker compose -f infra/docker-compose.yml --env-file .env +PYTHON ?= python3 +HELM ?= $(shell command -v helm 2>/dev/null || if [ -x "$$HOME/.local/bin/helm" ]; then printf "%s" "$$HOME/.local/bin/helm"; else printf "%s" "helm"; fi) +KUBECTL ?= $(shell command -v kubectl 2>/dev/null || if [ -x "$$HOME/.local/bin/kubectl" ]; then printf "%s" "$$HOME/.local/bin/kubectl"; else printf "%s" "kubectl"; fi) + +RAILIANCE_STATE_HUB_RELEASE ?= state-hub +RAILIANCE_STATE_HUB_NAMESPACE ?= state-hub +RAILIANCE_STATE_HUB_CHART ?= deploy/railiance/apps/charts/state-hub +RAILIANCE_STATE_HUB_VALUES ?= deploy/railiance/apps/helm/state-hub-values.yaml +RAILIANCE_STATE_HUB_IMAGE_TAG ?= main-51bc70d@sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015 +RAILIANCE_STATE_HUB_PLATFORM_DIR ?= deploy/railiance/platform +RAILIANCE_STATE_HUB_APP_MANIFESTS ?= deploy/railiance/apps/manifests +# Codex/WSL non-login shells may not source ~/.profile; keep uv discoverable. +UV ?= $(shell command -v uv 2>/dev/null || if [ -x "$$HOME/.local/bin/uv" ]; then printf "%s" "$$HOME/.local/bin/uv"; else printf "%s" "uv"; fi) +RMGR ?= $(shell command -v rmgr 2>/dev/null || if [ -x "$$HOME/repo-manager/.venv/bin/rmgr" ]; then printf "%s" "$$HOME/repo-manager/.venv/bin/rmgr"; else printf "%s" "rmgr"; fi) +STATE_HUB_API_BASE ?= http://127.0.0.1:8000 +SYNC_PATH ?= . +SYNC_PUSH ?= 1 + +start: + @echo "# Normal production UI access (run in order)" + @echo "make bridges # connect local :8000 to the railiance01 primary" + @echo "make dashboard # verify the primary, then serve the UI on :3000" + @echo "make sync # push file-backed records and reconcile the exact forge commit" + @echo "" + @echo "# Optional local MCP adapter" + @echo "make mcp-http # local SSE adapter on :8001; not needed by the UI" + @echo "" + @echo "# Local fallback/development only" + @echo "make api # local Postgres + API; conflicts with the production :8000 tunnel" + @echo "make dashboard-local # dashboard against the deliberate local API" + +install: + $(UV) sync + +dashboard/node_modules/.bin/observable: dashboard/package.json dashboard/package-lock.json + cd dashboard && npm ci + +dashboard-install: dashboard/node_modules/.bin/observable + +dashboard-check: dashboard-install + cd dashboard && npm run build + +## Symlink the custodian CLI into ~/.local/bin so it's on PATH system-wide +install-cli: install + mkdir -p ~/.local/bin + ln -sf "$(shell pwd)/.venv/bin/custodian" ~/.local/bin/custodian + ln -sf "$(shell pwd)/.venv/bin/statehub" ~/.local/bin/statehub + @echo "Installed: custodian → $$(readlink -f ~/.local/bin/custodian)" + @echo "Installed: statehub → $$(readlink -f ~/.local/bin/statehub)" + @echo "Make sure ~/.local/bin is on your PATH:" + @echo " echo 'export PATH=\"\$$HOME/.local/bin:\$$PATH\"' >> ~/.bashrc && source ~/.bashrc" + +db: + $(COMPOSE) up -d postgres + +db-tools: + $(COMPOSE) --profile tools up -d + +migrate: + $(UV) run alembic upgrade head + +seed: + $(UV) run python scripts/seed.py + +## Start (or restart) the MCP SSE server on :8001 — primary transport for Claude Code. +## Remote clients (e.g. COULOMBCORE) connect via the ops-bridge tunnel (port 18001). +## Registration: claude mcp add-json -s user state-hub '{"type":"sse","url":"http://127.0.0.1:8001/sse"}' +mcp-http: + @fuser -k 8001/tcp 2>/dev/null && echo "Stopped running MCP server" || true + MCP_TRANSPORT=sse MCP_PORT=8001 $(UV) run python mcp_server/server.py + +## Require the production identity before serving the normal dashboard. This +## prevents an accidental local `make api` from presenting an empty fallback DB +## as the live State Hub. +check-primary: + @health="$$(curl -fsS --max-time 5 http://127.0.0.1:8000/state/health 2>/dev/null)" || { \ + echo "ERROR: State Hub primary is not reachable on 127.0.0.1:8000." >&2; \ + echo "Run 'make bridges' first." >&2; \ + exit 1; \ + }; \ + identity="$$(printf '%s' "$$health" | $(PYTHON) -c 'import json, sys; d=json.load(sys.stdin); print("{}/{}".format(d.get("instance_role", ""), d.get("instance_label", "")))')"; \ + if [ "$$identity" != "primary/railiance01" ]; then \ + echo "ERROR: 127.0.0.1:8000 is '$$identity', not the primary/railiance01 State Hub." >&2; \ + echo "Stop the local API, then run 'make bridges'. Use 'make dashboard-local' only for intentional local development." >&2; \ + exit 1; \ + fi + +## Refuse to start the production tunnel when a non-primary API already owns +## port 8000. With no listener, bridge is free to establish the tunnel. +primary-port-clear: + @if health="$$(curl -fsS --max-time 2 http://127.0.0.1:8000/state/health 2>/dev/null)"; then \ + identity="$$(printf '%s' "$$health" | $(PYTHON) -c 'import json, sys; d=json.load(sys.stdin); print("{}/{}".format(d.get("instance_role", ""), d.get("instance_label", "")))')"; \ + if [ "$$identity" != "primary/railiance01" ]; then \ + echo "ERROR: port 8000 is occupied by '$$identity'." >&2; \ + echo "Stop the local 'make api' process before starting the production tunnel." >&2; \ + exit 1; \ + fi; \ + fi + +dashboard: check-primary + @fuser -k 3000/tcp 2>/dev/null && echo "Stopped running dashboard" || true + $(MAKE) dashboard-install + cd dashboard && npm run dev + +## Deliberate local-development dashboard; bypasses the production identity gate. +dashboard-local: + @echo "WARNING: serving the dashboard against the local/fallback API on :8000." + @fuser -k 3000/tcp 2>/dev/null && echo "Stopped running dashboard" || true + $(MAKE) dashboard-install + cd dashboard && npm run dev + +## Fast work-record path. Repo Manager assigns only missing deterministic IDs, +## verifies the primary/railliance01 bridge, and asks central to derive the +## exact pushed Forgejo commit in one transactional request. +sync: + $(RMGR) sync --path "$(SYNC_PATH)" --api-base "$(STATE_HUB_API_BASE)" $(if $(filter 1 true yes,$(SYNC_PUSH)),--push,) + +check: check-primary + @echo "State Hub primary/railiance01 is healthy." + +check-local: + curl -sf http://127.0.0.1:8000/state/health | python3 -m json.tool + +# CUST-WP-0067-T09. The chart ships a copy of the-custodian canon allowed-values +# because a container has no checkout to read it from. Regenerate after any +# canon vocabulary change; the -check target fails on drift so a release cannot +# validate classification against a stale vocabulary. +sync-classification-allowed: + python3 scripts/sync_classification_allowed.py + +check-classification-allowed: + python3 scripts/sync_classification_allowed.py --check + +# STATE-WP-0091: read-only surge-headroom check. Does not start Helm. +# Live: make railiance-state-hub-headroom +# Fixtures: make railiance-state-hub-headroom HEADROOM_FIXTURE=tests/fixtures/release_headroom/rev63-105m.json +HEADROOM_FIXTURE ?= +railiance-state-hub-headroom: + @if [ -n "$(HEADROOM_FIXTURE)" ]; then \ + $(PYTHON) scripts/release_headroom_preflight.py --fixture "$(HEADROOM_FIXTURE)"; \ + else \ + $(PYTHON) scripts/release_headroom_preflight.py --kubectl "$(KUBECTL)"; \ + fi + +railiance-state-hub-render: check-classification-allowed + $(HELM) template $(RAILIANCE_STATE_HUB_RELEASE) $(RAILIANCE_STATE_HUB_CHART) \ + --namespace $(RAILIANCE_STATE_HUB_NAMESPACE) \ + -f $(RAILIANCE_STATE_HUB_VALUES) \ + --set image.tag=$(RAILIANCE_STATE_HUB_IMAGE_TAG) + +railiance-state-hub-client-dry-run: + @set -e; \ + tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + $(HELM) template $(RAILIANCE_STATE_HUB_RELEASE) $(RAILIANCE_STATE_HUB_CHART) \ + --namespace $(RAILIANCE_STATE_HUB_NAMESPACE) \ + -f $(RAILIANCE_STATE_HUB_VALUES) \ + --set image.tag=$(RAILIANCE_STATE_HUB_IMAGE_TAG) > "$$tmpdir/state-hub.yaml"; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-credentials.sops.yaml.template; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-cluster.yaml; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-networkpolicies.yaml; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-namespace.yaml; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-env.secret.sops.yaml.template; \ + $(KUBECTL) apply --dry-run=client -n $(RAILIANCE_STATE_HUB_NAMESPACE) -f "$$tmpdir/state-hub.yaml" + +railiance-state-hub-server-dry-run: + @set -e; \ + tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + $(HELM) template $(RAILIANCE_STATE_HUB_RELEASE) $(RAILIANCE_STATE_HUB_CHART) \ + --namespace $(RAILIANCE_STATE_HUB_NAMESPACE) \ + -f $(RAILIANCE_STATE_HUB_VALUES) \ + --set image.tag=$(RAILIANCE_STATE_HUB_IMAGE_TAG) > "$$tmpdir/state-hub.yaml"; \ + $(KUBECTL) apply --server-side --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-credentials.sops.yaml.template; \ + $(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-cluster.yaml; \ + $(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-networkpolicies.yaml; \ + $(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-namespace.yaml; \ + if $(KUBECTL) get namespace $(RAILIANCE_STATE_HUB_NAMESPACE) >/dev/null 2>&1; then \ + $(KUBECTL) apply --server-side --dry-run=server -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-env.secret.sops.yaml.template; \ + $(KUBECTL) apply --dry-run=server -n $(RAILIANCE_STATE_HUB_NAMESPACE) -f "$$tmpdir/state-hub.yaml"; \ + else \ + echo "Namespace $(RAILIANCE_STATE_HUB_NAMESPACE) does not exist; validating namespaced app manifests with client dry-run."; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-namespace.yaml; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-env.secret.sops.yaml.template; \ + $(KUBECTL) apply --dry-run=client -n $(RAILIANCE_STATE_HUB_NAMESPACE) -f "$$tmpdir/state-hub.yaml"; \ + fi + +test: test-python dashboard-check + +test-python: + TEST_DATABASE_URL=postgresql+asyncpg://custodian:changeme@127.0.0.1:5432/custodian_test \ + $(UV) run pytest -x -q + +## Benchmark /state/summary revision cache (API must be running on :8000) +benchmark-summary-cache: + $(UV) run python scripts/benchmark_summary_cache.py + +## ops-bridge managed tunnels +## Requires ops-bridge: bridge is at /home/worsch/.local/bin/bridge +tunnels-up: primary-port-clear + bridge up + +tunnels-status: + bridge status + +## End-to-end check: verifies SSH process alive + remote port listening on COULOMBCORE. +## Exits non-zero if any tunnel is not fully operational. +tunnels-check: + bridge check + +## Ensure all ops-bridge tunnels are up and healthy. +## Brings up any stopped/stale tunnels, shows final status, exits non-zero if anything is still down. +bridges: primary-port-clear + @echo "==> Bringing up all tunnels..." + bridge up + @echo "" + @echo "==> Tunnel status:" + bridge status + @echo "" + @echo "==> Checking tunnel health..." + bridge check + +## Start (or restart) the LOCAL FALLBACK backend — db + migrate + uvicorn. +## This replaces anything on :8000, including the production State Hub tunnel. +api: db + @echo "WARNING: starting the local fallback API; this is not the railiance01 primary." + @echo "Waiting for postgres..."; \ + for i in 1 2 3 4 5 6 7 8 9 10; do \ + nc -z 127.0.0.1 5432 2>/dev/null && break; \ + sleep 1; \ + done + $(MAKE) migrate + @fuser -k 8000/tcp 2>/dev/null && echo "Stopped running API" || true + $(UV) run uvicorn api.main:app --reload --reload-dir api --reload-dir mcp_server --reload-dir task_flow_engine --host 127.0.0.1 --port 8000 + +## Register a project (Claude Code): make register-project DOMAIN=railiance PROJECT_PATH=/home/worsch/railiance +register-project: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required. Usage: make register-project DOMAIN= PROJECT_PATH="; exit 1) + @test -n "$(PROJECT_PATH)" || (echo "ERROR: PROJECT_PATH is required."; exit 1) + scripts/register_project.sh "$(DOMAIN)" "$(PROJECT_PATH)" + +## Register a Codex project (AGENTS.md + HTTP API): make register-codex-project DOMAIN=capabilities PROJECT_PATH=/home/worsch/my-repo +register-codex-project: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required. Usage: make register-codex-project DOMAIN= PROJECT_PATH="; exit 1) + @test -n "$(PROJECT_PATH)" || (echo "ERROR: PROJECT_PATH is required."; exit 1) + scripts/register_project.sh "$(DOMAIN)" "$(PROJECT_PATH)" --codex + +## Register State Hub MCP for Claude Code. Optional: make register-mcp MCP_URL=http://127.0.0.1:18001/sse +register-mcp: + scripts/register-mcp.sh \ + $(if $(MCP_URL),--url "$(MCP_URL)",) \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(DRY_RUN),--dry-run,) + +## Configure Codex workspace-write networking for local State Hub access. +configure-codex: + scripts/configure-codex.sh \ + $(if $(CODEX_HOME),--codex-home "$(CODEX_HOME)",) \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(WITH_MCP),--with-mcp,) \ + $(if $(DRY_RUN),--dry-run,) + +## Bootstrap a new operator/collaborator environment. Optional: make bootstrap-env ARGS="--install-missing" +bootstrap-env: + scripts/bootstrap-env.sh $(ARGS) + +## Files-first local dev hub (CUST-WP-0054-T07): postgres + migrate + seed + API + repo registration. +## Optional: WITH_EDGE=1 WITH_MCP=1 make dev-hub +## Fleet tunnel when reachable: make dev-hub PROFILE=fleet +dev-hub: + bash scripts/dev_hub_up.sh $(if $(PROFILE),$(PROFILE),dev) + +## Local edge relay for offline write buffering (STATE-WP-0068). +edge-relay: + @fuser -k 18080/tcp 2>/dev/null && echo "Stopped running edge relay" || true + STATEHUB_UPSTREAM_URL=$${STATEHUB_UPSTREAM_URL:-http://127.0.0.1:8000} \ + $(UV) run uvicorn api.edge.relay:app --host 127.0.0.1 --port 18080 + +## Switch MCP/API profile: make mcp-profile PROFILE=dev|fleet|status +mcp-profile: + @test -n "$(PROFILE)" || (echo "ERROR: PROFILE is required (dev|fleet|status)."; exit 1) + bash scripts/mcp_hub_profile.sh "$(PROFILE)" + +## Add a second repo to an existing domain: make add-repo DOMAIN=railiance REPO_PATH=/home/worsch/railiance-infra +add-repo: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required."; exit 1) + @test -n "$(REPO_PATH)" || (echo "ERROR: REPO_PATH is required."; exit 1) + scripts/register_project.sh "$(DOMAIN)" "$(REPO_PATH)" --additional + +## Create a new domain: make add-domain DOMAIN=my_domain NAME="My Domain" +add-domain: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required (slug)."; exit 1) + @test -n "$(NAME)" || (echo "ERROR: NAME is required (display name)."; exit 1) + curl -sf -X POST http://127.0.0.1:8000/domains/ \ + -H "Content-Type: application/json" \ + -d "{\"slug\": \"$(DOMAIN)\", \"name\": \"$(NAME)\"}" | python3 -m json.tool + +## Rename a domain: make rename-domain DOMAIN=old_slug NEW_SLUG=new_slug NEW_NAME="New Name" +rename-domain: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN (old slug) is required."; exit 1) + @test -n "$(NEW_SLUG)" || (echo "ERROR: NEW_SLUG is required."; exit 1) + @test -n "$(NEW_NAME)" || (echo "ERROR: NEW_NAME is required."; exit 1) + curl -sf -X PATCH http://127.0.0.1:8000/domains/$(DOMAIN)/rename \ + -H "Content-Type: application/json" \ + -d "{\"new_slug\": \"$(NEW_SLUG)\", \"new_name\": \"$(NEW_NAME)\"}" | python3 -m json.tool + +## Register this machine's local path for a repo: make register-path REPO=marki-docx PATH=/home/tegwick/marki-docx +register-path: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make register-path REPO= PATH="; exit 1) + @test -n "$(PATH)" || (echo "ERROR: PATH is required. Usage: make register-path REPO= PATH="; exit 1) + curl -sf -X POST "http://127.0.0.1:8000/repos/$(REPO)/paths" \ + -H "Content-Type: application/json" \ + -d "{\"host\": \"$$(hostname)\", \"path\": \"$(PATH)\"}" | python3 -m json.tool + +## List repos for a domain: make list-repos DOMAIN=railiance +list-repos: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required."; exit 1) + curl -sf "http://127.0.0.1:8000/repos/?domain=$(DOMAIN)" | python3 -m json.tool + +## Ingest SBOM data for a repo (all mechanisms: lockfiles + ansible + sbom-tools.yaml). +## Auto-detect all sources: make ingest-sbom REPO=the-custodian REPO_PATH=/home/worsch/the-custodian +## Single lockfile (explicit): make ingest-sbom REPO=the-custodian LOCKFILE=/path/to/uv.lock +## Dry-run (no submit): make ingest-sbom REPO=the-custodian REPO_PATH=... DRY_RUN=1 +## Tip: run capture-tools first for repos with system-level tool dependencies. +ingest-sbom: + @test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1) + $(UV) run python scripts/ingest_sbom.py --repo "$(REPO)" \ + $(if $(LOCKFILE),--lockfile "$(LOCKFILE)") \ + $(if $(REPO_PATH),--repo-path "$(REPO_PATH)") \ + $(if $(DRY_RUN),--dry-run) + +## Ingest capability declarations from SCOPE.md into the catalog. +## Usage: make ingest-capabilities REPO=the-custodian [REPO_PATH=/home/worsch/the-custodian] +## Or: make ingest-capabilities-all +## Add DRY_RUN=1 to preview without writing. +ingest-capabilities: + @test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1) + $(UV) run python scripts/ingest_capabilities.py --repo "$(REPO)" \ + $(if $(REPO_PATH),--repo-path "$(REPO_PATH)") \ + $(if $(DRY_RUN),--dry-run) + +ingest-capabilities-all: + $(UV) run python scripts/ingest_capabilities.py --all \ + $(if $(DRY_RUN),--dry-run) + +## Check Repository Definition of Integrated (DoI) criteria for a repo. +## Usage: make check-doi REPO=llm-connect +## Or: make check-doi-all +## Add JSON=1 for machine-readable output. +check-doi: + @test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1) + $(UV) run python scripts/check_doi.py --repo "$(REPO)" $(if $(JSON),--json) + +check-doi-all: + $(UV) run python scripts/check_doi.py --all $(if $(JSON),--json) + +## Ingest tpsc.yaml service declarations from a repo into the TPSC catalog. +## Usage: make ingest-tpsc REPO=llm-connect +## Or: make ingest-tpsc-all +## Add DRY_RUN=1 to preview without writing. +ingest-tpsc: + @test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1) + $(UV) run python scripts/ingest_tpsc.py --repo "$(REPO)" \ + $(if $(DRY_RUN),--dry-run) + +ingest-tpsc-all: + $(UV) run python scripts/ingest_tpsc.py --all \ + $(if $(DRY_RUN),--dry-run) + +## Run SBOM capture agent for a repo — generates/updates sbom-tools.yaml. +## Usage: make capture-tools REPO=railiance-infra [REPO_PATH=/home/worsch/railiance-infra] +## Add DRY_RUN=1 to preview without writing. +capture-tools: + @test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1) + $(UV) run python scripts/capture_sbom_tools.py --repo "$(REPO)" \ + $(if $(REPO_PATH),--repo-path "$(REPO_PATH)") \ + $(if $(DRY_RUN),--dry-run) + +## Check a repo for ADR-001 compliance: make validate-adr REPO=/path/to/repo [DOMAIN=custodian] +validate-adr: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make validate-adr REPO= [DOMAIN=]"; exit 1) + $(UV) run python scripts/validate_repo_adr.py "$(REPO)" $(if $(DOMAIN),--domain "$(DOMAIN)",) + +## Consistency exit contract: +## - Direct scripts/consistency_check.py: 0 clean, 2 warnings-only, 1 failures. +## - Agent/operator Make wrappers below normalize warning-only 2 to shell success +## while preserving visible WARN output and keeping real failures non-zero. +## Check a single repo for ADR-001 consistency: make check-consistency REPO=the-custodian [REPO_PATH=/override] +## Exit 0 = clean or warnings-only (warnings stay visible), exit 1 = failures +check-consistency: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make check-consistency REPO="; exit 1) + $(UV) run python scripts/consistency_check.py --repo "$(REPO)" \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(REPO_PATH),--repo-path "$(REPO_PATH)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Check and auto-fix a single repo: make fix-consistency REPO=the-custodian [REPO_PATH=/override] +## Exit 0 = clean or warnings-only (warnings stay visible), exit 1 = failures +fix-consistency: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make fix-consistency REPO="; exit 1) + $(UV) run python scripts/consistency_check.py --repo "$(REPO)" --fix \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(REPO_PATH),--repo-path "$(REPO_PATH)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Normalize workplan frontmatter and task status literals in attached repos. +## Usage: make normalize-attached-workplans REPO=artifact-store +## make normalize-attached-workplans DIRTY=1 +normalize-attached-workplans: + $(UV) run python scripts/normalize_attached_repo_workplans.py \ + $(if $(REPO),--repo "$(REPO)",) \ + $(if $(DIRTY),--dirty,) \ + $(if $(DRY_RUN),--dry-run,) + @test -n "$(REPO)$(DIRTY)" || (echo "ERROR: set REPO= or DIRTY=1"; exit 1) + +## Regenerate AGENTS.md / CLAUDE.md / .claude/rules from templates. +## Usage: make update-agent-instructions REPO=artifact-store +## make update-agent-instructions DIRTY=1 +update-agent-instructions: + $(UV) run python scripts/update_agent_instruction_files.py \ + $(if $(REPO),--repo "$(REPO)",) \ + $(if $(DIRTY),--dirty,) + @test -n "$(REPO)$(DIRTY)" || (echo "ERROR: set REPO= or DIRTY=1"; exit 1) + +## Reconcile measured token sources against State Hub. +## Usage: make token-reconcile [SINCE=2026-05-19] [APPLY=1] [ZERO_FALLBACKS=1] +token-reconcile: + $(PYTHON) scripts/token_reconcile.py \ + $(if $(SINCE),--since "$(SINCE)",) \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(CODEX_HOME),--codex-home "$(CODEX_HOME)",) \ + $(if $(CLAUDE_HOME),--claude-home "$(CLAUDE_HOME)",) \ + $(if $(APPLY),--apply,) \ + $(if $(ZERO_FALLBACKS),--zero-superseded-fallbacks,) + +## Pull then fix: single repo or all repos if REPO omitted +## make fix-consistency-remote — smart pull+fix all repos that need it +## make fix-consistency-remote REPO=slug — pull+fix one repo +fix-consistency-remote: + $(UV) run python scripts/consistency_check.py \ + $(if $(REPO),--repo "$(REPO)",--all) \ + --remote \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(NO_WRITEBACK),--no-writeback,); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Infer repo slug from git remote URL and check: make check-consistency-here [REPO_PATH=/path/to/repo] +## Omit REPO_PATH to use the Python script's CWD (i.e. pass an empty --here flag). +check-consistency-here: + $(UV) run python scripts/consistency_check.py \ + --here $(if $(REPO_PATH),"$(REPO_PATH)",) \ + $(if $(API_BASE),--api-base "$(API_BASE)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Infer repo slug from git remote URL and fix: make fix-consistency-here [REPO_PATH=/path/to/repo] +fix-consistency-here: + $(UV) run python scripts/consistency_check.py \ + --here $(if $(REPO_PATH),"$(REPO_PATH)",) \ + --fix \ + $(if $(API_BASE),--api-base "$(API_BASE)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Check all registered repos for ADR-001 consistency +check-consistency-all: + $(UV) run python scripts/consistency_check.py --all $(if $(API_BASE),--api-base "$(API_BASE)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Check and auto-fix all registered repos +fix-consistency-all: + $(UV) run python scripts/consistency_check.py --all --fix $(if $(API_BASE),--api-base "$(API_BASE)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Cancel open tasks belonging to completed/archived workstreams. +## Safe to run at any time; also suitable for a daily cron job. +## Cron example: 0 3 * * * cd ~/state-hub && make cleanup-stale +cleanup-stale: + $(UV) run python scripts/cleanup_stale_tasks.py + +## Install custodian post-commit sync hook into one repo: make install-hooks REPO=marki-docx +install-hooks: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make install-hooks REPO="; exit 1) + bash scripts/install_hooks.sh --repo "$(REPO)" + +## Install custodian post-commit sync hook into all active registered repos +install-hooks-all: + bash scripts/install_hooks.sh --all + +## Remove custodian post-commit sync hook from one repo: make remove-hooks REPO=marki-docx +remove-hooks: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make remove-hooks REPO="; exit 1) + bash scripts/install_hooks.sh --repo "$(REPO)" --remove + +## Compare Gitea coulomb org repos against state-hub registered repos +## Requires GITEA_TOKEN in env or .env: make gitea-inventory GITEA_TOKEN= +gitea-inventory: + $(UV) run python scripts/gitea_inventory.py $(if $(JSON),--json) + +## Register/update one repo from .repo-classification.yaml: +## make register-from-classification REPO=state-hub +## make register-from-classification PATH=/path/to/repo +## Optional: DRY_RUN=1 +register-from-classification: + @test -n "$(REPO)" -o -n "$(PATH)" || (echo "ERROR: REPO or PATH is required."; exit 1) + $(UV) run python scripts/register_from_classification.py \ + $(if $(PATH),--repo-path "$(PATH)",--slug "$(REPO)") \ + $(if $(DRY_RUN),--dry-run,) + +## Bulk register/update all active repos with accessible local paths +register-from-classification-all: + $(UV) run python scripts/register_from_classification.py --bulk \ + $(if $(DRY_RUN),--dry-run,) + +clean: + $(COMPOSE) down -v diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/deploy/railiance/apps/charts/state-hub/Chart.yaml b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/deploy/railiance/apps/charts/state-hub/Chart.yaml new file mode 100644 index 00000000..1cb3040f --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/deploy/railiance/apps/charts/state-hub/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: state-hub +description: State Hub API service for private Railiance operation +type: application +version: 0.1.0 +appVersion: "main-51bc70d" diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/deploy/railiance/apps/charts/state-hub/files/repo-classification.allowed.yaml b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/deploy/railiance/apps/charts/state-hub/files/repo-classification.allowed.yaml new file mode 100644 index 00000000..2eec2808 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/deploy/railiance/apps/charts/state-hub/files/repo-classification.allowed.yaml @@ -0,0 +1,127 @@ +# GENERATED — do not edit. +# Synced from the-custodian/canon/standards/repo-classification.allowed.yaml +# by scripts/sync_classification_allowed.py (CUST-WP-0067-T09). +# Canon is authoritative; this copy exists only so the container has one. +# Machine-readable allowed-values for the Repo Classification Standard. +# +# Single source of truth for the standard's controlled vocabularies, derived +# from canon/standards/repo-classification-standard_v1.0.md. Consumed by: +# - the per-repo .repo-classification.yaml linter (tools/validate_repo_classification.py) +# - the State Hub registration validator (CUST-WP-0050 T04) +# +# When the standard's vocabularies change, update this file and bump `version` +# to match the standard version. CUST-WP-0050 T01. + +standard: "Repo Classification Standard" +version: "1.1" +canon_id: "canon-repo-classification" + +# category — exactly 1 required (§5) +categories: + - experimental + - research + - project + - tooling + - product + - business + - publication + +# domain / secondary_domains — primary exactly 1; secondaries 0..n (§6) +domains: + - infotech + - financials + - communication + - consumer + - health + - industrials + - energy + - utilities + - materials + - realestate + - crypto + - agents + - space + - government + +# business_stake — 0..n; 2..6 recommended (§8) +business_stake: + - execution + - intelligence + - finance + - legal + - sales + - experience + - technology + - operations + - product + - people + - procurement + - sustainability + - automation + +# business_mechanics — 0..n, optional (§9) +business_mechanics: + - intention + - control + - coordination + - operation + - adaptation + +# capability_tags are intentionally OPEN-ENDED (§7): lowercase kebab-case, not +# restricted to this set. The families below are the standard's recommended +# canonical tags — used to warn on likely synonyms/typos, never to reject. +capability_families: + identity_and_access: + - identity + - authentication + - authorization + - access-control + - user-management + - tenancy + knowledge_and_evidence: + - knowledge + - citations + - evidence + - source-management + - traceability + - documentation + - decision-support + - history + - participation + platform_and_operations: + - platform + - deployment + - operations + - observability + - feature-control + - configuration + - orchestration + market_and_coordination: + - marketplace + - pricing + - monetization + - reputation + - challenges + - bounties + - collaboration + - coordination + product_and_lifecycle: + - product-development + - lifecycle + governance_and_control: + - governance + - policy + - compliance + - risk + - audit + - control + games_and_entertainment: + - game + - entertainment + +# Validation guidance (advisory bounds the linter applies as warnings) +guidance: + secondary_domains_max: 3 + business_stake_recommended_min: 2 + business_stake_recommended_max: 6 + capability_tag_pattern: "^[a-z0-9]+(-[a-z0-9]+)*$" diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/deploy/railiance/apps/helm/state-hub-values.yaml b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/deploy/railiance/apps/helm/state-hub-values.yaml new file mode 100644 index 00000000..02a4bd62 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/deploy/railiance/apps/helm/state-hub-values.yaml @@ -0,0 +1,69 @@ +# Production values for the State Hub Railiance chart handoff. +# Non-secret values only. DATABASE_URL comes from the Secret `state-hub-env`. + +namespace: + create: false + +image: + repository: forgejo.coulomb.social/coulomb/state-hub + tag: "main-51bc70d@sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015" + +ingress: + enabled: false + +# MCP layer on central (CUST-WP-0067-T08). Enabled together with the image tag +# above: MCP_HOST landed in main-6c1262e, and enabling this on an older image +# would deploy a pod that binds loopback and never becomes reachable. +# ClusterIP only — no ingress, same reason the API's is disabled. +mcp: + enabled: true + resources: + requests: + # 2026-09-10: 1,078 observed 5-minute samples, p99 2.35m, peak 30.45m. + # Preserve the 500m limit for bursts; free 40m for the API rollout surge. + cpu: 10m + +config: + # This deployment is the authoritative hub. Callers verify this rather than + # trusting that whatever answered on a port is central (CUST-WP-0067-T03). + instanceRole: primary + instanceLabel: railiance01 + sbomNexusUrl: "http://sbom-nexus.sbom-nexus.svc.cluster.local:8010" + # Reversible T04 read cutover; set back to `legacy` to roll back. + sbomNexusReadMode: nexus + # Reversible T04 write cutover; set back to `legacy` to roll back future writes. + sbomNexusWriteMode: nexus + +resources: + # Restored to the chart's 250m on 2026-09-21. The earlier 100m was a workaround + # for a node with <250m unallocated; ~830m of Knative/Kourier requests were + # released that day (the-custodian docs/kubernetes-change-gate-decision.md). + # State Hub idles near 5m but bursts past 400m under concurrent agent load. + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 2Gi + +sweep: + # RMGR-WP-0005-T11: disabled while railiance01 checkouts still target the + # stale gitea-remote lineage. Re-enable only after the governed remote + # reconciliation and registrar preflight are complete. + enabled: false + # Present in the live release but previously missing here, so deploying from + # this file alone silently dropped it. Inert while enabled is false. + hostname: 239.62.205.92.host.secureserver.net + hostPath: /home/tegwick + sshHostPath: /home/tegwick/.ssh + +# Forge read credential (STATE-WP-0084-T02). Coordinates only — the token lives +# in OpenBao at platform/workloads/state-hub/forge-derivation and reaches the +# pod through Kubernetes auth, never through this file or a Secret. +forgeRead: + enabled: true + +# RPF-WP-0035-T04: dedicated ESO signing key, API-only explicit reference. +renamePreflight: + enabled: true + secretName: state-hub-rename-preflight diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/docs/evidence/2026-10-17-cluster-historical-receiving-rollout.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/docs/evidence/2026-10-17-cluster-historical-receiving-rollout.json new file mode 100644 index 00000000..8e952e6b --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/docs/evidence/2026-10-17-cluster-historical-receiving-rollout.json @@ -0,0 +1,102 @@ +{ + "schema": "statehub.cluster-historical-receiving-delivered.v1", + "status": "verified", + "proposal_id": "STATE-WP-0095-T06-cluster-frozen-receiving-r1", + "approved_revision": 4, + "approved_digest_sha256": "sha256:63a02e8296d126c0158b7eb2c4ce79c7eeb6f8d6f61298d473c1065c7557ba32", + "approval_decision_id": "e3798419-8aa9-5425-8c08-20df6bd82a19", + "source_revision": "51bc70d1ff8f25450a8650d8e50caf053093a02e", + "readiness_source_revision": "086d79a2736e34b916bd7de3c8a65e2ea38a3d64", + "helm_revision": 76, + "image_reference": "forgejo.coulomb.social/coulomb/state-hub:main-51bc70d@sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015", + "expected_schema": "c4d5e6f7a8b9", + "health": { + "status": "ok", + "instance_role": "primary", + "instance_label": "railiance01", + "schema": { + "status": "ok", + "applied": "c4d5e6f7a8b9", + "expected": "c4d5e6f7a8b9" + } + }, + "pods_ready": true, + "historical_module_sha256": "sha256:e3c4bf965764b32a6df5d61c71272ee22569df9a9ddae2a3fc3df97ec554e0b2", + "historical_preservation_verified": true, + "parents": 3, + "tasks": 15, + "full_historical_rows_equal": true, + "native_rollback_probes": { + "foreign-holder": { + "full_historical_rows_restored": true, + "historical_binding_refused": true, + "native_outcome_status": "refused", + "rolled_back": true, + "status": "passed" + }, + "missing-task": { + "full_historical_rows_restored": true, + "historical_binding_refused": true, + "native_outcome_status": "refused", + "rolled_back": true, + "status": "passed" + }, + "nonterminal-parent": { + "full_historical_rows_restored": true, + "historical_binding_refused": true, + "native_outcome_status": "refused", + "rolled_back": true, + "status": "passed" + }, + "nonterminal-task": { + "full_historical_rows_restored": true, + "historical_binding_refused": true, + "native_outcome_status": "refused", + "rolled_back": true, + "status": "passed" + }, + "preserve": { + "full_historical_rows_equal": true, + "full_historical_rows_restored": true, + "native_outcome_status": "applied", + "rolled_back": true, + "status": "passed" + } + }, + "native_probe_commits": 0, + "fresh_transaction_full_rows_equal": true, + "source_receiving_gate_delivered": true, + "normal_cluster_publication_verified": false, + "registrar_admission_or_cluster_publication_not_inferred": true, + "authority_limits": [ + "Rollback probe SQL effects rolled back; foreign-holder case is historical-binding refusal, not separate global holder branch.", + "Actual historical native receiving is verified; owner-native TLS/private admission remains unresolved.", + "Migration schema remains additive; Helm rollback does not downgrade schema or stop an active hook Job." + ], + "evidence": [ + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/approval-decision.json", + "sha256": "sha256:b51554c004daa882f3709f67bdae7f2e939d611120383e61be04ec6969878134" + }, + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/helm-return.json", + "sha256": "sha256:61727deacf22d37da2d28cbaf04f5a3ddee0f9ff7571c77f1e85dba97963b572" + }, + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/delivered-runtime.json", + "sha256": "sha256:89681066e8fe139456163ebfafa3efe77ffae8534ea167219230b0a74367f129" + }, + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/native-refusal-probes.json", + "sha256": "sha256:673ccef965c820704dc310a1bf7832b583c30c3edfeb814528cf892555199172" + }, + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/before-full-row-hashes.json", + "sha256": "sha256:75cdd87eef79059ff3f484ed24ca69e285899cdf8d851cc4e472653c417eb0ee" + }, + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/after-full-row-hashes.json", + "sha256": "sha256:2156aad18f190384838c27e792706a156a7423391408afbc6107e6c28c5872d8" + } + ] +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/scripts/sync_classification_allowed.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/scripts/sync_classification_allowed.py new file mode 100644 index 00000000..322770f5 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/scripts/sync_classification_allowed.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Sync the canon classification allowed-values into the chart (CUST-WP-0067-T09). + +The State Hub validates repository classification against +``the-custodian/canon/standards/repo-classification.allowed.yaml``. That file is +canon and must stay authoritative, but a container has no the-custodian +checkout, so a copy has to travel with the release. + +A copy that nobody checks is a copy that silently drifts. This script owns the +copy: ``--check`` fails when it diverges from canon, so the release can refuse +to ship a stale vocabulary rather than validate against yesterday's rules. + +Usage: + python scripts/sync_classification_allowed.py # write the copy + python scripts/sync_classification_allowed.py --check # verify, exit 1 on drift +""" +from __future__ import annotations + +import argparse +import difflib +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +CHART_COPY = ( + REPO_ROOT + / "deploy/railiance/apps/charts/state-hub/files/repo-classification.allowed.yaml" +) +CANON_CANDIDATES = ( + Path("/home/worsch/the-custodian/canon/standards/repo-classification.allowed.yaml"), + Path("/home/tegwick/the-custodian/canon/standards/repo-classification.allowed.yaml"), + REPO_ROOT.parent / "the-custodian/canon/standards/repo-classification.allowed.yaml", +) + +HEADER = ( + "# GENERATED — do not edit.\n" + "# Synced from the-custodian/canon/standards/repo-classification.allowed.yaml\n" + "# by scripts/sync_classification_allowed.py (CUST-WP-0067-T09).\n" + "# Canon is authoritative; this copy exists only so the container has one.\n" +) + + +def find_canon() -> Path: + for candidate in CANON_CANDIDATES: + if candidate.is_file(): + return candidate + raise SystemExit( + "ERROR: canon allowed-values not found. Looked in:\n " + + "\n ".join(str(c) for c in CANON_CANDIDATES) + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="Verify the chart copy matches canon; exit 1 on drift", + ) + args = parser.parse_args() + + canon = find_canon() + desired = HEADER + canon.read_text(encoding="utf-8") + + if args.check: + if not CHART_COPY.is_file(): + print(f"DRIFT: {CHART_COPY} is missing; run without --check to create it") + return 1 + current = CHART_COPY.read_text(encoding="utf-8") + if current != desired: + print(f"DRIFT: chart copy differs from canon ({canon})") + sys.stdout.writelines( + difflib.unified_diff( + current.splitlines(keepends=True), + desired.splitlines(keepends=True), + fromfile="chart copy", + tofile="canon", + ) + ) + return 1 + print(f"OK: chart copy matches canon ({canon})") + return 0 + + CHART_COPY.parent.mkdir(parents=True, exist_ok=True) + CHART_COPY.write_text(desired, encoding="utf-8") + print(f"Synced {canon} -> {CHART_COPY}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/workplans/STATE-WP-0095-historical-source-receiving.md b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/workplans/STATE-WP-0095-historical-source-receiving.md new file mode 100644 index 00000000..ad24e407 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r2/state-hub/workplans/STATE-WP-0095-historical-source-receiving.md @@ -0,0 +1,282 @@ +--- +id: STATE-WP-0095 +type: workplan +title: "Preserve exact closed historical source bindings in Forge receiving" +domain: infotech +repo: state-hub +status: active +owner: codex +created: "2026-10-04" +updated: "2026-10-17" +flavor: implementation +depends_on: [] +state_hub_workstream_id: "79579572-8120-5d3b-aca5-dd4aff60d058" +--- + +## Goal + +Normal Forge receiving must preserve the seven bound Activity legacy source +views and the Custodian retired ADHOC source without new identities, source +rewrites, task movement, deletion, relabeling or unretirement. This implements +receiving support before RMGR-WP-0019 registrar admission. CUST-WP-0077 owns +cross-owner coordination; deployment and replay remain explicit owner returns. + +## Verify source bindings and shared views + +```task +id: STATE-WP-0095-T01 +status: done +priority: high +state_hub_task_id: "6a5f87bb-e63d-5e20-b9fc-c1b5cd403ff7" +``` + +Reviewed seven exact Activity files: four existing workplans and 64 UUID-bound +closed tasks. The archived custodian-WP-0003 parent and three split_from views +share one existing parent UUID; every repeated task binding/content agrees. +Custodian ADHOC-2026-08-25-retired retains parent +1c53d3db-6633-5e2d-987f-10706d73be9e and task +d8af3e6b-26ae-580c-bbeb-caa8ad419bec. No identity migration is authorized. + +## Implement preservation receiving and global collision refusal + +```task +id: STATE-WP-0095-T02 +status: done +priority: high +state_hub_task_id: "ebb24194-e4cc-5130-aaa6-663f2ea2c822" +``` + +Prepared exact source digest manifest, existing-parent/task verification, +immutable receiving and UUID-based historical diff. Ordinary canonical sources +continue deriving their identities. Missing/changed/symlinked source, absent +bound rows or nonterminal hub histories refuse. Check task UUID holders globally, +including retired and acknowledged-stale parent workplans, before INSERT. + +Offline preview tests and snapshot proof are evidence of source behavior only; +finish after applying reviewed source and its focused tests to the owning repo. + +## Deliver receiving through normal deployment and verify owner return + +```task +id: STATE-WP-0095-T03 +status: done +priority: high +depends_on: [STATE-WP-0095-T02] +state_hub_task_id: "e8566341-cf6e-5b9c-ad80-8564d14cd52a" +decision_id: "36772026-f43d-4331-b466-5f4b1afe4eca" +``` + +No direct live code patch. This task is a deployment return, not policy authority +or permission to resolve identity decisions. Exact receiving fixture acceptance +and global collision rejection must hold in the delivered version. + +## Verify normal historical replay preserves existing identities + +```task +id: STATE-WP-0095-T04 +status: done +priority: high +depends_on: [STATE-WP-0095-T03, RMGR-WP-0019-T03] +state_hub_task_id: "3bbb311f-194e-5862-8aed-358a580f22b4" +``` + +Verify Activity four historical parent UUIDs/64 task UUIDs and Custodian retired +parent/task unchanged; no historical row creation, movement, deletion, +relabeling or unretirement. Unrelated stale projections remain separate decisions. + +2026-10-05: Source exact-set retirement support now exists. Repository projection +reconcile accepts `expected_retirement_uuids`; that set must equal the stale +projections, and any task or dependency edge on a stale row refuses the call. +Live apply still needs a reviewed rollout. On 2026-10-05 the founder accepted +review `review-1ece4f3e-d0e3-4fd6-abc8-272f1b1876f4`, which names only the three +empty projections and preserves parent `b4eb45a9-69e3-4ab0-b00c-67a53c3117c5`. +That acceptance is not a live retirement and does not resolve +`91b7c633-a66f-5d43-9f4b-6565aefb9356`. + +## Offline evidence, 2026-10-04 + +Prepared receiver suite: Forge projection regression plus historical preservation, +missing/changed source, terminality, immutable UUID diff, and global holder query. +Real-source offline snapshot proof preserves four Activity parents/64 tasks and +one Custodian parent/task without mutations; no API writes or deployment occurred. +Exact original bindings are maintained in api/services/historical_projection.py. + +2026-10-04 actual owning-source validation: 155 focused receiver/qualifier/planner +cases passed; the independent offline current-source/current-hub-snapshot proof +retained five existing parents and 65 task UUIDs, no mutations, clean historical +diff. T02 is source-complete; T03/T04 remain wait for reviewed deployment and +normal registrar/replay receiving, respectively. Evidence: docs/evidence/2026-10-04-historical-receiving-source.json. + + +## Verified receiving rollout and remaining replay gate, 2026-10-04 + +T03 is done on actual founder approval of proposal +STATE-WP-0095-T03-historical-receiving revision 1, SHA256 +`d6789359b3fe7988a6201fff6f367cb46948aa197e1cc0ca66c31f45e1ea9981`, +approval receipt reference `user_reply:call_o87uEfFs3AtXfxO5YqUnSpcg:0`. +The normal owner Helm path delivered `main-5dd3df9`, commit +`5dd3df9c69eb564af10ea5ea01e349786212501a`, revision 72; API and MCP are Ready, +primary/railiance01 health and schema a0b1c2d3e4f5 verified, running source +hashes match. Rollback remains revision 71; no receipt-table downgrade is needed. +Actual historical preservation and global task-holder refusal were verified in +the delivered version. Durable receipt: +[verified historical receiving rollout](../docs/evidence/2026-10-04-historical-receiving-rollout.json). + +Custodian immutable replay applied while preserving the retired historical +parent/task UUIDs. Activity immutable replay retained all four historical parent +and 64 task UUIDs and historical row contents, but refused transactionally on +three unrelated stale projections: custodian-wp-0003 (archived), +custodian-wp-0003a (done), custodian-wp-0003b (done). No retirement acknowledgement +was issued and that refusal is not represented as normal replay success. + +Registrar source `c41d36c2` has 38 actual passing tests and Repo Manager's own +normal sync succeeded; registrar admission does not authorize retiring Activity +projections. T04 therefore remains wait, retaining RMGR-WP-0019-T03 and awaiting +Activity Core's precise reviewed disposition of those three stale rows. No +rename, identity migration, new paid native trial or further live action is +included in this source status update. + +## Activity Core return, 2026-10-06 + +Activity Core returned the exact disposition (message +`cf2d5847-8b3f-4653-9d9f-d5fcfbe3f1e4`, ACTIVITY-WP-0045, activity-core commit +`941fac0`). `expected_retirement_uuids`: +`683724d9-4249-5e0e-88e0-7baff4330d58` (custodian-wp-0003), +`644017c7-c5f2-5a64-81c3-d8ee806f8251` (custodian-wp-0003a), +`a5806df9-5b9a-50da-81c1-7e606076bed9` (custodian-wp-0003b). Each has zero +tasks and zero dependency edges. None is preserved parent +`b4eb45a9-69e3-4ab0-b00c-67a53c3117c5`. Repo Manager independently verified the +set (message `916841f6`, repo-manager commit `498682c`). Nothing was retired. +T04 now waits on decision `91b7c633` and a reviewed live rollout. After that, +Repo Manager re-runs the Activity owner sync and closes RMGR-WP-0019-T03. + +## Retirement rollout and T04 verification, 2026-10-06 + +The founder resolved decision `91b7c633` (retire exactly the three projections) +and approved the rollout. The founder ran the normal Helm upgrade to +`main-968a811` (revision 74; rollback revision 73, `main-fad9b3f`). API and MCP +are Ready, health is ok on primary/railiance01, and schema `b3c4d5e6f7a8` is +unchanged; no migrations. The activity-core reconcile at commit +`67a92d2ff8e033baf2c36998d24187fe83edec84` with `expected_retirement_uuids` set +to the three UUIDs applied: 0 created, 3 retired (tombstoned with an +`@retired-20261006` slug; UUIDs and statuses kept), 0 refused, 0 tasks +cancelled. All 7 bound Activity source files still match their digests. The four +historical parents and 64 task UUIDs are present under their bound parents with +unchanged fields. The Custodian retired parent and task are unchanged. Activity +had 54 workplans and 288 tasks before and after. Evidence: +[activity stale retirement rollout](../docs/evidence/2026-10-06-activity-stale-retirement-rollout.json). +Repo Manager's re-run of the Activity owner sync (RMGR-WP-0019-T03) and +ACTIVITY-WP-0045-T02 remain with their owners. + + +## Cluster exact historical receiving successor, 2026-10-17 + +This scoped extension retains the completed Activity/Custodian outcomes above. +Three frozen cluster archives contain three existing parent UUIDs and 15 existing +closed task UUIDs. Extend the existing exact historical receiving path before +registrar admission; never derive replacement identities or modify those +sources. Cross-owner observation attaches to existing SHR-WP-0001, with +RMGR-WP-0019 as obligor/beneficiary and NK-WP-0049-T35 benefiting indirectly. +The local derived case is waiting; no live worker injection is authorized. + +## Prepare exact cluster source preservation and refusal cases + +```task +id: STATE-WP-0095-T05 +status: done +priority: high +state_hub_task_id: "339b9a37-181c-5599-9a89-0393e199bd4b" +``` + +Prepared the same full-file SHA256 and 15 task binding manifest as Repo Manager. +Candidate receiving tests pass 26: existing rows remain entirely unchanged; +missing/foreign/nonterminal rows and unknown, changed, missing or symlinked +source refuse. Read-only public metadata confirms all 18 existing rows have the +expected ownership and terminal status. These are source and metadata checks, +not delivered receiving or a committed native trial. + +## Deliver reviewed cluster receiving successor before registrar admission + +```task +id: STATE-WP-0095-T06 +status: done +priority: high +depends_on: [STATE-WP-0095-T05] +needs_human: false +intervention_note: "Exact revision 4 approved and verified; no pending rollout intervention." +state_hub_task_id: "6af973b4-1964-5d4d-a2f8-8dbd4c01fda4" +``` + +Completed 2026-10-17 on the founder's approval of exact revision 4 of stable +proposal STATE-WP-0095-T06-cluster-frozen-receiving-r1 (decision + e3798419-8aa9-5425-8c08-20df6bd82a19; approved digest + sha256:63a02e8296d126c0158b7eb2c4ce79c7eeb6f8d6f61298d473c1065c7557ba32). +The ordinary owner Helm path delivered revision 76, runtime source 51bc70d and +exact image manifest sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015. +API/MCP Ready and primary/railiance01 health report expected schema +c4d5e6f7a8b9; delivered historical module sha256:e3c4bf965764b32a6df5d61c71272ee22569df9a9ddae2a3fc3df97ec554e0b2 matches. +All three parent and 15 task full DB row hashes/columns/UUIDs remained unchanged. +Five actual native probes passed: preserve, actual foreign-parent holder move, +nonterminal task, nonterminal parent and missing task. Every probe rolled back; +a fresh transaction confirmed complete row restoration. The foreign-holder +case precisely verified historical binding refusal, not the separate global +UUID-holder creation branch. No committed probe writes were made. + +Evidence: [verified receiving rollout](../docs/evidence/2026-10-17-cluster-historical-receiving-rollout.json). +This actual receiving return satisfies the prerequisite of RMGR-WP-0019-T05. +Normal cluster publication and its immutable-source/row return remain T07; +source readiness and this deployment do not count that later acceptance twice. +The additive inbox migration is retained; Helm rollback does not downgrade +schema or stop an active migration Job. Original failures and exact-UID recovery +limits remain in the referenced approved packet. No TLS recovery/native private +admission or new autonomy/cost grant is inferred. + +## Verify ordinary cluster receiving and immutable historical rows + +```task +id: STATE-WP-0095-T07 +status: wait +priority: high +depends_on: [STATE-WP-0095-T06, RMGR-WP-0019-T06] +state_hub_task_id: "cfd1ea99-e75b-542e-a434-acefb62f3c7d" +``` + +Capture normal cluster owner publication/receiving receipts and compare entire +existing row snapshots and exact source bytes. No creation, movement, relabel, +unretirement or retirement is authorized for these histories. A rollback first +fences cluster sync and restores registrar refusal before removing receiving +support; it never rewrites archived IDs or UUIDs. This work does not establish +original TLS recovery or owner-native admission. + +## Fix policy-compatible Secret-template server dry-run preparation + +```task +id: STATE-WP-0095-T08 +status: done +priority: high +state_hub_task_id: "2dd9b04f-2250-5a8a-b5bc-c8a009872858" +``` + +Prepared the narrow Makefile successor after the existing readiness command +refused its first Secret-template phase. Ordinary client-side apply adds the +last-applied annotation, correctly denied by the existing Platform +reject-secret-last-applied policy. Only the two Secret-template server +dry-run requests now use server-side apply with dry-run; no force-conflicts, +Secret value change, policy relaxation or actual Secret write is introduced. +Command-fixture tests verify both requests and preserve other dry-run phases. +Evidence and original refusal remain in the Custodian cluster historical +publication packet, revisions/r2 and r3-readiness. This is completed source +preparation; T06 separately records the verified actual rollout. The runtime image +continues to be main-51bc70d; Makefile/workplan changes do not create a new +application image or native acceptance. + + +T08 source repair refinement, 2026-10-17: the required normal Make render +correctly refused a pre-existing generated chart classification copy at v1.0 +after authoritative canon reached v1.1. Regenerated only through the existing +scripts/sync_classification_allowed.py, preserving canon and strict check. +The exact source diff is version 1.0→1.1 plus publication/history/participation. +Normal Make rendering and the focused classification check passed in an +isolated owner-layout copy. The original refusal is retained in the Custodian +owner-followthrough-r1 validation receipt. This is source repair only: no live +ConfigMap apply, new runtime image, policy relaxation or T07 completion. diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/Makefile b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/Makefile new file mode 100644 index 00000000..cec684ec --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/Makefile @@ -0,0 +1,517 @@ +.PHONY: start install install-cli dashboard-install dashboard-check db db-tools migrate seed api dashboard dashboard-local sync check-primary primary-port-clear check check-local test test-python clean register-project register-codex-project register-mcp configure-codex bootstrap-env dev-hub edge-relay mcp-profile validate-adr add-domain rename-domain add-repo list-repos register-path register-from-classification register-from-classification-all cleanup-stale tunnels-up tunnels-status tunnels-check bridges install-hooks install-hooks-all gitea-inventory token-reconcile railiance-state-hub-render railiance-state-hub-client-dry-run railiance-state-hub-server-dry-run railiance-state-hub-headroom + +COMPOSE = docker compose -f infra/docker-compose.yml --env-file .env +PYTHON ?= python3 +HELM ?= $(shell command -v helm 2>/dev/null || if [ -x "$$HOME/.local/bin/helm" ]; then printf "%s" "$$HOME/.local/bin/helm"; else printf "%s" "helm"; fi) +KUBECTL ?= $(shell command -v kubectl 2>/dev/null || if [ -x "$$HOME/.local/bin/kubectl" ]; then printf "%s" "$$HOME/.local/bin/kubectl"; else printf "%s" "kubectl"; fi) + +RAILIANCE_STATE_HUB_RELEASE ?= state-hub +RAILIANCE_STATE_HUB_NAMESPACE ?= state-hub +RAILIANCE_STATE_HUB_CHART ?= deploy/railiance/apps/charts/state-hub +RAILIANCE_STATE_HUB_VALUES ?= deploy/railiance/apps/helm/state-hub-values.yaml +RAILIANCE_STATE_HUB_IMAGE_TAG ?= main-51bc70d@sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015 +RAILIANCE_STATE_HUB_PLATFORM_DIR ?= deploy/railiance/platform +RAILIANCE_STATE_HUB_APP_MANIFESTS ?= deploy/railiance/apps/manifests +# Codex/WSL non-login shells may not source ~/.profile; keep uv discoverable. +UV ?= $(shell command -v uv 2>/dev/null || if [ -x "$$HOME/.local/bin/uv" ]; then printf "%s" "$$HOME/.local/bin/uv"; else printf "%s" "uv"; fi) +RMGR ?= $(shell command -v rmgr 2>/dev/null || if [ -x "$$HOME/repo-manager/.venv/bin/rmgr" ]; then printf "%s" "$$HOME/repo-manager/.venv/bin/rmgr"; else printf "%s" "rmgr"; fi) +STATE_HUB_API_BASE ?= http://127.0.0.1:8000 +SYNC_PATH ?= . +SYNC_PUSH ?= 1 + +start: + @echo "# Normal production UI access (run in order)" + @echo "make bridges # connect local :8000 to the railiance01 primary" + @echo "make dashboard # verify the primary, then serve the UI on :3000" + @echo "make sync # push file-backed records and reconcile the exact forge commit" + @echo "" + @echo "# Optional local MCP adapter" + @echo "make mcp-http # local SSE adapter on :8001; not needed by the UI" + @echo "" + @echo "# Local fallback/development only" + @echo "make api # local Postgres + API; conflicts with the production :8000 tunnel" + @echo "make dashboard-local # dashboard against the deliberate local API" + +install: + $(UV) sync + +dashboard/node_modules/.bin/observable: dashboard/package.json dashboard/package-lock.json + cd dashboard && npm ci + +dashboard-install: dashboard/node_modules/.bin/observable + +dashboard-check: dashboard-install + cd dashboard && npm run build + +## Symlink the custodian CLI into ~/.local/bin so it's on PATH system-wide +install-cli: install + mkdir -p ~/.local/bin + ln -sf "$(shell pwd)/.venv/bin/custodian" ~/.local/bin/custodian + ln -sf "$(shell pwd)/.venv/bin/statehub" ~/.local/bin/statehub + @echo "Installed: custodian → $$(readlink -f ~/.local/bin/custodian)" + @echo "Installed: statehub → $$(readlink -f ~/.local/bin/statehub)" + @echo "Make sure ~/.local/bin is on your PATH:" + @echo " echo 'export PATH=\"\$$HOME/.local/bin:\$$PATH\"' >> ~/.bashrc && source ~/.bashrc" + +db: + $(COMPOSE) up -d postgres + +db-tools: + $(COMPOSE) --profile tools up -d + +migrate: + $(UV) run alembic upgrade head + +seed: + $(UV) run python scripts/seed.py + +## Start (or restart) the MCP SSE server on :8001 — primary transport for Claude Code. +## Remote clients (e.g. COULOMBCORE) connect via the ops-bridge tunnel (port 18001). +## Registration: claude mcp add-json -s user state-hub '{"type":"sse","url":"http://127.0.0.1:8001/sse"}' +mcp-http: + @fuser -k 8001/tcp 2>/dev/null && echo "Stopped running MCP server" || true + MCP_TRANSPORT=sse MCP_PORT=8001 $(UV) run python mcp_server/server.py + +## Require the production identity before serving the normal dashboard. This +## prevents an accidental local `make api` from presenting an empty fallback DB +## as the live State Hub. +check-primary: + @health="$$(curl -fsS --max-time 5 http://127.0.0.1:8000/state/health 2>/dev/null)" || { \ + echo "ERROR: State Hub primary is not reachable on 127.0.0.1:8000." >&2; \ + echo "Run 'make bridges' first." >&2; \ + exit 1; \ + }; \ + identity="$$(printf '%s' "$$health" | $(PYTHON) -c 'import json, sys; d=json.load(sys.stdin); print("{}/{}".format(d.get("instance_role", ""), d.get("instance_label", "")))')"; \ + if [ "$$identity" != "primary/railiance01" ]; then \ + echo "ERROR: 127.0.0.1:8000 is '$$identity', not the primary/railiance01 State Hub." >&2; \ + echo "Stop the local API, then run 'make bridges'. Use 'make dashboard-local' only for intentional local development." >&2; \ + exit 1; \ + fi + +## Refuse to start the production tunnel when a non-primary API already owns +## port 8000. With no listener, bridge is free to establish the tunnel. +primary-port-clear: + @if health="$$(curl -fsS --max-time 2 http://127.0.0.1:8000/state/health 2>/dev/null)"; then \ + identity="$$(printf '%s' "$$health" | $(PYTHON) -c 'import json, sys; d=json.load(sys.stdin); print("{}/{}".format(d.get("instance_role", ""), d.get("instance_label", "")))')"; \ + if [ "$$identity" != "primary/railiance01" ]; then \ + echo "ERROR: port 8000 is occupied by '$$identity'." >&2; \ + echo "Stop the local 'make api' process before starting the production tunnel." >&2; \ + exit 1; \ + fi; \ + fi + +dashboard: check-primary + @fuser -k 3000/tcp 2>/dev/null && echo "Stopped running dashboard" || true + $(MAKE) dashboard-install + cd dashboard && npm run dev + +## Deliberate local-development dashboard; bypasses the production identity gate. +dashboard-local: + @echo "WARNING: serving the dashboard against the local/fallback API on :8000." + @fuser -k 3000/tcp 2>/dev/null && echo "Stopped running dashboard" || true + $(MAKE) dashboard-install + cd dashboard && npm run dev + +## Fast work-record path. Repo Manager assigns only missing deterministic IDs, +## verifies the primary/railliance01 bridge, and asks central to derive the +## exact pushed Forgejo commit in one transactional request. +sync: + $(RMGR) sync --path "$(SYNC_PATH)" --api-base "$(STATE_HUB_API_BASE)" $(if $(filter 1 true yes,$(SYNC_PUSH)),--push,) + +check: check-primary + @echo "State Hub primary/railiance01 is healthy." + +check-local: + curl -sf http://127.0.0.1:8000/state/health | python3 -m json.tool + +# CUST-WP-0067-T09. The chart ships a copy of the-custodian canon allowed-values +# because a container has no checkout to read it from. Regenerate after any +# canon vocabulary change; the -check target fails on drift so a release cannot +# validate classification against a stale vocabulary. +sync-classification-allowed: + python3 scripts/sync_classification_allowed.py + +check-classification-allowed: + python3 scripts/sync_classification_allowed.py --check + +# STATE-WP-0091: read-only surge-headroom check. Does not start Helm. +# Live: make railiance-state-hub-headroom +# Fixtures: make railiance-state-hub-headroom HEADROOM_FIXTURE=tests/fixtures/release_headroom/rev63-105m.json +HEADROOM_FIXTURE ?= +railiance-state-hub-headroom: + @if [ -n "$(HEADROOM_FIXTURE)" ]; then \ + $(PYTHON) scripts/release_headroom_preflight.py --fixture "$(HEADROOM_FIXTURE)"; \ + else \ + $(PYTHON) scripts/release_headroom_preflight.py --kubectl "$(KUBECTL)"; \ + fi + +railiance-state-hub-render: check-classification-allowed + $(HELM) template $(RAILIANCE_STATE_HUB_RELEASE) $(RAILIANCE_STATE_HUB_CHART) \ + --namespace $(RAILIANCE_STATE_HUB_NAMESPACE) \ + -f $(RAILIANCE_STATE_HUB_VALUES) \ + --set image.tag=$(RAILIANCE_STATE_HUB_IMAGE_TAG) + +railiance-state-hub-client-dry-run: + @set -e; \ + tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + $(HELM) template $(RAILIANCE_STATE_HUB_RELEASE) $(RAILIANCE_STATE_HUB_CHART) \ + --namespace $(RAILIANCE_STATE_HUB_NAMESPACE) \ + -f $(RAILIANCE_STATE_HUB_VALUES) \ + --set image.tag=$(RAILIANCE_STATE_HUB_IMAGE_TAG) > "$$tmpdir/state-hub.yaml"; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-credentials.sops.yaml.template; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-cluster.yaml; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-networkpolicies.yaml; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-namespace.yaml; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-env.secret.sops.yaml.template; \ + $(KUBECTL) apply --dry-run=client -n $(RAILIANCE_STATE_HUB_NAMESPACE) -f "$$tmpdir/state-hub.yaml" + +railiance-state-hub-server-dry-run: + @set -e; \ + tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + $(HELM) template $(RAILIANCE_STATE_HUB_RELEASE) $(RAILIANCE_STATE_HUB_CHART) \ + --namespace $(RAILIANCE_STATE_HUB_NAMESPACE) \ + -f $(RAILIANCE_STATE_HUB_VALUES) \ + --set image.tag=$(RAILIANCE_STATE_HUB_IMAGE_TAG) > "$$tmpdir/state-hub.yaml"; \ + $(KUBECTL) apply --server-side --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-credentials.sops.yaml.template; \ + $(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-cluster.yaml; \ + $(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-networkpolicies.yaml; \ + $(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-namespace.yaml; \ + if $(KUBECTL) get namespace $(RAILIANCE_STATE_HUB_NAMESPACE) >/dev/null 2>&1; then \ + $(KUBECTL) apply --server-side --dry-run=server -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-env.secret.sops.yaml.template; \ + $(KUBECTL) apply --dry-run=server -n $(RAILIANCE_STATE_HUB_NAMESPACE) -f "$$tmpdir/state-hub.yaml"; \ + else \ + echo "Namespace $(RAILIANCE_STATE_HUB_NAMESPACE) does not exist; validating namespaced app manifests with client dry-run."; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-namespace.yaml; \ + $(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-env.secret.sops.yaml.template; \ + $(KUBECTL) apply --dry-run=client -n $(RAILIANCE_STATE_HUB_NAMESPACE) -f "$$tmpdir/state-hub.yaml"; \ + fi + +test: test-python dashboard-check + +test-python: + TEST_DATABASE_URL=postgresql+asyncpg://custodian:changeme@127.0.0.1:5432/custodian_test \ + $(UV) run pytest -x -q + +## Benchmark /state/summary revision cache (API must be running on :8000) +benchmark-summary-cache: + $(UV) run python scripts/benchmark_summary_cache.py + +## ops-bridge managed tunnels +## Requires ops-bridge: bridge is at /home/worsch/.local/bin/bridge +tunnels-up: primary-port-clear + bridge up + +tunnels-status: + bridge status + +## End-to-end check: verifies SSH process alive + remote port listening on COULOMBCORE. +## Exits non-zero if any tunnel is not fully operational. +tunnels-check: + bridge check + +## Ensure all ops-bridge tunnels are up and healthy. +## Brings up any stopped/stale tunnels, shows final status, exits non-zero if anything is still down. +bridges: primary-port-clear + @echo "==> Bringing up all tunnels..." + bridge up + @echo "" + @echo "==> Tunnel status:" + bridge status + @echo "" + @echo "==> Checking tunnel health..." + bridge check + +## Start (or restart) the LOCAL FALLBACK backend — db + migrate + uvicorn. +## This replaces anything on :8000, including the production State Hub tunnel. +api: db + @echo "WARNING: starting the local fallback API; this is not the railiance01 primary." + @echo "Waiting for postgres..."; \ + for i in 1 2 3 4 5 6 7 8 9 10; do \ + nc -z 127.0.0.1 5432 2>/dev/null && break; \ + sleep 1; \ + done + $(MAKE) migrate + @fuser -k 8000/tcp 2>/dev/null && echo "Stopped running API" || true + $(UV) run uvicorn api.main:app --reload --reload-dir api --reload-dir mcp_server --reload-dir task_flow_engine --host 127.0.0.1 --port 8000 + +## Register a project (Claude Code): make register-project DOMAIN=railiance PROJECT_PATH=/home/worsch/railiance +register-project: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required. Usage: make register-project DOMAIN= PROJECT_PATH="; exit 1) + @test -n "$(PROJECT_PATH)" || (echo "ERROR: PROJECT_PATH is required."; exit 1) + scripts/register_project.sh "$(DOMAIN)" "$(PROJECT_PATH)" + +## Register a Codex project (AGENTS.md + HTTP API): make register-codex-project DOMAIN=capabilities PROJECT_PATH=/home/worsch/my-repo +register-codex-project: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required. Usage: make register-codex-project DOMAIN= PROJECT_PATH="; exit 1) + @test -n "$(PROJECT_PATH)" || (echo "ERROR: PROJECT_PATH is required."; exit 1) + scripts/register_project.sh "$(DOMAIN)" "$(PROJECT_PATH)" --codex + +## Register State Hub MCP for Claude Code. Optional: make register-mcp MCP_URL=http://127.0.0.1:18001/sse +register-mcp: + scripts/register-mcp.sh \ + $(if $(MCP_URL),--url "$(MCP_URL)",) \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(DRY_RUN),--dry-run,) + +## Configure Codex workspace-write networking for local State Hub access. +configure-codex: + scripts/configure-codex.sh \ + $(if $(CODEX_HOME),--codex-home "$(CODEX_HOME)",) \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(WITH_MCP),--with-mcp,) \ + $(if $(DRY_RUN),--dry-run,) + +## Bootstrap a new operator/collaborator environment. Optional: make bootstrap-env ARGS="--install-missing" +bootstrap-env: + scripts/bootstrap-env.sh $(ARGS) + +## Files-first local dev hub (CUST-WP-0054-T07): postgres + migrate + seed + API + repo registration. +## Optional: WITH_EDGE=1 WITH_MCP=1 make dev-hub +## Fleet tunnel when reachable: make dev-hub PROFILE=fleet +dev-hub: + bash scripts/dev_hub_up.sh $(if $(PROFILE),$(PROFILE),dev) + +## Local edge relay for offline write buffering (STATE-WP-0068). +edge-relay: + @fuser -k 18080/tcp 2>/dev/null && echo "Stopped running edge relay" || true + STATEHUB_UPSTREAM_URL=$${STATEHUB_UPSTREAM_URL:-http://127.0.0.1:8000} \ + $(UV) run uvicorn api.edge.relay:app --host 127.0.0.1 --port 18080 + +## Switch MCP/API profile: make mcp-profile PROFILE=dev|fleet|status +mcp-profile: + @test -n "$(PROFILE)" || (echo "ERROR: PROFILE is required (dev|fleet|status)."; exit 1) + bash scripts/mcp_hub_profile.sh "$(PROFILE)" + +## Add a second repo to an existing domain: make add-repo DOMAIN=railiance REPO_PATH=/home/worsch/railiance-infra +add-repo: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required."; exit 1) + @test -n "$(REPO_PATH)" || (echo "ERROR: REPO_PATH is required."; exit 1) + scripts/register_project.sh "$(DOMAIN)" "$(REPO_PATH)" --additional + +## Create a new domain: make add-domain DOMAIN=my_domain NAME="My Domain" +add-domain: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required (slug)."; exit 1) + @test -n "$(NAME)" || (echo "ERROR: NAME is required (display name)."; exit 1) + curl -sf -X POST http://127.0.0.1:8000/domains/ \ + -H "Content-Type: application/json" \ + -d "{\"slug\": \"$(DOMAIN)\", \"name\": \"$(NAME)\"}" | python3 -m json.tool + +## Rename a domain: make rename-domain DOMAIN=old_slug NEW_SLUG=new_slug NEW_NAME="New Name" +rename-domain: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN (old slug) is required."; exit 1) + @test -n "$(NEW_SLUG)" || (echo "ERROR: NEW_SLUG is required."; exit 1) + @test -n "$(NEW_NAME)" || (echo "ERROR: NEW_NAME is required."; exit 1) + curl -sf -X PATCH http://127.0.0.1:8000/domains/$(DOMAIN)/rename \ + -H "Content-Type: application/json" \ + -d "{\"new_slug\": \"$(NEW_SLUG)\", \"new_name\": \"$(NEW_NAME)\"}" | python3 -m json.tool + +## Register this machine's local path for a repo: make register-path REPO=marki-docx PATH=/home/tegwick/marki-docx +register-path: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make register-path REPO= PATH="; exit 1) + @test -n "$(PATH)" || (echo "ERROR: PATH is required. Usage: make register-path REPO= PATH="; exit 1) + curl -sf -X POST "http://127.0.0.1:8000/repos/$(REPO)/paths" \ + -H "Content-Type: application/json" \ + -d "{\"host\": \"$$(hostname)\", \"path\": \"$(PATH)\"}" | python3 -m json.tool + +## List repos for a domain: make list-repos DOMAIN=railiance +list-repos: + @test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required."; exit 1) + curl -sf "http://127.0.0.1:8000/repos/?domain=$(DOMAIN)" | python3 -m json.tool + +## Ingest SBOM data for a repo (all mechanisms: lockfiles + ansible + sbom-tools.yaml). +## Auto-detect all sources: make ingest-sbom REPO=the-custodian REPO_PATH=/home/worsch/the-custodian +## Single lockfile (explicit): make ingest-sbom REPO=the-custodian LOCKFILE=/path/to/uv.lock +## Dry-run (no submit): make ingest-sbom REPO=the-custodian REPO_PATH=... DRY_RUN=1 +## Tip: run capture-tools first for repos with system-level tool dependencies. +ingest-sbom: + @test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1) + $(UV) run python scripts/ingest_sbom.py --repo "$(REPO)" \ + $(if $(LOCKFILE),--lockfile "$(LOCKFILE)") \ + $(if $(REPO_PATH),--repo-path "$(REPO_PATH)") \ + $(if $(DRY_RUN),--dry-run) + +## Ingest capability declarations from SCOPE.md into the catalog. +## Usage: make ingest-capabilities REPO=the-custodian [REPO_PATH=/home/worsch/the-custodian] +## Or: make ingest-capabilities-all +## Add DRY_RUN=1 to preview without writing. +ingest-capabilities: + @test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1) + $(UV) run python scripts/ingest_capabilities.py --repo "$(REPO)" \ + $(if $(REPO_PATH),--repo-path "$(REPO_PATH)") \ + $(if $(DRY_RUN),--dry-run) + +ingest-capabilities-all: + $(UV) run python scripts/ingest_capabilities.py --all \ + $(if $(DRY_RUN),--dry-run) + +## Check Repository Definition of Integrated (DoI) criteria for a repo. +## Usage: make check-doi REPO=llm-connect +## Or: make check-doi-all +## Add JSON=1 for machine-readable output. +check-doi: + @test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1) + $(UV) run python scripts/check_doi.py --repo "$(REPO)" $(if $(JSON),--json) + +check-doi-all: + $(UV) run python scripts/check_doi.py --all $(if $(JSON),--json) + +## Ingest tpsc.yaml service declarations from a repo into the TPSC catalog. +## Usage: make ingest-tpsc REPO=llm-connect +## Or: make ingest-tpsc-all +## Add DRY_RUN=1 to preview without writing. +ingest-tpsc: + @test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1) + $(UV) run python scripts/ingest_tpsc.py --repo "$(REPO)" \ + $(if $(DRY_RUN),--dry-run) + +ingest-tpsc-all: + $(UV) run python scripts/ingest_tpsc.py --all \ + $(if $(DRY_RUN),--dry-run) + +## Run SBOM capture agent for a repo — generates/updates sbom-tools.yaml. +## Usage: make capture-tools REPO=railiance-infra [REPO_PATH=/home/worsch/railiance-infra] +## Add DRY_RUN=1 to preview without writing. +capture-tools: + @test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1) + $(UV) run python scripts/capture_sbom_tools.py --repo "$(REPO)" \ + $(if $(REPO_PATH),--repo-path "$(REPO_PATH)") \ + $(if $(DRY_RUN),--dry-run) + +## Check a repo for ADR-001 compliance: make validate-adr REPO=/path/to/repo [DOMAIN=custodian] +validate-adr: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make validate-adr REPO= [DOMAIN=]"; exit 1) + $(UV) run python scripts/validate_repo_adr.py "$(REPO)" $(if $(DOMAIN),--domain "$(DOMAIN)",) + +## Consistency exit contract: +## - Direct scripts/consistency_check.py: 0 clean, 2 warnings-only, 1 failures. +## - Agent/operator Make wrappers below normalize warning-only 2 to shell success +## while preserving visible WARN output and keeping real failures non-zero. +## Check a single repo for ADR-001 consistency: make check-consistency REPO=the-custodian [REPO_PATH=/override] +## Exit 0 = clean or warnings-only (warnings stay visible), exit 1 = failures +check-consistency: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make check-consistency REPO="; exit 1) + $(UV) run python scripts/consistency_check.py --repo "$(REPO)" \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(REPO_PATH),--repo-path "$(REPO_PATH)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Check and auto-fix a single repo: make fix-consistency REPO=the-custodian [REPO_PATH=/override] +## Exit 0 = clean or warnings-only (warnings stay visible), exit 1 = failures +fix-consistency: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make fix-consistency REPO="; exit 1) + $(UV) run python scripts/consistency_check.py --repo "$(REPO)" --fix \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(REPO_PATH),--repo-path "$(REPO_PATH)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Normalize workplan frontmatter and task status literals in attached repos. +## Usage: make normalize-attached-workplans REPO=artifact-store +## make normalize-attached-workplans DIRTY=1 +normalize-attached-workplans: + $(UV) run python scripts/normalize_attached_repo_workplans.py \ + $(if $(REPO),--repo "$(REPO)",) \ + $(if $(DIRTY),--dirty,) \ + $(if $(DRY_RUN),--dry-run,) + @test -n "$(REPO)$(DIRTY)" || (echo "ERROR: set REPO= or DIRTY=1"; exit 1) + +## Regenerate AGENTS.md / CLAUDE.md / .claude/rules from templates. +## Usage: make update-agent-instructions REPO=artifact-store +## make update-agent-instructions DIRTY=1 +update-agent-instructions: + $(UV) run python scripts/update_agent_instruction_files.py \ + $(if $(REPO),--repo "$(REPO)",) \ + $(if $(DIRTY),--dirty,) + @test -n "$(REPO)$(DIRTY)" || (echo "ERROR: set REPO= or DIRTY=1"; exit 1) + +## Reconcile measured token sources against State Hub. +## Usage: make token-reconcile [SINCE=2026-05-19] [APPLY=1] [ZERO_FALLBACKS=1] +token-reconcile: + $(PYTHON) scripts/token_reconcile.py \ + $(if $(SINCE),--since "$(SINCE)",) \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(CODEX_HOME),--codex-home "$(CODEX_HOME)",) \ + $(if $(CLAUDE_HOME),--claude-home "$(CLAUDE_HOME)",) \ + $(if $(APPLY),--apply,) \ + $(if $(ZERO_FALLBACKS),--zero-superseded-fallbacks,) + +## Pull then fix: single repo or all repos if REPO omitted +## make fix-consistency-remote — smart pull+fix all repos that need it +## make fix-consistency-remote REPO=slug — pull+fix one repo +fix-consistency-remote: + $(UV) run python scripts/consistency_check.py \ + $(if $(REPO),--repo "$(REPO)",--all) \ + --remote \ + $(if $(API_BASE),--api-base "$(API_BASE)",) \ + $(if $(NO_WRITEBACK),--no-writeback,); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Infer repo slug from git remote URL and check: make check-consistency-here [REPO_PATH=/path/to/repo] +## Omit REPO_PATH to use the Python script's CWD (i.e. pass an empty --here flag). +check-consistency-here: + $(UV) run python scripts/consistency_check.py \ + --here $(if $(REPO_PATH),"$(REPO_PATH)",) \ + $(if $(API_BASE),--api-base "$(API_BASE)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Infer repo slug from git remote URL and fix: make fix-consistency-here [REPO_PATH=/path/to/repo] +fix-consistency-here: + $(UV) run python scripts/consistency_check.py \ + --here $(if $(REPO_PATH),"$(REPO_PATH)",) \ + --fix \ + $(if $(API_BASE),--api-base "$(API_BASE)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Check all registered repos for ADR-001 consistency +check-consistency-all: + $(UV) run python scripts/consistency_check.py --all $(if $(API_BASE),--api-base "$(API_BASE)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Check and auto-fix all registered repos +fix-consistency-all: + $(UV) run python scripts/consistency_check.py --all --fix $(if $(API_BASE),--api-base "$(API_BASE)",); \ + e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e + +## Cancel open tasks belonging to completed/archived workstreams. +## Safe to run at any time; also suitable for a daily cron job. +## Cron example: 0 3 * * * cd ~/state-hub && make cleanup-stale +cleanup-stale: + $(UV) run python scripts/cleanup_stale_tasks.py + +## Install custodian post-commit sync hook into one repo: make install-hooks REPO=marki-docx +install-hooks: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make install-hooks REPO="; exit 1) + bash scripts/install_hooks.sh --repo "$(REPO)" + +## Install custodian post-commit sync hook into all active registered repos +install-hooks-all: + bash scripts/install_hooks.sh --all + +## Remove custodian post-commit sync hook from one repo: make remove-hooks REPO=marki-docx +remove-hooks: + @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make remove-hooks REPO="; exit 1) + bash scripts/install_hooks.sh --repo "$(REPO)" --remove + +## Compare Gitea coulomb org repos against state-hub registered repos +## Requires GITEA_TOKEN in env or .env: make gitea-inventory GITEA_TOKEN= +gitea-inventory: + $(UV) run python scripts/gitea_inventory.py $(if $(JSON),--json) + +## Register/update one repo from .repo-classification.yaml: +## make register-from-classification REPO=state-hub +## make register-from-classification PATH=/path/to/repo +## Optional: DRY_RUN=1 +register-from-classification: + @test -n "$(REPO)" -o -n "$(PATH)" || (echo "ERROR: REPO or PATH is required."; exit 1) + $(UV) run python scripts/register_from_classification.py \ + $(if $(PATH),--repo-path "$(PATH)",--slug "$(REPO)") \ + $(if $(DRY_RUN),--dry-run,) + +## Bulk register/update all active repos with accessible local paths +register-from-classification-all: + $(UV) run python scripts/register_from_classification.py --bulk \ + $(if $(DRY_RUN),--dry-run,) + +clean: + $(COMPOSE) down -v diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/api/routers/workstreams.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/api/routers/workstreams.py new file mode 100644 index 00000000..cb6724a5 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/api/routers/workstreams.py @@ -0,0 +1,591 @@ +import asyncio +import logging +import uuid +import socket +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import yaml +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response, status +from sqlalchemy import or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from api.database import get_session +from api.events import EventEnvelope, publish_event +from api.models.managed_repo import ManagedRepo +from api.models.workplan import Workplan +from api.models.task import Task +from api.services.historical_projection import frozen_cluster_binding, validate_frozen_cluster_rows +from api.schemas.workplan import ( + WorkplanBindingsSync, + WorkplanCreate, + WorkplanRead, + WorkplanUpdate, +) +from api.services.lifecycle import transition_workplan_status +from api.services.legacy_compat import retire_legacy_route +from api.services.wait_kind import annotate_workplans +from api.work_record_flavor import ( + RESIDUAL_FLAVOR, + WORK_RECORD_FLAVORS, + normalize_flavor, +) +from api.workplan_status import ( + is_supported_workplan_status, + normalize_workplan_status, + ready_review_status, +) + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/workstreams", tags=["workstreams"]) +workplan_router = APIRouter(prefix="/workplans", tags=["workplans"]) + +_INDEX_CACHE: dict[str, Any] | None = None +_INDEX_CACHE_AT: float = 0.0 +_INDEX_TTL = 30.0 +_INDEX_REFRESH_TASK: asyncio.Task | None = None +_INDEX_LAST_ERROR: str | None = None + +_COMPLETED_WORKPLAN_EVENT = "org.statehub.workplan.completed" + + +def _repo_path(repo: ManagedRepo) -> Path | None: + hostname = socket.gethostname() + candidates = [] + host_paths = repo.host_paths or {} + if host_paths.get(hostname): + candidates.append(host_paths[hostname]) + if repo.local_path: + candidates.append(repo.local_path) + for raw in candidates: + path = Path(raw).expanduser() + if path.is_dir(): + return path + return None + + +def _frontmatter(path: Path) -> dict[str, Any]: + try: + text = path.read_text(encoding="utf-8") + except OSError: + return {} + if not text.startswith("---\n"): + return {} + end = text.find("\n---", 4) + if end == -1: + return {} + + try: + return yaml.safe_load(text[4:end].strip()) or {} + except yaml.YAMLError: + return {} + + +def _legacy_key(method: str, route: str) -> str: + return f"rest_api:{method} {route}" + + +async def _list_workplans( + *, + topic_id: uuid.UUID | None, + repo_id: uuid.UUID | None, + repo_goal_id: uuid.UUID | None, + status_filter: str | None, + owner: str | None, + slug: str | None, + flavor: str | None = None, + include_residuals: bool = True, + session: AsyncSession, +) -> list[Workplan]: + q = select(Workplan) + if topic_id: + q = q.where(Workplan.topic_id == topic_id) + if repo_id: + q = q.where(Workplan.repo_id == repo_id) + if repo_goal_id: + q = q.where(Workplan.repo_goal_id == repo_goal_id) + if status_filter: + normalised_status = normalize_workplan_status(status_filter) + if not is_supported_workplan_status(status_filter): + raise HTTPException(status_code=422, detail=f"Unsupported workplan status '{status_filter}'") + q = q.where(Workplan.status == normalised_status) + if owner: + q = q.where(Workplan.owner == owner) + if slug: + q = q.where(Workplan.slug == slug) + wanted = normalize_flavor(flavor) + if wanted is not None: + if wanted not in WORK_RECORD_FLAVORS: + raise HTTPException( + status_code=422, + detail=( + f"Unknown work-record flavor {wanted!r}; " + f"expected one of {', '.join(WORK_RECORD_FLAVORS)}" + ), + ) + q = q.where(Workplan.flavor == wanted) + elif not include_residuals: + q = q.where(or_(Workplan.flavor.is_(None), Workplan.flavor != RESIDUAL_FLAVOR)) + q = q.order_by( + Workplan.planning_priority.asc().nullslast(), + Workplan.planning_order.asc().nullslast(), + Workplan.updated_at.desc(), + ) + result = await session.execute(q) + return list(result.scalars().all()) + + +async def _build_workplan_index(session: AsyncSession) -> dict[str, Any]: + result = await session.execute( + select(ManagedRepo).where(ManagedRepo.status == "active").order_by(ManagedRepo.slug) + ) + index: dict[str, Any] = {} + for repo in result.scalars().all(): + root = _repo_path(repo) + if root is None: + continue + for directory, archived in ( + (root / "workplans", False), + (root / "workplans" / "archived", True), + ): + if not directory.is_dir(): + continue + for path in sorted(directory.glob("*.md")): + data = _frontmatter(path) + workplan_id = data.get("state_hub_workstream_id") or data.get("state_hub_workplan_id") + if not workplan_id: + continue + file_status = normalize_workplan_status(data.get("status", "")) + review = ( + ready_review_status( + root, + data.get("reviewed_against_commit"), + data.get("context_paths"), + ) + if file_status == "ready" + else None + ) + index[str(workplan_id)] = { + "filename": path.name, + "relative_path": str(path.relative_to(root)), + "repo_slug": repo.slug, + "archived": archived, + "status": file_status or None, + "needs_review": bool(review and review.needs_review), + "health_labels": ["needs_review"] if review and review.needs_review else [], + } + await _merge_db_backing_index(session, index) + return {"workplans": index, "workstreams": index} + + +async def _merge_db_backing_index(session: AsyncSession, index: dict[str, Any]) -> None: + """Fill index gaps from DB-backed file bindings synced by fix-consistency.""" + result = await session.execute( + select(Workplan, ManagedRepo.slug) + .join(ManagedRepo, Workplan.repo_id == ManagedRepo.id) + .where(Workplan.backing_filename.isnot(None)) + ) + for wp, repo_slug in result.all(): + key = str(wp.id) + if key in index: + continue + index[key] = { + "filename": wp.backing_filename, + "relative_path": wp.backing_relative_path, + "repo_slug": repo_slug, + "archived": bool(wp.backing_archived), + "status": normalize_workplan_status(wp.status) if wp.status else None, + "needs_review": False, + "health_labels": [], + } + + +def _invalidate_workplan_index_cache() -> None: + global _INDEX_CACHE, _INDEX_CACHE_AT + _INDEX_CACHE = None + _INDEX_CACHE_AT = 0.0 + + +def _index_with_meta(*, stale: bool, refresh_in_progress: bool) -> dict[str, Any]: + age = time.monotonic() - _INDEX_CACHE_AT if _INDEX_CACHE_AT else None + return { + **(_INDEX_CACHE or {"workplans": {}, "workstreams": {}}), + "_meta": { + "generated_at": _INDEX_CACHE.get("_meta", {}).get("generated_at") if _INDEX_CACHE else None, + "stale": stale, + "cache_age_seconds": round(age, 3) if age is not None else None, + "refresh_in_progress": refresh_in_progress, + "last_error": _INDEX_LAST_ERROR, + }, + } + + +async def _refresh_workplan_index_background() -> None: + global _INDEX_CACHE, _INDEX_CACHE_AT, _INDEX_LAST_ERROR + from api.database import async_session_factory + + try: + async with async_session_factory() as session: + index = await _build_workplan_index(session) + index["_meta"] = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "stale": False, + "cache_age_seconds": 0.0, + "refresh_in_progress": False, + "last_error": None, + } + _INDEX_CACHE = index + _INDEX_CACHE_AT = time.monotonic() + _INDEX_LAST_ERROR = None + except Exception as exc: + _INDEX_LAST_ERROR = str(exc) + + +def _ensure_index_refresh_started() -> None: + global _INDEX_REFRESH_TASK + if _INDEX_REFRESH_TASK is not None and not _INDEX_REFRESH_TASK.done(): + return + _INDEX_REFRESH_TASK = asyncio.create_task(_refresh_workplan_index_background()) + + +async def _workplan_index( + *, + refresh: bool, + session: AsyncSession, +) -> dict[str, Any]: + """Map file-backed workplan ids to their local workplan filenames.""" + global _INDEX_CACHE, _INDEX_CACHE_AT, _INDEX_LAST_ERROR + cache_age = time.monotonic() - _INDEX_CACHE_AT if _INDEX_CACHE_AT else None + if not refresh and _INDEX_CACHE is not None and cache_age is not None and cache_age < _INDEX_TTL: + refresh_running = _INDEX_REFRESH_TASK is not None and not _INDEX_REFRESH_TASK.done() + return _index_with_meta(stale=False, refresh_in_progress=refresh_running) + + if not refresh and _INDEX_CACHE is not None: + _ensure_index_refresh_started() + return _index_with_meta(stale=True, refresh_in_progress=True) + + index = await _build_workplan_index(session) + index["_meta"] = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "stale": False, + "cache_age_seconds": 0.0, + "refresh_in_progress": False, + "last_error": None, + } + _INDEX_CACHE = index + _INDEX_CACHE_AT = time.monotonic() + _INDEX_LAST_ERROR = None + return _INDEX_CACHE + + +async def _create_workplan( + *, + body: WorkplanCreate, + session: AsyncSession, +) -> Workplan: + wp = Workplan(**body.model_dump()) + session.add(wp) + await session.commit() + await session.refresh(wp) + return wp + + +async def _get_workplan( + *, + workplan_id: uuid.UUID, + session: AsyncSession, +) -> Workplan: + wp = await session.get(Workplan, workplan_id) + if wp is None: + raise HTTPException(status_code=404, detail="Workplan not found") + return wp + + +async def _update_workplan( + *, + workplan_id: uuid.UUID, + body: WorkplanUpdate, + session: AsyncSession, +) -> Workplan: + wp = await session.get(Workplan, workplan_id) + if wp is None: + raise HTTPException(status_code=404, detail="Workplan not found") + update_data = body.model_dump(exclude_unset=True) + status_update = update_data.pop("status", None) + prev_status = wp.status + for field, value in update_data.items(): + setattr(wp, field, value) + if status_update is not None: + transition_workplan_status(wp, status_update) + await session.commit() + await session.refresh(wp) + + if normalize_workplan_status(prev_status) != "finished" and wp.status == "finished": + await _publish_completion_events(wp, session) + + return wp + + +async def _archive_workplan( + *, + workplan_id: uuid.UUID, + session: AsyncSession, +) -> Workplan: + wp = await session.get(Workplan, workplan_id) + if wp is None: + raise HTTPException(status_code=404, detail="Workplan not found") + transition_workplan_status(wp, "archived") + await session.commit() + await session.refresh(wp) + return wp + + +async def _publish_completion_events(wp: Workplan, session: AsyncSession) -> None: + workplan_envelope = EventEnvelope.new( + _COMPLETED_WORKPLAN_EVENT, + attributes={ + "workplan_id": str(wp.id), + "slug": wp.slug, + "title": wp.title, + "topic_id": str(wp.topic_id) if wp.topic_id else None, + "repo_id": str(wp.repo_id) if wp.repo_id else None, + "repo_goal_id": str(wp.repo_goal_id) if wp.repo_goal_id else None, + }, + ) + asyncio.create_task(publish_event(_COMPLETED_WORKPLAN_EVENT, workplan_envelope)) + + +@router.get("/", status_code=status.HTTP_410_GONE) +async def list_workstreams( + request: Request, + response: Response, + topic_id: uuid.UUID | None = None, + repo_id: uuid.UUID | None = None, + repo_goal_id: uuid.UUID | None = None, + status: str | None = None, + owner: str | None = None, + slug: str | None = None, + session: AsyncSession = Depends(get_session), +) -> None: + await retire_legacy_route( + session=session, + request=request, + response=response, + interface_key=_legacy_key("GET", "/workstreams/"), + replacement_ref="/workplans/", + detail="Legacy GET /workstreams/ retired; use GET /workplans/", + ) + + +@workplan_router.get("/", response_model=list[WorkplanRead]) +async def list_workplans( + topic_id: uuid.UUID | None = None, + repo_id: uuid.UUID | None = None, + repo_goal_id: uuid.UUID | None = None, + status: str | None = None, + owner: str | None = None, + slug: str | None = None, + flavor: str | None = Query(None), + include_residuals: bool = Query(True), + session: AsyncSession = Depends(get_session), +) -> list[Workplan]: + workplans = await _list_workplans( + topic_id=topic_id, + repo_id=repo_id, + repo_goal_id=repo_goal_id, + status_filter=status, + owner=owner, + slug=slug, + flavor=flavor, + include_residuals=include_residuals, + session=session, + ) + return list(await annotate_workplans(session, workplans)) + + +@router.get("/workplan-index", status_code=status.HTTP_410_GONE) +async def workplan_index( + request: Request, + response: Response, + refresh: bool = Query(False, description="Force cache invalidation"), + session: AsyncSession = Depends(get_session), +) -> None: + await retire_legacy_route( + session=session, + request=request, + response=response, + interface_key=_legacy_key("GET", "/workstreams/workplan-index"), + replacement_ref="/workplans/index", + detail="Legacy GET /workstreams/workplan-index retired; use GET /workplans/index", + ) + + +@workplan_router.get("/index") +async def workplan_index_preferred( + refresh: bool = Query(False, description="Force cache invalidation"), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + return await _workplan_index(refresh=refresh, session=session) + + +@workplan_router.put("/index/bindings") +async def sync_workplan_bindings( + body: WorkplanBindingsSync, + session: AsyncSession = Depends(get_session), +) -> dict[str, int]: + """Upsert workstation workplan file bindings for remote API index fallback.""" + # Preflight the complete batch before assigning even an ordinary row. + try: + frozen = {entry.workplan_id: bound for entry in body.bindings + if (bound := frozen_cluster_binding(entry.model_dump())) is not None} + except ValueError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + requested_ids = {entry.workplan_id for entry in body.bindings} + rows = await session.execute( + select(Workplan).where(Workplan.id.in_(requested_ids)) + .order_by(Workplan.id).with_for_update().execution_options(populate_existing=True) + ) + workplans = {workplan.id: workplan for workplan in rows.scalars().all()} + if frozen: + repo_rows = await session.execute( + select(ManagedRepo).where(ManagedRepo.slug == "railiance-cluster") + ) + repo = repo_rows.scalar_one_or_none() + task_rows = await session.execute( + select(Task).where(Task.workplan_id.in_(frozen)) + .order_by(Task.id).with_for_update().execution_options(populate_existing=True) + ) + tasks = task_rows.scalars().all() + try: + for workplan_id, bound in frozen.items(): + validate_frozen_cluster_rows( + bound, workplans.get(workplan_id), tasks, repo.id if repo else None + ) + except ValueError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + synced_at = datetime.now(timezone.utc) + updated = 0 + preserved = 0 + for entry in body.bindings: + if entry.workplan_id in frozen: + preserved += 1 + continue # No setters, including backing_synced_at/updated_at. + wp = workplans.get(entry.workplan_id) + if wp is None: + continue + wp.backing_filename = entry.filename + wp.backing_relative_path = entry.relative_path + wp.backing_archived = entry.archived + wp.backing_synced_at = synced_at + updated += 1 + if updated: + await session.commit() + _invalidate_workplan_index_cache() + return {"updated": updated, "received": len(body.bindings), "preserved": preserved} + + +@router.post("/", status_code=status.HTTP_410_GONE) +async def create_workstream( + request: Request, + response: Response, + body: WorkplanCreate, + session: AsyncSession = Depends(get_session), +) -> None: + await retire_legacy_route( + session=session, + request=request, + response=response, + interface_key=_legacy_key("POST", "/workstreams/"), + replacement_ref="/workplans/", + detail="Legacy POST /workstreams/ retired; use POST /workplans/", + ) + + +@workplan_router.post("/", response_model=WorkplanRead, status_code=status.HTTP_201_CREATED) +async def create_workplan( + body: WorkplanCreate, + session: AsyncSession = Depends(get_session), +) -> Workplan: + return await _create_workplan(body=body, session=session) + + +@router.get("/{workstream_id}", status_code=status.HTTP_410_GONE) +async def get_workstream( + request: Request, + response: Response, + workstream_id: uuid.UUID, + session: AsyncSession = Depends(get_session), +) -> None: + await retire_legacy_route( + session=session, + request=request, + response=response, + interface_key=_legacy_key("GET", "/workstreams/{workstream_id}"), + replacement_ref="/workplans/{workplan_id}", + detail="Legacy GET /workstreams/{workstream_id} retired; use GET /workplans/{workplan_id}", + ) + + +@workplan_router.get("/{workplan_id}", response_model=WorkplanRead) +async def get_workplan( + workplan_id: uuid.UUID, + session: AsyncSession = Depends(get_session), +) -> Workplan: + wp = await _get_workplan(workplan_id=workplan_id, session=session) + await annotate_workplans(session, [wp]) + return wp + + +@router.patch("/{workstream_id}", status_code=status.HTTP_410_GONE) +async def update_workstream( + request: Request, + response: Response, + workstream_id: uuid.UUID, + body: WorkplanUpdate, + session: AsyncSession = Depends(get_session), +) -> None: + await retire_legacy_route( + session=session, + request=request, + response=response, + interface_key=_legacy_key("PATCH", "/workstreams/{workstream_id}"), + replacement_ref="/workplans/{workplan_id}", + detail="Legacy PATCH /workstreams/{workstream_id} retired; use PATCH /workplans/{workplan_id}", + ) + + +@workplan_router.patch("/{workplan_id}", response_model=WorkplanRead) +async def update_workplan( + workplan_id: uuid.UUID, + body: WorkplanUpdate, + session: AsyncSession = Depends(get_session), +) -> Workplan: + return await _update_workplan(workplan_id=workplan_id, body=body, session=session) + + +@router.delete("/{workstream_id}", status_code=status.HTTP_410_GONE) +async def archive_workstream( + request: Request, + response: Response, + workstream_id: uuid.UUID, + session: AsyncSession = Depends(get_session), +) -> None: + await retire_legacy_route( + session=session, + request=request, + response=response, + interface_key=_legacy_key("DELETE", "/workstreams/{workstream_id}"), + replacement_ref="DELETE /workplans/{workplan_id}", + detail="Legacy DELETE /workstreams/{workstream_id} retired; use DELETE /workplans/{workplan_id}", + ) + + +@workplan_router.delete("/{workplan_id}", response_model=WorkplanRead) +async def archive_workplan( + workplan_id: uuid.UUID, + session: AsyncSession = Depends(get_session), +) -> Workplan: + return await _archive_workplan(workplan_id=workplan_id, session=session) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/api/services/historical_projection.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/api/services/historical_projection.py new file mode 100644 index 00000000..97a50a3b --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/api/services/historical_projection.py @@ -0,0 +1,238 @@ +"""Exact closed source bindings predating canonical Forge identity. + +These are frozen historical source views, not aliases or UUID allocation rules. +An altered/missing source requires explicit owner review; generic source UUIDs +remain untrusted. Shared Activity views represent the one archived parent. +""" +from pathlib import Path +from hashlib import sha256 +import json + +BINDINGS = json.loads(r'''[ + { + "repo": "activity-core", + "path": "workplans/archived/260707-custodian-WP-0003-event-bridge.md", + "sha256": "83ea8bcb151e003419b73c59dbd028f4a1ce3d1120a82e4a128a772503ff3b76", + "record_id": "custodian-WP-0003", + "uuid": "b4eb45a9-69e3-4ab0-b00c-67a53c3117c5", + "parent_record_id": "custodian-WP-0003" + }, + { + "repo": "activity-core", + "path": "workplans/custodian-WP-0001-temporal-backbone.md", + "sha256": "020e2eac973f2daa41c1687f8e867da9d521f945547e6aa82c87993b3ff6fc64", + "record_id": "custodian-WP-0001", + "uuid": "e22516cc-4fdf-4dad-a928-a9e04a9fa7c7", + "parent_record_id": "custodian-WP-0001" + }, + { + "repo": "activity-core", + "path": "workplans/custodian-WP-0002-triggers-ops.md", + "sha256": "d8cbc61631c01b6f11a03a63f2fb6275fc853ff7f7459ce65973ed1a233daa01", + "record_id": "custodian-WP-0002", + "uuid": "3a4f47d9-8bc1-434e-acb4-bed5d4dacda0", + "parent_record_id": "custodian-WP-0002" + }, + { + "repo": "activity-core", + "path": "workplans/custodian-WP-0003a-event-bridge-model-rules-registry.md", + "sha256": "ffa6682e9ba3aaf559bd44203d561c90dea69bcf1b1e778209a7b209e6e5cdd3", + "record_id": "custodian-WP-0003a", + "uuid": "b4eb45a9-69e3-4ab0-b00c-67a53c3117c5", + "parent_record_id": "custodian-WP-0003" + }, + { + "repo": "activity-core", + "path": "workplans/custodian-WP-0003b-event-bridge-parser-workflow-triggers.md", + "sha256": "4fc34e0f821e2df51e5c08c7ab34da35b65690381d01f0cfdf55d00c970b414c", + "record_id": "custodian-WP-0003b", + "uuid": "b4eb45a9-69e3-4ab0-b00c-67a53c3117c5", + "parent_record_id": "custodian-WP-0003" + }, + { + "repo": "activity-core", + "path": "workplans/custodian-WP-0003c-event-bridge-context-integration.md", + "sha256": "b6cd62b26515e509670ca39736b6bcec9633beedd46c9bd8f7598c06bac7276b", + "record_id": "custodian-WP-0003c", + "uuid": "b4eb45a9-69e3-4ab0-b00c-67a53c3117c5", + "parent_record_id": "custodian-WP-0003" + }, + { + "repo": "activity-core", + "path": "workplans/custodian-WP-0004-railiance-ops.md", + "sha256": "20df02eea740e109c937130e7f8801a636fc0990ed8c6d2ced55659f8957adb1", + "record_id": "custodian-WP-0004", + "uuid": "759b1255-aa78-42b5-8fab-c5fcf168f5f4", + "parent_record_id": "custodian-WP-0004" + }, + { + "repo": "the-custodian", + "path": "workplans/ADHOC-2026-08-25-retired.md", + "sha256": "344a1293dbb3a67a944eb34ca4a0337487d8f047226027a2107922346fbab2dd", + "record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED", + "uuid": "1c53d3db-6633-5e2d-987f-10706d73be9e", + "parent_record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED" + }, + { + "repo": "railiance-cluster", + "path": "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md", + "sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9", + "record_id": "RAIL-BS-WP-0002", + "uuid": "4c63dfc6-9eac-4e79-9f77-8f644ad7147d", + "parent_record_id": "RAIL-BS-WP-0002", + "tasks": [ + { + "label": "T01", + "uuid": "3f042630-eab0-4c6a-9167-e2b28ff20e40", + "status": "done" + }, + { + "label": "T02", + "uuid": "e8510646-46ed-4697-a345-f3d3009eea78", + "status": "done" + }, + { + "label": "T03", + "uuid": "dab2c07f-8aa0-4635-8df6-857e87e93fc5", + "status": "done" + }, + { + "label": "T04", + "uuid": "5c3d40e4-239b-488e-9519-6f7a38d2325f", + "status": "done" + }, + { + "label": "T05", + "uuid": "7f9e0e58-a130-467a-a2d0-b3f2564e496f", + "status": "done" + } + ] + }, + { + "repo": "railiance-cluster", + "path": "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md", + "sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a", + "record_id": "RAIL-BS-WP-0003", + "uuid": "7ee9ee22-1fae-4567-9194-8d70a9e0f45b", + "parent_record_id": "RAIL-BS-WP-0003", + "tasks": [ + { + "label": "T01", + "uuid": "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc", + "status": "done" + }, + { + "label": "T02", + "uuid": "ab166073-30a7-4702-a037-4091e8706e20", + "status": "done" + }, + { + "label": "T03", + "uuid": "140da396-8e30-4f4d-b88c-c42c0cd46c01", + "status": "done" + }, + { + "label": "T04", + "uuid": "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d", + "status": "done" + } + ] + }, + { + "repo": "railiance-cluster", + "path": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md", + "sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3", + "record_id": "RAIL-BS-WP-0004", + "uuid": "7e8b0c20-51eb-40c9-9e3b-85dd380d7625", + "parent_record_id": "RAIL-BS-WP-0004", + "tasks": [ + { + "label": "T01", + "uuid": "4526a842-ea31-4874-9231-92ab556cfe7b", + "status": "done" + }, + { + "label": "T02", + "uuid": "a6313e06-1976-46a7-8e31-df4eb2eca880", + "status": "done" + }, + { + "label": "T03", + "uuid": "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3", + "status": "done" + }, + { + "label": "T04", + "uuid": "08233868-d522-4117-bc4e-6c0f52545665", + "status": "done" + }, + { + "label": "T05", + "uuid": "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8", + "status": "done" + }, + { + "label": "T06", + "uuid": "f8e4a094-c367-40eb-b895-da17bc144b07", + "status": "done" + } + ] + } +]''') + + +def bound_sources(repo_root: Path, repo_slug: str): + entries = [entry for entry in BINDINGS if entry["repo"] == repo_slug] + result = [] + for entry in entries: + path = repo_root / entry["path"] + if path.is_symlink() or not path.is_file(): + raise ValueError("Historical source binding missing or symlinked: " + entry["path"]) + data = path.read_bytes() + if sha256(data).hexdigest() != entry["sha256"]: + raise ValueError("Historical source binding changed: " + entry["path"]) + result.append((entry, data.decode("utf-8"))) + return result + + +# Only the three existing full-task Cluster bindings have immutable DB rows. +# Other historical views retain their existing compatibility behavior. +FROZEN_CLUSTER_BINDINGS = tuple( + entry for entry in BINDINGS + if entry["repo"] == "railiance-cluster" and "tasks" in entry +) + + +def frozen_cluster_binding(binding): + """Validate a claimed frozen binding; unrelated requests remain ordinary.""" + matches = [entry for entry in FROZEN_CLUSTER_BINDINGS + if str(binding["workplan_id"]) == entry["uuid"] + or (binding["repo_slug"] == "railiance-cluster" + and (binding["relative_path"] == entry["path"] + or binding["filename"] == Path(entry["path"]).name))] + if not matches: + return None + if len(matches) != 1: + raise ValueError("frozen historical binding is ambiguous") + entry = matches[0] + expected = { + "workplan_id": entry["uuid"], "repo_slug": entry["repo"], + "filename": Path(entry["path"]).name, "relative_path": entry["path"], + "archived": True, "status": "finished", + } + if any((str(binding[key]) if key == "workplan_id" else binding[key]) != value + for key, value in expected.items()): + raise ValueError("frozen historical binding metadata changed") + return entry + + +def validate_frozen_cluster_rows(entry, parent, tasks, repo_id): + """Receiving requires the exact existing terminal closure, never repair.""" + if (parent is None or str(parent.id) != entry["uuid"] + or repo_id is None or parent.repo_id != repo_id + or parent.status not in {"done", "finished", "archived", "completed"}): + raise ValueError("frozen historical parent missing, foreign or nonterminal") + local = {str(task.id): task for task in tasks if task.workplan_id == parent.id} + if any(task["uuid"] not in local or local[task["uuid"]].status not in {"done", "cancel"} + for task in entry["tasks"]): + raise ValueError("frozen historical task missing, foreign or nonterminal") diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/deploy/railiance/apps/charts/state-hub/Chart.yaml b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/deploy/railiance/apps/charts/state-hub/Chart.yaml new file mode 100644 index 00000000..1cb3040f --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/deploy/railiance/apps/charts/state-hub/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: state-hub +description: State Hub API service for private Railiance operation +type: application +version: 0.1.0 +appVersion: "main-51bc70d" diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/deploy/railiance/apps/charts/state-hub/files/repo-classification.allowed.yaml b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/deploy/railiance/apps/charts/state-hub/files/repo-classification.allowed.yaml new file mode 100644 index 00000000..2eec2808 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/deploy/railiance/apps/charts/state-hub/files/repo-classification.allowed.yaml @@ -0,0 +1,127 @@ +# GENERATED — do not edit. +# Synced from the-custodian/canon/standards/repo-classification.allowed.yaml +# by scripts/sync_classification_allowed.py (CUST-WP-0067-T09). +# Canon is authoritative; this copy exists only so the container has one. +# Machine-readable allowed-values for the Repo Classification Standard. +# +# Single source of truth for the standard's controlled vocabularies, derived +# from canon/standards/repo-classification-standard_v1.0.md. Consumed by: +# - the per-repo .repo-classification.yaml linter (tools/validate_repo_classification.py) +# - the State Hub registration validator (CUST-WP-0050 T04) +# +# When the standard's vocabularies change, update this file and bump `version` +# to match the standard version. CUST-WP-0050 T01. + +standard: "Repo Classification Standard" +version: "1.1" +canon_id: "canon-repo-classification" + +# category — exactly 1 required (§5) +categories: + - experimental + - research + - project + - tooling + - product + - business + - publication + +# domain / secondary_domains — primary exactly 1; secondaries 0..n (§6) +domains: + - infotech + - financials + - communication + - consumer + - health + - industrials + - energy + - utilities + - materials + - realestate + - crypto + - agents + - space + - government + +# business_stake — 0..n; 2..6 recommended (§8) +business_stake: + - execution + - intelligence + - finance + - legal + - sales + - experience + - technology + - operations + - product + - people + - procurement + - sustainability + - automation + +# business_mechanics — 0..n, optional (§9) +business_mechanics: + - intention + - control + - coordination + - operation + - adaptation + +# capability_tags are intentionally OPEN-ENDED (§7): lowercase kebab-case, not +# restricted to this set. The families below are the standard's recommended +# canonical tags — used to warn on likely synonyms/typos, never to reject. +capability_families: + identity_and_access: + - identity + - authentication + - authorization + - access-control + - user-management + - tenancy + knowledge_and_evidence: + - knowledge + - citations + - evidence + - source-management + - traceability + - documentation + - decision-support + - history + - participation + platform_and_operations: + - platform + - deployment + - operations + - observability + - feature-control + - configuration + - orchestration + market_and_coordination: + - marketplace + - pricing + - monetization + - reputation + - challenges + - bounties + - collaboration + - coordination + product_and_lifecycle: + - product-development + - lifecycle + governance_and_control: + - governance + - policy + - compliance + - risk + - audit + - control + games_and_entertainment: + - game + - entertainment + +# Validation guidance (advisory bounds the linter applies as warnings) +guidance: + secondary_domains_max: 3 + business_stake_recommended_min: 2 + business_stake_recommended_max: 6 + capability_tag_pattern: "^[a-z0-9]+(-[a-z0-9]+)*$" diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/deploy/railiance/apps/helm/state-hub-values.yaml b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/deploy/railiance/apps/helm/state-hub-values.yaml new file mode 100644 index 00000000..02a4bd62 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/deploy/railiance/apps/helm/state-hub-values.yaml @@ -0,0 +1,69 @@ +# Production values for the State Hub Railiance chart handoff. +# Non-secret values only. DATABASE_URL comes from the Secret `state-hub-env`. + +namespace: + create: false + +image: + repository: forgejo.coulomb.social/coulomb/state-hub + tag: "main-51bc70d@sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015" + +ingress: + enabled: false + +# MCP layer on central (CUST-WP-0067-T08). Enabled together with the image tag +# above: MCP_HOST landed in main-6c1262e, and enabling this on an older image +# would deploy a pod that binds loopback and never becomes reachable. +# ClusterIP only — no ingress, same reason the API's is disabled. +mcp: + enabled: true + resources: + requests: + # 2026-09-10: 1,078 observed 5-minute samples, p99 2.35m, peak 30.45m. + # Preserve the 500m limit for bursts; free 40m for the API rollout surge. + cpu: 10m + +config: + # This deployment is the authoritative hub. Callers verify this rather than + # trusting that whatever answered on a port is central (CUST-WP-0067-T03). + instanceRole: primary + instanceLabel: railiance01 + sbomNexusUrl: "http://sbom-nexus.sbom-nexus.svc.cluster.local:8010" + # Reversible T04 read cutover; set back to `legacy` to roll back. + sbomNexusReadMode: nexus + # Reversible T04 write cutover; set back to `legacy` to roll back future writes. + sbomNexusWriteMode: nexus + +resources: + # Restored to the chart's 250m on 2026-09-21. The earlier 100m was a workaround + # for a node with <250m unallocated; ~830m of Knative/Kourier requests were + # released that day (the-custodian docs/kubernetes-change-gate-decision.md). + # State Hub idles near 5m but bursts past 400m under concurrent agent load. + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 2Gi + +sweep: + # RMGR-WP-0005-T11: disabled while railiance01 checkouts still target the + # stale gitea-remote lineage. Re-enable only after the governed remote + # reconciliation and registrar preflight are complete. + enabled: false + # Present in the live release but previously missing here, so deploying from + # this file alone silently dropped it. Inert while enabled is false. + hostname: 239.62.205.92.host.secureserver.net + hostPath: /home/tegwick + sshHostPath: /home/tegwick/.ssh + +# Forge read credential (STATE-WP-0084-T02). Coordinates only — the token lives +# in OpenBao at platform/workloads/state-hub/forge-derivation and reaches the +# pod through Kubernetes auth, never through this file or a Secret. +forgeRead: + enabled: true + +# RPF-WP-0035-T04: dedicated ESO signing key, API-only explicit reference. +renamePreflight: + enabled: true + secretName: state-hub-rename-preflight diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/docs/evidence/2026-10-17-cluster-historical-receiving-rollout.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/docs/evidence/2026-10-17-cluster-historical-receiving-rollout.json new file mode 100644 index 00000000..c2f7f1c3 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/docs/evidence/2026-10-17-cluster-historical-receiving-rollout.json @@ -0,0 +1,116 @@ +{ + "schema": "statehub.cluster-historical-receiving-delivered.v1", + "status": "verified", + "proposal_id": "STATE-WP-0095-T06-cluster-frozen-receiving-r1", + "approved_revision": 4, + "approved_digest_sha256": "sha256:63a02e8296d126c0158b7eb2c4ce79c7eeb6f8d6f61298d473c1065c7557ba32", + "approval_decision_id": "e3798419-8aa9-5425-8c08-20df6bd82a19", + "source_revision": "51bc70d1ff8f25450a8650d8e50caf053093a02e", + "readiness_source_revision": "086d79a2736e34b916bd7de3c8a65e2ea38a3d64", + "helm_revision": 76, + "image_reference": "forgejo.coulomb.social/coulomb/state-hub:main-51bc70d@sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015", + "expected_schema": "c4d5e6f7a8b9", + "health": { + "status": "ok", + "instance_role": "primary", + "instance_label": "railiance01", + "schema": { + "status": "ok", + "applied": "c4d5e6f7a8b9", + "expected": "c4d5e6f7a8b9" + } + }, + "pods_ready": true, + "historical_module_sha256": "sha256:e3c4bf965764b32a6df5d61c71272ee22569df9a9ddae2a3fc3df97ec554e0b2", + "historical_preservation_verified": true, + "parents": 3, + "tasks": 15, + "full_historical_rows_equal": true, + "native_rollback_probes": { + "foreign-holder": { + "full_historical_rows_restored": true, + "historical_binding_refused": true, + "native_outcome_status": "refused", + "rolled_back": true, + "status": "passed" + }, + "missing-task": { + "full_historical_rows_restored": true, + "historical_binding_refused": true, + "native_outcome_status": "refused", + "rolled_back": true, + "status": "passed" + }, + "nonterminal-parent": { + "full_historical_rows_restored": true, + "historical_binding_refused": true, + "native_outcome_status": "refused", + "rolled_back": true, + "status": "passed" + }, + "nonterminal-task": { + "full_historical_rows_restored": true, + "historical_binding_refused": true, + "native_outcome_status": "refused", + "rolled_back": true, + "status": "passed" + }, + "preserve": { + "full_historical_rows_equal": true, + "full_historical_rows_restored": true, + "native_outcome_status": "applied", + "rolled_back": true, + "status": "passed" + } + }, + "native_probe_commits": 0, + "fresh_transaction_full_rows_equal": true, + "source_receiving_gate_delivered": true, + "normal_cluster_publication_verified": false, + "registrar_admission_or_cluster_publication_not_inferred": true, + "authority_limits": [ + "Rollback probe SQL effects rolled back; foreign-holder case is historical-binding refusal, not separate global holder branch.", + "Actual historical native receiving is verified; owner-native TLS/private admission remains unresolved.", + "Migration schema remains additive; Helm rollback does not downgrade schema or stop an active hook Job." + ], + "evidence": [ + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/approval-decision.json", + "sha256": "sha256:b51554c004daa882f3709f67bdae7f2e939d611120383e61be04ec6969878134" + }, + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/helm-return.json", + "sha256": "sha256:61727deacf22d37da2d28cbaf04f5a3ddee0f9ff7571c77f1e85dba97963b572" + }, + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/delivered-runtime.json", + "sha256": "sha256:89681066e8fe139456163ebfafa3efe77ffae8534ea167219230b0a74367f129" + }, + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/native-refusal-probes.json", + "sha256": "sha256:673ccef965c820704dc310a1bf7832b583c30c3edfeb814528cf892555199172" + }, + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/before-full-row-hashes.json", + "sha256": "sha256:75cdd87eef79059ff3f484ed24ca69e285899cdf8d851cc4e472653c417eb0ee" + }, + { + "path": "docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/after-full-row-hashes.json", + "sha256": "sha256:2156aad18f190384838c27e792706a156a7423391408afbc6107e6c28c5872d8" + } + ], + "later_ordinary_cluster_publication": { + "status": "failed_full_parent_row_preservation", + "changed_parent_hashes": 3, + "unchanged_task_hashes": 15, + "exact_changed_columns": "unknown_hash_only_baseline", + "original_parent_preimages": "unavailable_no_guessed_restoration", + "receipt": "the-custodian/docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/post-publication-full-row-hashes-FAILED.json", + "T07": "wait", + "RMGR_WP_0019_T06": "wait", + "followup": [ + "STATE-WP-0095-T09", + "STATE-WP-0095-T10" + ] + } +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/scripts/consistency_check.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/scripts/consistency_check.py new file mode 100644 index 00000000..e282c298 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/scripts/consistency_check.py @@ -0,0 +1,4810 @@ +#!/usr/bin/env python3 +"""consistency_check.py — ADR-001 consistency checking engine. + +Runs bidirectional checks between workplan files in a registered repo and the +state-hub DB. The file is always authoritative; the DB is the cache/index layer. + +Checks: + C-01 workplans-dir FAIL No workplans/ directory missing + C-02 workplan-parse FAIL No Workplan file cannot be parsed + C-03 workstream-stale-ref FAIL No state_hub_workstream_id in file not in DB + C-04 workstream-status-drift WARN Yes File status != DB status (file wins) + C-05 workstream-title-drift WARN Yes File title != DB title (file wins) + C-06 workstream-unlinked WARN Yes Workplan has no state_hub_workstream_id + --fix derives the UUID from the canonical record id and writes it back + C-07 orphan-db-active FAIL No Active DB workstream, no backing file + C-08 orphan-db-closed INFO No Finished/archived (or projection-retired) DB workstream, no file + C-09 workstream-repo-mismatch FAIL Yes DB workstream repo_id != file location + C-10 task-status-drift WARN Yes Task status differs between file and DB + C-11 task-unlinked WARN Yes Task block has no state_hub_task_id + C-12 orphan-db-task WARN Yes DB task in workstream has no file backing unless terminal in a closed workstream + C-13 workstream-auto-complete WARN Yes All DB tasks done but workstream still active + C-14 ghost-duplicate WARN No Active topic workstream with no repo_id matches a file-backed title — probable ghost from premature create_workstream() call + C-15 task-db-ahead WARN No DB task is terminal and file is not — report only; file wins (ADR-001) + C-16 repo-behind-remote WARN No Local repo is behind remote tracking branch — --fix skipped to avoid clobbering remote progress + C-17 repo-ahead-push-failed WARN No Local repo has unpushed commits and push failed — writes skipped to prevent runaway divergence + C-19 workstream-planning-drift WARN Yes planning_priority/planning_order differs between file and DB + C-20 workstream-dependency-missing WARN Yes Workplan dependency frontmatter missing from DB graph + C-22 task-description-drift WARN Yes Task description/content differs between file and DB + C-23 workstream-active-task-planning-status WARN Yes Workstream/workplan is planning while a task is progress or wait + C-24 repo-classification-missing WARN No Registered repo lacks a valid .repo-classification.yaml on disk + C-25 blocked-unblock-sweep WARN No Blocked workplan has unread inbox from counterpart — blocker may have cleared + C-26 workplan-id-prefix WARN No Workplan frontmatter id uses non-canonical prefix for this repo + C-27 workplan-id-collision WARN No Same workplan id appears in multiple repos + C-28 inbox-stale-unread WARN No Unread inbox messages older than INBOX_STALE_DAYS + C-29 inbox-work-unpromoted WARN No Stale unread message looks like a multi-step work request without a workplan + C-30 scope-current-state-stale WARN No SCOPE.md Current State contradicts live workplan statuses + C-31 work-record-unregistered WARN No YAML-block id matches no kind in the canon work-record type registry (sidetrack detector, CUST-WP-0060) + C-32 work-record-not-indexed WARN Yes kind: intake/decision YAML block has no hub id — not indexed in DB (registration, CUST-WP-0061-T02) + C-33 work-record-index-stale WARN Yes WORK-RECORDS.md missing or stale — generated per-repo index (CUST-WP-0061-T04) + C-34 quality-dor-ready WARN No status=ready without quality_dor DoR-Ok (STATE-WP-0077 soft) + C-35 repo-manager-conformance WARN No Repo Manager flavor/standards contract reports findings + C-36 work-record-flavor-unknown WARN No flavor is set but not in the closed list + C-37 residual-provenance-missing WARN No flavor: residual without origin/origin_ref + C-38 task-flavor-drift WARN Yes task flavor differs between file and DB (file wins) + C-39 task-wait-unqualified WARN No unrouted human wait (open non-residual) or wait task has none of task-block depends_on, needs_human: true, blocked_on: message-from:, wait_until: YYYY-MM-DD (CUST-WP-0074 rules 1/6) + C-40 task-wait-blocker-satisfied WARN No wait task whose depends_on targets are all terminal, whose decision_id is resolved, or whose wait_until has passed (CUST-WP-0074 rules 3/4) + C-41 task-wait-qualifier-drift WARN Yes needs_human/blocking_reason/decision_id/blocked_on/wait_until differ between file and DB (file wins) + C-42 workplan-blocked-no-wait WARN No blocked workplan has no task in status wait (CUST-WP-0074-T06) + C-43 decision-supersession-unlinked WARN No resolved decision of this repo says "supersedes " but the cited decision is not superseded_by it (STATE-WP-0097) + (C-20 also indexes task-block depends_on as workplan_dependencies rows with from_task_id set) + (finished¬DoD-Ok is listed by `statehub quality-debt`, not per-file C-warn — avoids historical flood) + +Usage: + python scripts/consistency_check.py --repo SLUG [--fix] [--no-writeback] [--json] [--api-base URL] + python scripts/consistency_check.py --all [--fix] [--no-writeback] [--json] [--api-base URL] + python scripts/consistency_check.py --here [PATH] [--fix] [--no-writeback] [--json] [--api-base URL] + +Identifiers (ADR-007 / RMGR-WP-0005): + C-06 / C-11 / C-32 derive UUIDv5 identifiers from the fleet namespace and + canonical record id. Any instance may reconcile; independent instances + produce the same identifier and byte-identical writeback. + +Exit codes (single-repo / local CLI): + 0 — clean (no FAILs or WARNs; INFOs are allowed) + 1 — one or more assessment FAILs or automation ERRORs (C-00) present + 2 — warnings-only strict CLI result (no FAILs, but WARNs present) + +Exit codes (--remote --all scheduled sweep): + 0 — automation completed and documented results (assessment failures OK) + 1 — automation error: API unreachable, repo list fetch failed, C-00 on + any repo, or other infrastructure fault that prevented a full run + +Assessment failures (C-01..C-24 except C-00) are repo hygiene gaps recorded +in the sweep report for later improvement. They do not fail the scheduler. + +Agent/operator Make wrappers normalize exit code 2 to shell success while +preserving visible warning output. Use the direct script when a machine caller +needs to distinguish clean from warnings-only. +""" +from __future__ import annotations + +import argparse +import os +import json +import re +import shutil +import socket +import subprocess +import sys +import time +import uuid +from contextlib import contextmanager +from dataclasses import dataclass, field +from collections import Counter +from datetime import date as _date, datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +_REPO_ROOT = Path(__file__).resolve().parent.parent +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +def _sweep_hostname() -> str: + """Hostname for host_paths resolution; override when pod hostname != fleet FQDN.""" + return os.environ.get("STATE_HUB_SWEEP_HOSTNAME", "").strip() or socket.gethostname() + + +from api.workplan_status import ( # noqa: E402 + CANONICAL_WORKSTREAM_STATUSES, + CLOSED_WORKSTREAM_STATUSES, + LEGACY_WORKSTREAM_STATUS_ALIASES, + OPEN_WORKSTREAM_STATUSES, + SUPPORTED_WORKSTREAM_STATUSES, + normalize_workstream_status as _normalize_workstream_status, + ready_review_status, +) +from api.classification import ( # noqa: E402 + CLASSIFICATION_FILENAME, + load_classification_file, + resolve_topic_domain_slug, +) +from api.services.lifecycle import should_activate_parent_for_active_tasks # noqa: E402 +from api.services.historical_projection import ( # noqa: E402 + BINDINGS as _HISTORICAL_BINDINGS, FROZEN_CLUSTER_BINDINGS, frozen_cluster_binding, +) +from api.task_status import ( # noqa: E402 + CANONICAL_TASK_STATUSES, + OPEN_TASK_STATUSES, + TASK_STATUS_ORDER, + TERMINAL_TASK_STATUSES, + normalize_task_status, +) +from api.work_record_flavor import ( # noqa: E402 + FLAVOR_PROMOTION_REASONS, + WORK_RECORD_FLAVORS, + is_residual_flavor, + normalize_flavor, + normalize_promotion_reason, +) + +_WORK_RECORD_NAMESPACE_UUID = uuid.UUID("a4058507-5c4a-5a00-ab06-fffa4fb46009") + + +def _derived_work_record_uuid(record_id: str) -> str: + return str(uuid.uuid5(_WORK_RECORD_NAMESPACE_UUID, f"helixforge\n{record_id}")) + +try: + import yaml as _yaml + _HAS_YAML = True +except ImportError: + _HAS_YAML = False + +try: + import httpx as _httpx + _HAS_HTTPX = True +except ImportError: + _HAS_HTTPX = False + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_LEGACY_METER_COMPONENT = "state-hub.fix-consistency" + + +def _legacy_meter_headers() -> dict[str, str]: + return {"X-StateHub-Component": _LEGACY_METER_COMPONENT} + + +_TASK_BLOCK_RE = re.compile(r"```task\s*\n(.*?)\n```", re.DOTALL) +_HEADING_RE = re.compile(r"^(#{1,4})\s+(.+?)$", re.MULTILINE) +_ARCHIVED_WP_RE = re.compile(r"^\d{6}-(.+\.md)$") +VALID_WP_STATUSES = set(CANONICAL_WORKSTREAM_STATUSES) +SUPPORTED_WP_STATUSES = set(SUPPORTED_WORKSTREAM_STATUSES) +VALID_TASK_STATUSES = set(CANONICAL_TASK_STATUSES) +VALID_TASK_PRIORITIES = {"low", "medium", "high", "critical"} +VALID_DEP_RELATIONSHIPS = {"blocks", "starts_after", "informs", "soft_dependency"} +DEFAULT_REMOTE_ALL_MAX_SECONDS = int(os.environ.get("CONSISTENCY_REMOTE_ALL_MAX_SECONDS", "300")) +STALE_UNREAD_DAYS = int(os.environ.get("INBOX_STALE_DAYS", "3")) +_API_GET_RETRIES = int(os.environ.get("CONSISTENCY_API_GET_RETRIES", "3")) +_API_GET_RETRY_BASE_DELAY = float(os.environ.get("CONSISTENCY_API_GET_RETRY_DELAY", "0.5")) + +_WP_FILE_PREFIX_RE = re.compile(r"^([A-Za-z][A-Za-z0-9-]*-WP)-\d+", re.IGNORECASE) +_WP_FILE_BARE_RE = re.compile(r"^(WP)-\d+", re.IGNORECASE) +_WP_ID_PREFIX_RE = re.compile(r"^([A-Z][A-Z0-9-]*-WP)-\d+", re.IGNORECASE) +_WP_ID_BARE_RE = re.compile(r"^(WP)-\d+", re.IGNORECASE) +_BLOCKED_ON_RE = re.compile(r"^message-from:(?P.+)$") +_WAIT_UNTIL_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +_WORK_REQUEST_RE = re.compile( + r"\b(implement|workplan|multi-?step|please\s+(implement|add|create|fix|build))\b", + re.IGNORECASE, +) +_OPEN_WORKPLAN_STATUSES = {"proposed", "ready", "active", "blocked", "backlog"} + + +def _check_repo_manager_conformance(repo_dir: Path, report: "ConsistencyReport") -> None: + """C-35: consume Repo Manager's conformance contract without reimplementing it.""" + configured = os.environ.get("RMGR_BIN", "").strip() + candidates = [configured] if configured else [] + candidates.extend( + [ + str(Path.home() / "repo-manager" / ".venv" / "bin" / "rmgr"), + shutil.which("rmgr") or "", + ] + ) + executable = next((item for item in candidates if item and Path(item).is_file()), None) + if executable is None: + report.add( + severity="INFO", + check_id="C-35", + message="Repo Manager conformance adapter unavailable; set RMGR_BIN or install rmgr", + fixable=False, + ) + return + try: + completed = subprocess.run( + [executable, "conform", "--path", str(repo_dir)], + capture_output=True, + text=True, + check=False, + timeout=30, + ) + payload = json.loads(completed.stdout) + except (OSError, subprocess.TimeoutExpired, json.JSONDecodeError) as exc: + report.add( + severity="INFO", + check_id="C-35", + message=f"Repo Manager conformance adapter could not run: {exc}", + fixable=False, + ) + return + for finding in payload.get("findings") or []: + severity = str(finding.get("severity") or "warning") + report.add( + severity="WARN" if severity in {"missing", "contradictory", "warning"} else "INFO", + check_id="C-35", + message=( + f"Repo Manager {severity}: {finding.get('path') or '(repository)'}: " + f"{finding.get('message') or finding.get('code') or 'conformance finding'}" + ), + file_path=finding.get("path"), + fixable=False, + ) + +# Legacy file/API aliases translated before comparison and PATCHing. +FILE_TO_DB_WORKSTREAM_STATUS: dict[str, str] = dict(LEGACY_WORKSTREAM_STATUS_ALIASES) + +# Ordinal ranking for task statuses used by the no-regress rule (T01/C-15). +STATUS_ORDER: dict[str, int] = dict(TASK_STATUS_ORDER) + + +def normalise_decision_type(meta: dict) -> str: + """Map a file's decision_type onto the hub's two values, made and pending. + + Repos record decided outcomes with words like ``approved``; the hub rejects + anything else with a 422, which left those decisions unindexed. + """ + value = str(meta.get("decision_type") or "").strip().lower() + if value in {"made", "pending"}: + return value + status = str(meta.get("status") or "").strip().lower() + if status == "resolved" or value in { + "approved", "accepted", "decided", "rejected", "declined", "resolved", + }: + return "made" + return "pending" + + +def historical_view_paths(repo_slug: str) -> set[str]: + """Repo-relative paths of bound historical views that share a parent row. + + A split view carries its parent's UUID but does not supply the parent's + fields; only the bound parent file does (see historical_projection). Letting + each view write the shared row made its status flip on every run. + """ + return { + entry["path"] + for entry in _HISTORICAL_BINDINGS + if entry["repo"] == repo_slug and entry["record_id"] != entry["parent_record_id"] + } + + +def normalise_workstream_status(status: str, *, has_started: bool | None = None) -> str: + """Translate a workplan file status value to its DB-canonical equivalent.""" + return _normalize_workstream_status(status, has_started=has_started) + + +def normalise_task_status(status: Any, *, default: str = "todo") -> str: + try: + return normalize_task_status(status, default=default) + except ValueError: + return default + + +def canonical_workplan_filename(path: Path) -> str: + """Return the workplan filename without an archive completion-date prefix.""" + return _ARCHIVED_WP_RE.sub(r"\1", path.name) + + +def workplan_display_path(repo_dir: Path, path: Path) -> str: + """Stable relative path for reports, including archived/ when applicable.""" + try: + return str(path.relative_to(repo_dir)) + except ValueError: + return path.name + + +def iter_workplan_files(workplans_dir: Path, include_archived: bool = True) -> list[Path]: + """Return active root workplans plus archived workplans when requested.""" + files = [ + path for path in sorted(workplans_dir.glob("*.md")) + if path.name not in _NON_WORKPLAN_WORKPLAN_FILES + ] + archived_dir = workplans_dir / "archived" + if include_archived and archived_dir.is_dir(): + files.extend(sorted(archived_dir.glob("*.md"))) + return files + + +# C-00 marks infrastructure/automation faults (API down, repo missing in DB). +# All other FAIL severities are assessment findings for follow-up. +_AUTOMATION_ERROR_CHECKS: frozenset[str] = frozenset({"C-00", "C-44"}) +_NON_WORKPLAN_WORKPLAN_FILES: frozenset[str] = frozenset({"README.md"}) + + +# --------------------------------------------------------------------------- +# Data types +# --------------------------------------------------------------------------- + +@dataclass +class Issue: + severity: str # FAIL | WARN | INFO + check_id: str # C-01 through C-12 + message: str + file_path: str = "" + db_id: str = "" + file_value: str = "" + db_value: str = "" + fixable: bool = False + _fix_context: dict = field(default_factory=dict, repr=False) + + +@dataclass +class ConsistencyReport: + repo_slug: str + repo_path: str + issues: list[Issue] = field(default_factory=list) + fixes_applied: list[str] = field(default_factory=list) + # Commit the workplans/ scan read (STATE-WP-0094): a full 40-hex SHA, or + # None with a machine-readable reason. Never a branch name or a guess. + source_revision: str | None = None + source_revision_reason: str | None = "not_scanned" + + def add(self, **kwargs) -> Issue: + issue = Issue(**kwargs) + self.issues.append(issue) + return issue + + @property + def failures(self) -> list[Issue]: + return [i for i in self.issues if i.severity == "FAIL"] + + @property + def automation_errors(self) -> list[Issue]: + return [ + i for i in self.issues + if i.severity == "FAIL" and i.check_id in _AUTOMATION_ERROR_CHECKS + ] + + @property + def assessment_failures(self) -> list[Issue]: + return [ + i for i in self.issues + if i.severity == "FAIL" and i.check_id not in _AUTOMATION_ERROR_CHECKS + ] + + @property + def warnings(self) -> list[Issue]: + return [i for i in self.issues if i.severity == "WARN"] + + @property + def infos(self) -> list[Issue]: + return [i for i in self.issues if i.severity == "INFO"] + + +@dataclass(frozen=True) +class RenormalizationRule: + check_id: str + invariant: str + detection: str + repair: str + test_anchor: str + + +RENORMALIZATION_RULES: tuple[RenormalizationRule, ...] = ( + RenormalizationRule( + check_id="C-23", + invariant="Planning-state workplans cannot contain active task work.", + detection=( + "Workplan status is proposed, ready, or backlog while a linked " + "task is progress or wait." + ), + repair=( + "ready/backlog: patch DB and file to active. " + "proposed: report only (ADR-007 decision 3)." + ), + test_anchor="tests/test_consistency_check.py::TestLifecycleRenormalization", + ), +) + + +RENORMALIZATION_NEXT_GUARD_CHECKLIST: tuple[str, ...] = ( + "Name the invariant and assign the next C-id.", + "Add rule metadata to RENORMALIZATION_RULES.", + "Add detection in check_repo before generic drift rules that could fight it.", + "Add the repair branch in fix_repo, using file writes only through helpers.", + "Add detection and repair tests under TestLifecycleRenormalization.", +) + + +@contextmanager +def run_lock(name: str): + """Hold a nonblocking process lock for long-running consistency modes.""" + try: + import fcntl + except ImportError: + yield True + return + + lock_path = Path(os.environ.get("CONSISTENCY_LOCK_DIR", "/tmp")) / f"custodian-{name}.lock" + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = lock_path.open("w", encoding="utf-8") + try: + try: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + yield False + return + handle.seek(0) + handle.truncate() + handle.write(f"{os.getpid()} {datetime.utcnow().isoformat()}Z\n") + handle.flush() + yield True + finally: + try: + fcntl.flock(handle, fcntl.LOCK_UN) + except OSError: + pass + handle.close() + + +# --------------------------------------------------------------------------- +# YAML / frontmatter parsing +# --------------------------------------------------------------------------- + +def _parse_yaml_block(raw: str) -> dict: + """Parse a YAML string; fallback to simple key:value if pyyaml unavailable.""" + if _HAS_YAML: + try: + return _yaml.safe_load(raw) or {} + except _yaml.YAMLError: + return {"_parse_error": True} + result: dict = {} + for line in raw.splitlines(): + if ":" in line and not line.startswith(" "): + k, _, v = line.partition(":") + result[k.strip()] = v.strip().strip('"').strip("'") + return result + + +def parse_frontmatter(text: str) -> tuple[dict, str]: + """Split YAML frontmatter from body. Returns ({}, text) if no frontmatter.""" + if not text.startswith("---"): + return {}, text + parts = text.split("---", 2) + if len(parts) < 3: + return {}, text + meta = _parse_yaml_block(parts[1].strip()) + return meta, parts[2] + + +def _clean_task_description(raw: str) -> str | None: + """Trim section chrome while preserving task markdown content.""" + lines = raw.splitlines() + while lines and not lines[0].strip(): + lines.pop(0) + while lines and not lines[-1].strip(): + lines.pop() + while lines and lines[-1].strip() in {"---", "***", "___"}: + lines.pop() + while lines and not lines[-1].strip(): + lines.pop() + while lines and lines[0].strip() in {"---", "***", "___"}: + lines.pop(0) + while lines and not lines[0].strip(): + lines.pop(0) + text = "\n".join(lines).strip() + return text or None + + +def parse_task_blocks(body: str) -> list[dict]: + """Extract task blocks, injecting title and markdown content from the task section.""" + headings = [ + (m.start(), len(m.group(1)), m.group(2).strip()) + for m in _HEADING_RE.finditer(body) + ] + task_matches = list(_TASK_BLOCK_RE.finditer(body)) + results = [] + for idx, m in enumerate(task_matches): + task = _parse_yaml_block(m.group(1).strip()) + prev = [(pos, level, text) for pos, level, text in headings if pos < m.start()] + prev_heading = prev[-1] if prev else None + if "title" not in task: + if prev_heading: + task["title"] = prev_heading[2] + + content_end = len(body) + if idx + 1 < len(task_matches): + content_end = min(content_end, task_matches[idx + 1].start()) + if prev_heading: + current_level = prev_heading[1] + next_peer_heading = next( + ( + pos + for pos, level, _text in headings + if pos > m.end() and level <= current_level + ), + None, + ) + if next_peer_heading is not None: + content_end = min(content_end, next_peer_heading) + else: + next_heading = next((pos for pos, _level, _text in headings if pos > m.end()), None) + if next_heading is not None: + content_end = min(content_end, next_heading) + + description = _clean_task_description(body[m.end():content_end]) + if description: + task["description"] = description + results.append(task) + return results + + +def get_tasks_from_workplan(meta: dict, body: str) -> list[dict]: + """Get tasks from workplan — handles both ```task``` blocks and tasks: YAML list.""" + blocks = parse_task_blocks(body) + if blocks: + return blocks + # Fallback: tasks embedded as a YAML list in frontmatter (activity-core style) + tasks = meta.get("tasks") + if isinstance(tasks, list): + return tasks + return [] + + +def _as_list(value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(item).strip().strip('"') for item in value if str(item).strip()] + if isinstance(value, str): + return [item.strip().strip('"') for item in value.split(",") if item.strip()] + return [str(value).strip().strip('"')] + + +_TASK_RECORD_ID_RE = re.compile(r"-T\d{2,3}$") + + +def _dedupe_preserve(items: list[str]) -> list[str]: + seen: set[str] = set() + out: list[str] = [] + for item in items: + if item in seen: + continue + seen.add(item) + out.append(item) + return out + + +def _frontmatter_depends_on_workplans(meta: dict) -> list[str]: + """Canonical `depends_on` plus legacy `depends_on_workplans` alias.""" + combined = _as_list(meta.get("depends_on")) + _as_list(meta.get("depends_on_workplans")) + return _dedupe_preserve( + [item for item in combined if item and not _TASK_RECORD_ID_RE.search(item)] + ) + + +def _frontmatter_depends_on_tasks(meta: dict) -> list[str]: + from_alias = _as_list(meta.get("depends_on_tasks")) + from_depends_on = [ + item for item in _as_list(meta.get("depends_on")) if _TASK_RECORD_ID_RE.search(item) + ] + return _dedupe_preserve(from_alias + from_depends_on) + + +_UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I) + + +def _as_bool(value: Any) -> bool: + if isinstance(value, bool): + return value + return str(value).strip().lower() in {"true", "yes", "1", "on"} + + +def _task_wait_until(task: dict) -> str | None: + """``wait_until`` as an ISO date string, or None when absent or not a + ``YYYY-MM-DD`` date (PyYAML hands a bare date over as ``datetime.date``).""" + raw = task.get("wait_until") + if raw in (None, ""): + return None + if isinstance(raw, datetime): + raw = raw.date() + if isinstance(raw, _date): + return raw.isoformat() + text = str(raw).strip().strip('"').strip("'") + if not _WAIT_UNTIL_RE.match(text): + return None + try: + return _date.fromisoformat(text).isoformat() + except ValueError: + return None + + +def _task_wait_fields(task: dict) -> dict[str, Any]: + """File → hub fields that qualify a wait (CUST-WP-0074): needs_human, + blocking_reason, decision_id, blocked_on, wait_until. TaskCreate/TaskUpdate + require an intervention_note whenever needs_human is set; the blocking + reason is by rule 2 what the human must do, so it stands in when the block + has none.""" + needs_human = _as_bool(task.get("needs_human")) + blocking_reason = str(task.get("blocking_reason") or "").strip() or None + decision_id = str(task.get("decision_id") or "").strip().strip('"') or None + blocked_on = str(task.get("blocked_on") or "").strip().strip('"') or None + fields: dict[str, Any] = { + "needs_human": needs_human, + "blocking_reason": blocking_reason, + "decision_id": decision_id, + "blocked_on": blocked_on, + "wait_until": _task_wait_until(task), + } + if needs_human: + fields["intervention_note"] = ( + str(task.get("intervention_note") or "").strip() or blocking_reason + ) + return fields + + +_TASK_WAIT_FIELD_KEYS = ("needs_human", "blocking_reason", "decision_id", "blocked_on", "wait_until") + + +def _dependency_target_satisfied(record_id: str, status: str) -> bool: + """Rule 3: task targets are satisfied at done/cancel, workplan targets at + finished/archived.""" + if _TASK_RECORD_ID_RE.search(record_id): + return normalise_task_status(status, default="") in {"done", "cancel"} + return normalise_workstream_status(status) in CLOSED_WORKSTREAM_STATUSES + + +def _repo_decision_hub_ids(repo_dir: Path) -> dict[str, str]: + """Map decision record ids (e.g. CCR-2026-0004) to the hub uuid C-32 wrote + back into their YAML block.""" + out: dict[str, str] = {} + for md in sorted(repo_dir.rglob("*.md")): + if any(part in _WORK_RECORD_SKIP_DIRS for part in md.parts): + continue + try: + text = md.read_text(errors="replace") + except OSError: + continue + if "state_hub_decision_id" not in text: + continue + for block in _YAML_ONLY_FENCE_RE.findall(text): + meta = _parse_yaml_block(block.strip()) + if not isinstance(meta, dict) or meta.get("_parse_error"): + continue + rid = str(meta.get("id", "")).strip() + hub_id = str(meta.get("state_hub_decision_id", "")).strip().strip('"') + if rid and hub_id and hub_id not in ("~", "null", "None", "none"): + out[rid] = hub_id + return out + + +def _as_int_or_none(value: Any) -> int | None: + if value in (None, "", "~", "null", "None", "none"): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +# --------------------------------------------------------------------------- +# File update helpers +# --------------------------------------------------------------------------- + +def _add_frontmatter_field(file_path: Path, key: str, value: str) -> None: + """Insert key: "value" into frontmatter before the closing --- line.""" + text = file_path.read_text(encoding="utf-8") + lines = text.split("\n") + close_idx = None + for i, line in enumerate(lines[1:], 1): + if line.strip() == "---": + close_idx = i + break + if close_idx is None: + return + lines.insert(close_idx, f'{key}: "{value}"') + file_path.write_text("\n".join(lines), encoding="utf-8") + + +def _patch_frontmatter_field(file_path: Path, key: str, value: str) -> bool: + """Update or insert a scalar frontmatter field without rewriting the file.""" + text = file_path.read_text(encoding="utf-8") + if not text.startswith("---"): + return False + lines = text.split("\n") + close_idx = None + for i, line in enumerate(lines[1:], 1): + if line.strip() == "---": + close_idx = i + break + if close_idx is None: + return False + + new_line = f"{key}: {value}" + for i in range(1, close_idx): + if re.match(rf"^\s*{re.escape(key)}\s*:", lines[i]): + if lines[i] == new_line: + return False + lines[i] = new_line + file_path.write_text("\n".join(lines), encoding="utf-8") + return True + + lines.insert(close_idx, new_line) + file_path.write_text("\n".join(lines), encoding="utf-8") + return True + + +def _inject_task_id_into_block( + file_path: Path, field_name: str, field_value: str, match_id: str +) -> bool: + """Inject state_hub_task_id into the ```task``` block whose id == match_id.""" + text = file_path.read_text(encoding="utf-8") + + def _replace(m: re.Match) -> str: + block_content = m.group(1) + task_meta = _parse_yaml_block(block_content.strip()) + if str(task_meta.get("id", "")) != match_id: + return m.group(0) + existing_val = task_meta.get(field_name) + if existing_val is not None and str(existing_val).strip() not in ("", "~", "null", "None", "none"): + return m.group(0) + # Replace existing null/~ line if present, otherwise append + new_content = re.sub( + rf"^{re.escape(field_name)}:.*$", + f'{field_name}: "{field_value}"', + block_content, + flags=re.MULTILINE, + ) + if new_content == block_content: + new_content = block_content.rstrip() + f"\n{field_name}: \"{field_value}\"" + return f"```task\n{new_content}\n```" + + new_text = _TASK_BLOCK_RE.sub(_replace, text) + if new_text != text: + file_path.write_text(new_text, encoding="utf-8") + return True + return False + + +def _inject_task_id_frontmatter_list( + file_path: Path, field_value: str, match_id: str +) -> bool: + """Inject state_hub_task_id into a task entry in frontmatter tasks: list.""" + if not _HAS_YAML: + return False + import yaml + text = file_path.read_text(encoding="utf-8") + meta, body = parse_frontmatter(text) + tasks = meta.get("tasks", []) + changed = False + for t in tasks: + if str(t.get("id", "")) == match_id and "state_hub_task_id" not in t: + t["state_hub_task_id"] = field_value + changed = True + if not changed: + return False + try: + new_meta_str = yaml.dump(meta, allow_unicode=True, default_flow_style=False) + parts = text.split("---", 2) + if len(parts) < 3: + return False + new_text = f"---\n{new_meta_str}---{parts[2]}" + file_path.write_text(new_text, encoding="utf-8") + return True + except Exception: + return False + + +# --------------------------------------------------------------------------- +# API helpers +# --------------------------------------------------------------------------- + +_API_CLIENTS: dict[tuple[str, object], Any] = {} + + +def _api_client(api_base: str) -> Any: + """Reuse one keep-alive pool per API/client implementation for this run.""" + key = (api_base.rstrip("/"), _httpx.Client) + client = _API_CLIENTS.get(key) + if client is None: + client = _httpx.Client( + base_url=api_base, + timeout=10.0, + follow_redirects=True, + headers=_legacy_meter_headers(), + ) + _API_CLIENTS[key] = client + return client + + +def _api_get( + api_base: str, + path: str, + params: dict | None = None, + *, + return_error: bool = False, +) -> Any: + if not _HAS_HTTPX: + return {"_error": "httpx is not installed"} if return_error else None + # Only append trailing slash to the path component, not to query strings + if "?" not in path and not path.endswith("/"): + path += "/" + filtered = {k: v for k, v in (params or {}).items() if v is not None} + last_error: Exception | None = None + for attempt in range(_API_GET_RETRIES): + try: + r = _api_client(api_base).get(path, params=filtered if filtered else None) + r.raise_for_status() + return r.json() + except _httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + return None + last_error = exc + if exc.response.status_code >= 500 and attempt < _API_GET_RETRIES - 1: + time.sleep(_API_GET_RETRY_BASE_DELAY * (attempt + 1)) + continue + if return_error: + return {"_error": str(exc)} + return None + except (_httpx.TimeoutException, _httpx.ConnectError, _httpx.NetworkError) as exc: + last_error = exc + if attempt < _API_GET_RETRIES - 1: + time.sleep(_API_GET_RETRY_BASE_DELAY * (attempt + 1)) + continue + if return_error: + return {"_error": str(exc)} + return None + except Exception as exc: + last_error = exc + if return_error: + return {"_error": str(exc)} + return None + if return_error and last_error is not None: + return {"_error": str(last_error)} + return None + + +def _api_patch(api_base: str, path: str, body: dict) -> Any: + if not _HAS_HTTPX: + return None + if not path.endswith("/"): + path += "/" + try: + r = _api_client(api_base).patch(path, json=body) + r.raise_for_status() + return r.json() + except Exception as exc: + # Return a sentinel dict so callers can distinguish "API error" from "success" + # and report it rather than silently dropping the fix. + return {"_error": str(exc)} + + +def _api_put(api_base: str, path: str, body: dict) -> Any: + if not _HAS_HTTPX: + return {"_error": "httpx is not installed"} + if not path.endswith("/"): + path += "/" + try: + r = _api_client(api_base).put(path, json=body, timeout=30.0) + r.raise_for_status() + return r.json() + except Exception as exc: + return {"_error": str(exc)} + + +_FROM_TASK_ID_SUPPORT: dict[str, bool] = {} + + +def _hub_accepts_from_task_id(api_base: str) -> bool: + """True when the hub's dependency-create schema declares from_task_id. + + Cached per api_base. An unreadable schema counts as unsupported so the + fixer never writes a task-scoped edge into a hub that would drop the + attribution silently. + """ + if api_base not in _FROM_TASK_ID_SUPPORT: + # Not via _api_get: it appends a trailing slash, which /openapi.json rejects. + try: + r = _api_client(api_base).get("/openapi.json") + spec = r.json() if r.status_code == 200 else {} + except Exception: + spec = {} + schemas = spec.get("components", {}).get("schemas", {}) if isinstance(spec, dict) else {} + _FROM_TASK_ID_SUPPORT[api_base] = any( + "from_task_id" in (schemas.get(name) or {}).get("properties", {}) + for name in ("WorkplanDependencyCreate", "WorkstreamDependencyCreate") + ) + return _FROM_TASK_ID_SUPPORT[api_base] + + +def _api_post(api_base: str, path: str, body: dict) -> Any: + if not _HAS_HTTPX: + return {"_error": "httpx is not installed"} + if not path.endswith("/"): + path += "/" + try: + r = _api_client(api_base).post(path, json=body) + r.raise_for_status() + return r.json() + except _httpx.HTTPStatusError as exc: + detail = exc.response.text + if len(detail) > 500: + detail = detail[:497] + "..." + return {"_error": f"{exc.response.status_code} {exc.response.reason_phrase}: {detail}"} + except Exception as exc: + return {"_error": str(exc)} + + +# --------------------------------------------------------------------------- +# Core check engine +# --------------------------------------------------------------------------- + +def resolve_repo_path(repo: dict, override: str | None = None) -> str: + """Resolve the local filesystem path for a repo on the current machine. + + Priority: + 1. Explicit --repo-path CLI override + 2. host_paths[current_hostname] — per-machine path registered via POST /repos/{slug}/paths/ + 3. local_path — legacy single-path field (backward compat) + """ + if override: + return override + hostname = _sweep_hostname() + host_paths = repo.get("host_paths") or {} + return host_paths.get(hostname) or repo.get("local_path") or "" + + +def _infer_slug_from_path(api_base: str, path: str) -> "tuple[str, str] | None": + """Identify a registered repo from a local checkout path. + + Strategy (in order): + 1. Root-commit fingerprint — ``git rev-list --max-parents=0 HEAD`` produces + the same SHA-1 on every clone, independent of remote URL or checkout path. + Looked up via ``GET /repos/by-fingerprint?hash=``. + 2. Remote URL fallback — exact string match against ``remote_url`` field. + Less reliable across machines (SSH aliases, HTTP vs HTTPS, etc.) but + useful when fingerprint is not yet stored. + + Returns ``(slug, git_root)`` on success, ``None`` if no match found. + """ + try: + git_root = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], + cwd=path, stderr=subprocess.DEVNULL, text=True, + ).strip() + except (subprocess.CalledProcessError, FileNotFoundError, OSError): + return None + + # Strategy 1: fingerprint lookup (most reliable) + try: + fingerprint = subprocess.check_output( + ["git", "rev-list", "--max-parents=0", "HEAD"], + cwd=git_root, stderr=subprocess.DEVNULL, text=True, + ).strip() + except (subprocess.CalledProcessError, FileNotFoundError, OSError): + fingerprint = "" + + # Get local remote URL once — used for both disambiguation and fallback + try: + remote_url = subprocess.check_output( + ["git", "remote", "get-url", "origin"], + cwd=git_root, stderr=subprocess.DEVNULL, text=True, + ).strip() + except (subprocess.CalledProcessError, FileNotFoundError, OSError): + remote_url = "" + + if fingerprint: + # Try fingerprint + remote URL for precise match + if remote_url: + import urllib.parse as _up + candidates = _api_get( + api_base, + f"/repos/by-fingerprint?hash={fingerprint}&remote_url={_up.quote(remote_url, safe='')}", + ) + if isinstance(candidates, list) and len(candidates) == 1: + return candidates[0]["slug"], git_root + + # Fingerprint alone (works when repos don't share ancestry) + candidates = _api_get(api_base, f"/repos/by-fingerprint?hash={fingerprint}") + if isinstance(candidates, list): + if len(candidates) == 1: + return candidates[0]["slug"], git_root + if len(candidates) > 1: + # Disambiguate: prefer the repo whose slug appears in the git_root path + for repo in candidates: + if repo["slug"] in git_root: + return repo["slug"], git_root + # Can't disambiguate — return first match with a warning + print( + f" WARNING: {len(candidates)} repos share fingerprint {fingerprint[:12]}… " + f"— using '{candidates[0]['slug']}'. " + "Set remote_url on each repo for accurate matching.", + file=sys.stderr, + ) + return candidates[0]["slug"], git_root + + # Strategy 2: remote URL exact match (fallback) + if remote_url: + import urllib.parse as _up + repo = _api_get(api_base, f"/repos/by-remote?url={_up.quote(remote_url, safe='')}") + if repo and isinstance(repo, dict) and "slug" in repo: + return repo["slug"], git_root + + return None + + +_WORK_RECORD_FENCE_RE = re.compile(r"```(?:yaml|task)\n(.*?)```", re.S) +_WORK_RECORD_ID_LINE_RE = re.compile(r"^id:\s*[\"']?([A-Za-z0-9_-]+)[\"']?\s*$", re.M) +_WORK_RECORD_ID_LIKE_RE = re.compile(r"^[A-Z][A-Z0-9]*(-[A-Z0-9]+)+$") +_WORK_RECORD_SKIP_DIRS = frozenset({".git", "node_modules", ".venv", "history"}) + + +def _load_work_record_kind_registry() -> list[dict] | None: + """Load [{kind, patterns}] from the canon work-record type registry. + + Location: $WORK_RECORD_REGISTRY, else ~/the-custodian/canon/standards/ + work-record-types.yaml. Returns None (checks skipped) when unavailable. + Kind-aware: shared by C-31 (flat sidetrack detection) and C-32 + (kind-aware registration — needs to know intake vs. decision vs. + engagement to pick the right hub entity). + """ + if not _HAS_YAML: + return None + candidates = [] + env = os.environ.get("WORK_RECORD_REGISTRY") + if env: + candidates.append(Path(env)) + candidates.append( + Path.home() / "the-custodian/canon/standards/work-record-types.yaml") + for path in candidates: + if not path.is_file(): + continue + try: + reg = _yaml.safe_load(path.read_text()) + kinds = [] + for kind in reg.get("kinds", []): + patterns = [re.compile(p) for p in kind.get("id_patterns", [])] + patterns += [re.compile(lp["pattern"]) + for lp in kind.get("legacy_patterns", [])] + kinds.append({"kind": kind["kind"], "patterns": patterns}) + return kinds + except Exception: + return None + return None + + +def _load_work_record_patterns() -> list[re.Pattern] | None: + """Flat id-pattern list across all kinds — used by C-31's sidetrack + detector, which only needs to know "is this id registered at all", + not which kind it belongs to.""" + kinds = _load_work_record_kind_registry() + if kinds is None: + return None + return [p for k in kinds for p in k["patterns"]] + + +def _check_unregistered_work_records(repo_dir: Path, report: "ConsistencyReport") -> None: + """C-31: warn on YAML-block ids matching no registered work-record kind. + + Detection only (CUST-WP-0060-T05); registration of non-workplan kinds is + a later stage. An unregistered id scheme is a coordination sidetrack that + will need manual reintegration — surface it the day it appears. + """ + patterns = _load_work_record_patterns() + if patterns is None: + return + seen: set[str] = set() + for md in sorted(repo_dir.rglob("*.md")): + if any(part in _WORK_RECORD_SKIP_DIRS for part in md.parts): + continue + try: + text = md.read_text(errors="replace") + except OSError: + continue + for body in _WORK_RECORD_FENCE_RE.findall(text): + for rid in _WORK_RECORD_ID_LINE_RE.findall(body): + if rid in seen or "NNN" in rid or rid.endswith("-TNN"): + continue + if not _WORK_RECORD_ID_LIKE_RE.match(rid): + continue + if any(p.match(rid) for p in patterns): + continue + seen.add(rid) + report.add( + severity="WARN", + check_id="C-31", + message=( + f"[{md.relative_to(repo_dir)}] id '{rid}' matches no " + f"kind in the canon work-record type registry — " + f"unregistered species become sidetracks needing " + f"manual reintegration (work-record-types_v0.1)" + ), + fixable=False, + ) + + +_YAML_ONLY_FENCE_RE = re.compile(r"```yaml\n(.*?)\n```", re.S) +_WORK_RECORD_ID_FIELD_BY_KIND = { + "intake": "state_hub_intake_id", + "decision": "state_hub_decision_id", + "engagement": "state_hub_engagement_id", +} +_WORK_RECORD_CREATE_ENDPOINT_BY_KIND = { + "intake": "/intakes", + "decision": "/decisions", +} + + +def _classify_work_record_kind(record_id: str, kind_registry: list[dict]) -> str | None: + for k in kind_registry: + if any(p.match(record_id) for p in k["patterns"]): + return k["kind"] + return None + + +def _inject_yaml_block_field( + file_path: Path, field_name: str, field_value: str, match_id: str +) -> bool: + """Inject a field into the ```yaml``` block whose id == match_id. + Mirrors _inject_task_id_into_block, adapted for fenced yaml blocks + (intake/decision/engagement records live as ```yaml, not ```task).""" + text = file_path.read_text(encoding="utf-8") + + def _replace(m: re.Match) -> str: + block_content = m.group(1) + meta = _parse_yaml_block(block_content.strip()) + if not isinstance(meta, dict) or str(meta.get("id", "")) != match_id: + return m.group(0) + existing_val = meta.get(field_name) + if existing_val is not None and str(existing_val).strip() not in ("", "~", "null", "None", "none"): + return m.group(0) + new_content = re.sub( + rf"^{re.escape(field_name)}:.*$", + f'{field_name}: "{field_value}"', + block_content, + flags=re.MULTILINE, + ) + if new_content == block_content: + new_content = block_content.rstrip() + f"\n{field_name}: \"{field_value}\"" + return f"```yaml\n{new_content}\n```" + + new_text = _YAML_ONLY_FENCE_RE.sub(_replace, text) + if new_text != text: + file_path.write_text(new_text, encoding="utf-8") + return True + return False + + +def _check_work_record_registration( + repo_dir: Path, + report: "ConsistencyReport", + api_base: str = "http://127.0.0.1:8000", + bootstrap_empty_projection: bool = False, +) -> None: + """C-32: register kind: intake / kind: decision YAML blocks found + anywhere in the repo against the hub (CUST-WP-0061-T02, work-record + stage 3) — writes state_hub_intake_id / state_hub_decision_id back into + the source block, same write-back pattern as C-06 for workplans. + + kind: engagement has no hub entity yet (deferred; reported INFO, not + fixable) — building one is a separate stage-3 follow-on, not this task. + """ + kind_registry = _load_work_record_kind_registry() + if kind_registry is None: + return + for md in sorted(repo_dir.rglob("*.md")): + if any(part in _WORK_RECORD_SKIP_DIRS for part in md.parts): + continue + try: + text = md.read_text(errors="replace") + except OSError: + continue + for block in _YAML_ONLY_FENCE_RE.findall(text): + meta = _parse_yaml_block(block.strip()) + if not isinstance(meta, dict) or meta.get("_parse_error"): + continue + rid = str(meta.get("id", "")).strip() + if not rid or "NNN" in rid: + continue + kind = _classify_work_record_kind(rid, kind_registry) + if kind not in _WORK_RECORD_ID_FIELD_BY_KIND: + continue + + id_field = _WORK_RECORD_ID_FIELD_BY_KIND[kind] + existing = str(meta.get(id_field, "")).strip().strip('"') + if existing and existing not in ("~", "null", "None", "none"): + if not bootstrap_empty_projection: + continue + endpoint = _WORK_RECORD_CREATE_ENDPOINT_BY_KIND.get(kind) + if endpoint and _api_get(api_base, f"{endpoint}/{existing}") is not None: + continue + + rel = md.relative_to(repo_dir) + if kind not in _WORK_RECORD_CREATE_ENDPOINT_BY_KIND: + report.add( + severity="INFO", check_id="C-32", + message=( + f"[{rel}] engagement '{rid}' has no hub entity yet — " + f"registration deferred (CUST-WP-0061 stage-3 seed)" + ), + fixable=False, + ) + continue + + report.add( + severity="WARN", check_id="C-32", + message=( + f"[{rel}] {kind} '{rid}' is absent from the explicitly empty projection" + if existing + else f"[{rel}] {kind} '{rid}' has no {id_field} — not indexed in DB" + ), + file_path=str(rel), + file_value=rid, + fixable=True, + _fix_context={ + "md_path": md, + "kind": kind, + "rid": rid, + "meta": meta, + "desired_record_id": existing or None, + }, + ) + + +_WORK_RECORD_INDEX_NAME = "WORK-RECORDS.md" +_WORK_RECORD_INDEX_KIND_ORDER = { + "workplan": 0, "task": 1, "intake": 2, "decision": 3, "engagement": 4, +} + + +def _generate_work_record_index(repo_dir: Path, repo_slug: str) -> str | None: + """CUST-WP-0061-T04: the generated per-repo work-record index — every + registered work record (workplan, task, intake, decision, engagement) + with kind/id/status/lane/source, derivable purely from repo files (no + hub query — matches the ADR-001 rebuild principle). Full transclusion + rendering (markitect integration) is a follow-on; this is the flat + first cut. Returns None (no file written) when the repo has no work + records at all, to avoid clutter in small repos. + """ + rows: list[tuple[str, str, str, str, str]] = [] # kind, id, status, lane, source + + workplans_dir = repo_dir / "workplans" + if workplans_dir.is_dir(): + for wp_file in iter_workplan_files(workplans_dir, include_archived=False): + try: + text = wp_file.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + meta, body = parse_frontmatter(text) + wp_id = str(meta.get("id", "")).strip() + if not wp_id: + continue + source = str(wp_file.relative_to(repo_dir)) + rows.append(("workplan", wp_id, str(meta.get("status", "")).strip() or "—", "—", source)) + for task in get_tasks_from_workplan(meta, body): + if task.get("_parse_error"): + continue + t_id = str(task.get("id", "")).strip() + if not t_id: + continue + rows.append(("task", t_id, str(task.get("status", "")).strip() or "—", "—", source)) + + kind_registry = _load_work_record_kind_registry() + if kind_registry: + for md in sorted(repo_dir.rglob("*.md")): + if any(part in _WORK_RECORD_SKIP_DIRS for part in md.parts): + continue + if md.name == _WORK_RECORD_INDEX_NAME: + continue + try: + text = md.read_text(errors="replace") + except OSError: + continue + for block in _YAML_ONLY_FENCE_RE.findall(text): + meta = _parse_yaml_block(block.strip()) + if not isinstance(meta, dict) or meta.get("_parse_error"): + continue + rid = str(meta.get("id", "")).strip() + if not rid or "NNN" in rid: + continue + kind = _classify_work_record_kind(rid, kind_registry) + if kind not in ("intake", "decision", "engagement"): + continue + status = str(meta.get("status", "")).strip() or "—" + lane = str(meta.get("lane", "")).strip() or "—" + source = str(md.relative_to(repo_dir)) + rows.append((kind, rid, status, lane, source)) + + if not rows: + return None + + rows.sort(key=lambda r: (_WORK_RECORD_INDEX_KIND_ORDER.get(r[0], 99), r[4], r[1])) + return _work_record_index_document(repo_slug, rows) + + +def _work_record_index_document( + repo_slug: str, rows: list[tuple[str, str, str, str, str]] +) -> str: + """Render WORK-RECORDS.md. An empty row list still produces the header + and table so a leftover index can be rewritten after every live record + is archived or removed. + """ + lines = [ + f"# Work Records — {repo_slug}", + "", + "> Generated by `statehub fix-consistency` (CUST-WP-0061-T04, work-record", + "> stage 3). Do not edit by hand — edit the source file/block listed for", + "> each record and re-run fix-consistency to refresh this index. Archived", + "> workplans are omitted; closed decisions/intakes/engagements stay listed", + "> so recently-resolved work is still visible. [auto]", + "", + "| Kind | ID | Status | Lane | Source |", + "| --- | --- | --- | --- | --- |", + ] + lines.extend( + f"| {kind} | {rid} | {status} | {lane} | {source} |" + for kind, rid, status, lane, source in rows + ) + lines.append("") + return "\n".join(lines) + + +def _check_work_record_index_freshness( + repo_dir: Path, repo_slug: str, report: "ConsistencyReport" +) -> None: + """C-33: WORK-RECORDS.md missing or stale relative to the current + file-derived work-record set.""" + content = _generate_work_record_index(repo_dir, repo_slug) + index_file = repo_dir / _WORK_RECORD_INDEX_NAME + existing = index_file.read_text(encoding="utf-8") if index_file.is_file() else None + if content is None: + if existing is None: + return + empty = _work_record_index_document(repo_slug, []) + if existing == empty: + return + content = empty + elif existing == content: + return + report.add( + severity="WARN", check_id="C-33", + message=( + f"{_WORK_RECORD_INDEX_NAME} is missing or stale — regenerate via " + f"fix-consistency (CUST-WP-0061-T04 generated work-record index)" + ), + file_path=_WORK_RECORD_INDEX_NAME, + fixable=True, + _fix_context={"index_file": index_file, "content": content}, + ) + + +def check_repo( + api_base: str, + repo_slug: str, + repo_path_override: str | None = None, + bootstrap_empty_projection: bool = False, +) -> ConsistencyReport: + """Run all consistency checks for a registered repo.""" + repo = _api_get(api_base, f"/repos/{repo_slug}", return_error=True) + if isinstance(repo, dict) and "_error" in repo: + report = ConsistencyReport(repo_slug=repo_slug, repo_path=repo_path_override or "(unknown)") + report.add( + severity="FAIL", check_id="C-00", + message=( + f"Could not query State Hub API at {api_base}: {repo['_error']}" + ), + fixable=False, + ) + return report + if repo is None: + report = ConsistencyReport(repo_slug=repo_slug, repo_path="(unknown)") + report.add( + severity="FAIL", check_id="C-00", + message=f"Repo '{repo_slug}' not found in state-hub DB", + fixable=False, + ) + return report + + # Repository aliases are accepted at the API boundary, but every new + # binding, inbox write, and generated file must use the current identity. + repo_slug = str(repo.get("canonical_slug") or repo.get("slug") or repo_slug) + repo_id: str = repo["id"] + repo_path: str = resolve_repo_path(repo, repo_path_override) + report = ConsistencyReport(repo_slug=repo_slug, repo_path=repo_path) + + if bootstrap_empty_projection: + projected = _api_get(api_base, "/workplans", {"repo_id": repo_id}, return_error=True) + if not isinstance(projected, list): + report.add( + severity="FAIL", + check_id="C-36", + message="Could not prove that the repository projection is empty", + fixable=False, + ) + return report + if projected: + report.add( + severity="FAIL", + check_id="C-36", + message=( + "Empty-projection bootstrap refused: repository already has " + f"{len(projected)} projected workplan(s)" + ), + fixable=False, + ) + return report + + if not repo_path: + report.add( + severity="FAIL", check_id="C-00", + message=f"Repo '{repo_slug}' has no local_path — cannot check workplan files", + fixable=False, + ) + _check_orphan_db(api_base, repo_id, set(), report) + return report + + repo_dir = Path(repo_path) + workplans_dir = repo_dir / "workplans" + repo_market_domain = str(repo.get("domain_slug") or "").strip() + + # C-24: repo classification file missing or invalid (always WARN — migration rows too) + class_data, class_errors, class_warnings = load_classification_file(repo_dir) + if class_data is None: + classified_by = str(repo.get("classified_by") or "").strip() + if class_errors: + detail = "; ".join(class_errors) + else: + detail = f"{CLASSIFICATION_FILENAME} missing on disk" + if classified_by == "migration": + detail = f"{detail} (DB row is migration-derived — commit a human-reviewed file when ready)" + report.add( + severity="WARN", + check_id="C-24", + message=f"Repo classification gap: {detail}", + fixable=False, + ) + for warning in class_warnings: + report.add( + severity="WARN", + check_id="C-24", + message=f"Repo classification advisory: {warning}", + fixable=False, + ) + + # C-35: consume the repository-standard answer from Repo Manager. + _check_repo_manager_conformance(repo_dir, report) + + # C-31: work-record sidetrack detector (canon work-record-types registry) + _check_unregistered_work_records(repo_dir, report) + + # C-32: intake/decision work-record registration (CUST-WP-0061-T02) + _check_work_record_registration( + repo_dir, + report, + api_base=api_base, + bootstrap_empty_projection=bootstrap_empty_projection, + ) + + # C-33: generated per-repo work-record index (CUST-WP-0061-T04) + _check_work_record_index_freshness(repo_dir, repo_slug, report) + + # C-01: workplans/ directory missing + if not workplans_dir.is_dir(): + report.add( + severity="FAIL", check_id="C-01", + message=( + "workplans/ directory missing — ADR-001 requires workplan files " + "at /workplans/-.md" + ), + fixable=False, + ) + _check_orphan_db(api_base, repo_id, set(), report) + return report + + # Bind the scan to the commit it reads; re-verified after C-25 below. + report.source_revision, report.source_revision_reason = scan_source_revision(repo_dir) + + # Parse workplan files + workplan_infos: list[tuple[Path, dict, str]] = [] + file_ws_ids: dict[str, tuple[Path, dict, str]] = {} # ws_id → (file, meta, body) + active_file_ws_ids: set[str] = set() + + for wp_file in iter_workplan_files(workplans_dir): + try: + text = wp_file.read_text(encoding="utf-8") + except OSError as e: + report.add(severity="FAIL", check_id="C-02", + message=f"Cannot read file: {e}", file_path=workplan_display_path(repo_dir, wp_file)) + continue + if not text.startswith("---"): + report.add(severity="FAIL", check_id="C-02", + message="No YAML frontmatter found", file_path=workplan_display_path(repo_dir, wp_file)) + continue + meta, body = parse_frontmatter(text) + if not meta or meta.get("_parse_error"): + report.add(severity="FAIL", check_id="C-02", + message="YAML frontmatter parse error", file_path=workplan_display_path(repo_dir, wp_file)) + continue + workplan_infos.append((wp_file, meta, body)) + ws_id = str(meta.get("state_hub_workstream_id", "")).strip().strip('"') + if ws_id: + file_ws_ids[ws_id] = (wp_file, meta, body) + if wp_file.parent == workplans_dir: + active_file_ws_ids.add(ws_id) + + workplan_id_to_ws_id: dict[str, str] = {} + task_file_id_to_sh_id: dict[str, str] = {} + for _wp_file, meta, body in workplan_infos: + mapped_ws_id = str(meta.get("state_hub_workstream_id", "")).strip().strip('"') + wp_id = str(meta.get("id", "")).strip() + if wp_id and mapped_ws_id: + workplan_id_to_ws_id[wp_id] = mapped_ws_id + for task in get_tasks_from_workplan(meta, body): + if task.get("_parse_error"): + continue + task_file_id = str(task.get("id", "")).strip() + raw_sh = task.get("state_hub_task_id") + task_sh_id = "" if raw_sh is None else str(raw_sh).strip().strip('"') + if task_file_id and task_sh_id and task_sh_id not in ("~", "null", "None", "none"): + task_file_id_to_sh_id[task_file_id] = task_sh_id + + # Modern hubs expose the entire repo-scoped read model in one response. + # Keep the older per-record calls as a rolling-deploy compatibility path. + projection_snapshot = _api_get( + api_base, + f"/repos/{repo_slug}/work-record-projection/snapshot", + return_error=True, + ) + snapshot_available = ( + isinstance(projection_snapshot, dict) + and projection_snapshot.get("schema") + == "state-hub.repository-projection-snapshot.v1" + and str(projection_snapshot.get("repo_id")) == repo_id + ) + snapshot_workplans_by_id: dict[str, dict] = {} + snapshot_tasks_by_workplan: dict[str, list[dict]] = {} + snapshot_dependencies_by_workplan: dict[str, list[dict]] = {} + if snapshot_available: + for row in projection_snapshot.get("workplans", []): + if isinstance(row, dict) and row.get("id"): + snapshot_workplans_by_id[str(row["id"])] = row + for row in projection_snapshot.get("tasks", []): + if isinstance(row, dict) and row.get("workplan_id"): + snapshot_tasks_by_workplan.setdefault( + str(row["workplan_id"]), [] + ).append(row) + for row in projection_snapshot.get("dependencies", []): + if isinstance(row, dict) and row.get("from_workplan_id"): + snapshot_dependencies_by_workplan.setdefault( + str(row["from_workplan_id"]), [] + ).append(row) + snapshot_tasks_by_id: dict[str, dict] = { + str(row["id"]): row + for rows in snapshot_tasks_by_workplan.values() + for row in rows + if row.get("id") + } + + # Task-block depends_on targets (CUST-WP-0074) may live in other repos, so + # resolve them by the file mapping first and the ADR-007 derived uuid second. + dependency_target_cache: dict[str, tuple[str | None, str | None]] = {} + + def _resolve_dependency_target(record_id: str) -> tuple[str | None, str | None]: + """Return (hub uuid, hub status) for a workplan/task record id, or (None, None).""" + if record_id in dependency_target_cache: + return dependency_target_cache[record_id] + is_task = bool(_TASK_RECORD_ID_RE.search(record_id)) + mapped = (task_file_id_to_sh_id if is_task else workplan_id_to_ws_id).get(record_id) + hub_id = mapped or _derived_work_record_uuid(record_id) + row = (snapshot_tasks_by_id if is_task else snapshot_workplans_by_id).get(hub_id) + if row is None: + row = _api_get(api_base, f"/{'tasks' if is_task else 'workplans'}/{hub_id}") + result: tuple[str | None, str | None] = ( + (hub_id, str(row.get("status") or "")) if isinstance(row, dict) else (None, None) + ) + dependency_target_cache[record_id] = result + return result + + decision_hub_ids: dict[str, str] | None = None + + def _resolve_decision(decision_id: str) -> dict | None: + nonlocal decision_hub_ids + if decision_hub_ids is None: + decision_hub_ids = _repo_decision_hub_ids(repo_dir) + hub_id = ( + decision_id + if _UUID_RE.match(decision_id) + else decision_hub_ids.get(decision_id) or _derived_work_record_uuid(decision_id) + ) + row = _api_get(api_base, f"/decisions/{hub_id}") + return row if isinstance(row, dict) else None + + # Per-workplan checks + for wp_file, meta, body in workplan_infos: + fname = workplan_display_path(repo_dir, wp_file) + archived_file = wp_file.parent.name == "archived" + ws_id = str(meta.get("state_hub_workstream_id", "")).strip().strip('"') + file_status = str(meta.get("status", "")).strip() + file_title = str(meta.get("title", "")).strip() + file_domain = str(meta.get("domain", "")).strip() + + normalised_file_status = normalise_workstream_status(file_status) + if archived_file and normalised_file_status not in CLOSED_WORKSTREAM_STATUSES: + report.add( + severity="FAIL", check_id="C-18", + message="Archived workplan file has an open or planning status", + file_path=fname, + file_value=file_status, + fixable=False, + ) + + if not ws_id: + # C-06: workplan not linked to any DB workstream + report.add( + severity="WARN", check_id="C-06", + message="Workplan has no state_hub_workstream_id — not indexed in DB", + file_path=fname, + file_value=file_title or fname, + fixable=True, + _fix_context={ + "wp_file": str(wp_file), + "meta": meta, + "body": body, + "repo_id": repo_id, + "domain": file_domain, + "repo_market_domain": repo_market_domain, + "repo_slug": repo_slug, + }, + ) + continue + + ws = ( + snapshot_workplans_by_id.get(ws_id) + if snapshot_available + else _api_get(api_base, f"/workplans/{ws_id}") + ) + if ws is None: + wp_id = str(meta.get("id", "")).strip() + if wp_id and ( + ws_id == _derived_work_record_uuid(wp_id) or bootstrap_empty_projection + ): + # A deterministic file identifier missing from a replaceable + # projection is registration work, not a stale reference. + report.add( + severity="WARN", + check_id="C-06", + message=( + f"Authoritative workplan {ws_id[:8]}… is absent from this " + + ( + "explicitly empty projection" + if bootstrap_empty_projection + else "projection" + ) + ), + file_path=fname, + db_id=ws_id, + fixable=True, + _fix_context={ + "wp_file": str(wp_file), + "meta": meta, + "body": body, + "repo_id": repo_id, + "domain": file_domain, + "repo_market_domain": repo_market_domain, + "repo_slug": repo_slug, + "desired_ws_id": ws_id, + }, + ) + else: + # A non-derived missing identifier may be genuine stale state; + # preserve the conservative manual-review behavior. + report.add( + severity="FAIL", check_id="C-03", + message=f"state_hub_workstream_id {ws_id[:8]}… not found in DB (stale reference)", + file_path=fname, + db_id=ws_id, + fixable=False, + ) + continue + + # C-09: repo mismatch — file is here but DB says different repo + db_repo_id: str = ws.get("repo_id") or "" + if db_repo_id != repo_id: + report.add( + severity="FAIL", check_id="C-09", + message=( + f"Workstream '{ws.get('slug')}' repo_id in DB is " + f"{db_repo_id[:8] if db_repo_id else 'None'} " + f"but backing file lives in {repo_slug} ({repo_id[:8]}…)" + ), + file_path=fname, + db_id=ws_id, + file_value=repo_id, + db_value=db_repo_id, + fixable=True, + _fix_context={"ws_id": ws_id, "correct_repo_id": repo_id}, + ) + # Continue to check drift even with mismatched repo + + tasks = get_tasks_from_workplan(meta, body) + db_tasks = ( + snapshot_tasks_by_workplan.get(ws_id, []) + if snapshot_available + else _api_get(api_base, "/tasks", {"workplan_id": ws_id}) + ) + db_tasks_by_id = { + str(row.get("id")): row + for row in db_tasks + if isinstance(row, dict) and row.get("id") + } if isinstance(db_tasks, list) else {} + file_task_statuses = [ + str(task.get("status", "")).strip() + for task in tasks + if not task.get("_parse_error") + ] + db_task_statuses = [ + str(task.get("status", "")).strip() + for task in db_tasks + if isinstance(db_tasks, list) + ] if isinstance(db_tasks, list) else [] + active_task_requires_activation = should_activate_parent_for_active_tasks( + parent_workstream_status=file_status, + task_statuses=[*file_task_statuses, *db_task_statuses], + ) + if active_task_requires_activation: + # ADR-007 decision 3 / RMGR-WP-0005-T06: automation may report + # that a proposed workplan has started work, but must not promote + # proposed → active. proposed is a human review gate. + proposed_gate = file_status == "proposed" + report.add( + severity="WARN", + check_id="C-23", + message=( + f"Lifecycle drift in '{ws.get('slug')}': workplan status " + f"{file_status!r} has an active task" + + ( + " — report only (ADR-007: do not auto-promote proposed)" + if proposed_gate + else " — repair to 'active'" + ) + ), + file_path=fname, + db_id=ws_id, + file_value=file_status, + db_value=ws.get("status", ""), + fixable=not proposed_gate, + _fix_context={ + "ws_id": ws_id, + "wp_file": str(wp_file), + "file_status": file_status, + "db_status": ws.get("status", ""), + "target_status": "active", + }, + ) + + # C-42: a blocked workplan must say what it waits for — at least one + # task in `wait` (which C-39 then requires to be qualified). Blocked + # with no wait task is a stale status, not a block (CUST-WP-0074-T06). + if file_status == "blocked" and not any( + normalise_task_status(s) == "wait" for s in file_task_statuses + ): + report.add( + severity="WARN", + check_id="C-42", + message=( + f"Blocked workplan '{ws.get('slug')}' has no task in status " + f"wait — nothing states what it waits for; qualify a wait " + f"task or re-status the workplan" + ), + file_path=fname, + db_id=ws_id, + file_value=file_status, + fixable=False, + ) + + # A bound historical split view shares its parent's row but supplies + # none of its fields or edges; only the bound parent file does. + historical_view = workplan_display_path(repo_dir, wp_file) in historical_view_paths(repo_slug) + + # C-04: status drift — normalise file value before comparing so that + # legacy file/API aliases are not treated as drift. Lifecycle repairs + # take precedence so a stale planning file cannot regress active work. + db_status = ws.get("status", "") + normalised_db_status = normalise_workstream_status(db_status) + if ( + not active_task_requires_activation + and file_status + and db_status + and normalised_file_status != normalised_db_status + and not historical_view + ): + report.add( + severity="WARN", check_id="C-04", + message=( + f"Status drift in '{ws.get('slug')}': " + f"file={file_status!r} db={db_status!r} (file wins)" + ), + file_path=fname, + db_id=ws_id, + file_value=file_status, + db_value=db_status, + fixable=True, + _fix_context={ + "ws_id": ws_id, + "field": "status", + "value": normalised_file_status, # always send DB-valid value + }, + ) + + if normalised_file_status == "ready": + review = ready_review_status( + repo_dir, + meta.get("reviewed_against_commit"), + meta.get("context_paths"), + ) + if review.needs_review: + detail = f"Ready workplan may be stale: {review.reason}" + if review.changed_paths: + preview = ", ".join(review.changed_paths[:5]) + extra = "" if len(review.changed_paths) <= 5 else ", ..." + detail = f"{detail}; changed paths: {preview}{extra}" + report.add( + severity="WARN", + check_id="C-21", + message=detail, + file_path=fname, + file_value=file_status, + db_value="needs_review", + fixable=False, + ) + # C-34: soft DoR quality debt (STATE-WP-0077) — never blocks + try: + from quality_assessment import assessment_from_mapping, is_ok + except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from quality_assessment import assessment_from_mapping, is_ok # type: ignore + dor = assessment_from_mapping(meta, "DoR") + if not is_ok(dor): + report.add( + severity="WARN", + check_id="C-34", + message=( + "status=ready without quality_dor DoR-Ok — process claim " + "without DoR assessment (soft; see docs/work-record-quality-gates.md)" + ), + file_path=fname, + file_value=str(dor or "unassessed"), + fixable=False, + ) + + # C-05: title drift + db_title = ws.get("title", "") + if file_title and db_title and file_title != db_title: + report.add( + severity="WARN", check_id="C-05", + message=( + f"Title drift in '{ws.get('slug')}': " + f"file={file_title!r} db={db_title!r} (file wins)" + ), + file_path=fname, + db_id=ws_id, + file_value=file_title, + db_value=db_title, + fixable=True, + _fix_context={"ws_id": ws_id, "field": "title", "value": file_title}, + ) + + planning_priority = str(meta.get("planning_priority", "")).strip() or None + if planning_priority != (ws.get("planning_priority") or None): + report.add( + severity="WARN", check_id="C-19", + message=( + f"Planning priority drift in '{ws.get('slug')}': " + f"file={planning_priority!r} db={ws.get('planning_priority')!r} (file wins)" + ), + file_path=fname, + db_id=ws_id, + file_value=planning_priority, + db_value=ws.get("planning_priority"), + fixable=True, + _fix_context={"ws_id": ws_id, "field": "planning_priority", "value": planning_priority}, + ) + + planning_order = _as_int_or_none(meta.get("planning_order")) + if planning_order != ws.get("planning_order"): + report.add( + severity="WARN", check_id="C-19", + message=( + f"Planning order drift in '{ws.get('slug')}': " + f"file={planning_order!r} db={ws.get('planning_order')!r} (file wins)" + ), + file_path=fname, + db_id=ws_id, + file_value=planning_order, + db_value=ws.get("planning_order"), + fixable=True, + _fix_context={"ws_id": ws_id, "field": "planning_order", "value": planning_order}, + ) + + file_flavor = normalize_flavor(meta.get("flavor")) + if file_flavor is not None and file_flavor not in WORK_RECORD_FLAVORS: + report.add( + severity="WARN", + check_id="C-36", + message=( + f"Unknown work-record flavor {file_flavor!r} in '{ws.get('slug')}'; " + f"expected one of {', '.join(WORK_RECORD_FLAVORS)}" + ), + file_path=fname, + db_id=ws_id, + file_value=file_flavor, + fixable=False, + ) + file_flavor = None + db_flavor = normalize_flavor(ws.get("flavor")) + if file_flavor != db_flavor: + report.add( + severity="WARN", check_id="C-19", + message=( + f"Flavor drift in '{ws.get('slug')}': " + f"file={file_flavor!r} db={db_flavor!r} (file wins)" + ), + file_path=fname, + db_id=ws_id, + file_value=file_flavor, + db_value=db_flavor, + fixable=True, + _fix_context={"ws_id": ws_id, "field": "flavor", "value": file_flavor}, + ) + file_reason = normalize_promotion_reason(meta.get("flavor_promotion_reason")) + if file_reason is not None and file_reason not in FLAVOR_PROMOTION_REASONS: + report.add( + severity="WARN", + check_id="C-36", + message=( + f"Unknown flavor_promotion_reason {file_reason!r} in '{ws.get('slug')}'; " + f"expected one of {', '.join(FLAVOR_PROMOTION_REASONS)}" + ), + file_path=fname, + db_id=ws_id, + file_value=file_reason, + fixable=False, + ) + file_reason = None + db_reason = normalize_promotion_reason(ws.get("flavor_promotion_reason")) + if file_reason != db_reason: + report.add( + severity="WARN", check_id="C-19", + message=( + f"Flavor promotion reason drift in '{ws.get('slug')}': " + f"file={file_reason!r} db={db_reason!r} (file wins)" + ), + file_path=fname, + db_id=ws_id, + file_value=file_reason, + db_value=db_reason, + fixable=True, + _fix_context={ + "ws_id": ws_id, + "field": "flavor_promotion_reason", + "value": file_reason, + }, + ) + if is_residual_flavor(file_flavor): + origin = str(meta.get("origin") or "").strip() + origin_ref = str(meta.get("origin_ref") or "").strip() + if not origin and not origin_ref: + report.add( + severity="WARN", + check_id="C-37", + message=( + f"Residual workplan '{ws.get('slug')}' has no origin/origin_ref " + "provenance (not invented)" + ), + file_path=fname, + db_id=ws_id, + fixable=False, + ) + + # C-10, C-11, C-12: task-level checks + db_task_by_id: dict[str, dict] = {} + if isinstance(db_tasks, list): + for t in db_tasks: + db_task_by_id[t["id"]] = t + + existing_deps = ( + snapshot_dependencies_by_workplan.get(ws_id, []) + if snapshot_available + else _api_get(api_base, f"/workplans/{ws_id}/dependencies") or [] + ) + existing_dep_keys = set() + if isinstance(existing_deps, list): + for dep in existing_deps: + from_id = dep.get("from_workstream_id") or dep.get("from_workplan_id") + if from_id != ws_id: + continue + rel = dep.get("relationship_type") or "blocks" + # Frontmatter edges carry no from_task_id; task-block edges do. + from_task_id = dep.get("from_task_id") or None + to_workplan_id = dep.get("to_workstream_id") or dep.get("to_workplan_id") + if to_workplan_id: + existing_dep_keys.add((from_task_id, "workstream", to_workplan_id, rel)) + if dep.get("to_task_id"): + existing_dep_keys.add((from_task_id, "task", dep["to_task_id"], rel)) + + for target_wp_id in ([] if historical_view else _frontmatter_depends_on_workplans(meta)): + target_ws_id = workplan_id_to_ws_id.get(target_wp_id) + if not target_ws_id: + report.add( + severity="WARN", + check_id="C-20", + message=f"Workplan dependency target '{target_wp_id}' is not linked to State Hub", + file_path=fname, + file_value=target_wp_id, + fixable=False, + ) + continue + dep_key = (None, "workstream", target_ws_id, "blocks") + if dep_key not in existing_dep_keys: + report.add( + severity="WARN", + check_id="C-20", + message=f"Missing DB dependency edge: {ws_id[:8]}… depends on workplan {target_wp_id}", + file_path=fname, + db_id=ws_id, + file_value=target_wp_id, + fixable=True, + _fix_context={ + "from_workstream_id": ws_id, + "to_workstream_id": target_ws_id, + "relationship_type": "blocks", + }, + ) + + for target_task_id in _frontmatter_depends_on_tasks(meta): + target_sh_id = task_file_id_to_sh_id.get(target_task_id) + if not target_sh_id: + report.add( + severity="WARN", + check_id="C-20", + message=f"Task dependency target '{target_task_id}' is not linked to State Hub", + file_path=fname, + file_value=target_task_id, + fixable=False, + ) + continue + dep_key = (None, "task", target_sh_id, "starts_after") + if dep_key not in existing_dep_keys: + report.add( + severity="WARN", + check_id="C-20", + message=f"Missing DB dependency edge: {ws_id[:8]}… starts after task {target_task_id}", + file_path=fname, + db_id=ws_id, + file_value=target_task_id, + fixable=True, + _fix_context={ + "from_workstream_id": ws_id, + "to_task_id": target_sh_id, + "relationship_type": "starts_after", + }, + ) + + file_task_sh_ids: set[str] = set() + + for task in tasks: + if task.get("_parse_error"): + continue + t_id = str(task.get("id", "")).strip() + _raw_sh = task.get("state_hub_task_id") + t_sh_id = "" if _raw_sh is None else str(_raw_sh).strip().strip('"') + if t_sh_id in ("~", "null", "None", "none"): + t_sh_id = "" + t_status = normalise_task_status(task.get("status", "todo")) + + # CUST-WP-0074: task-block depends_on and wait qualifiers. + task_dep_targets = _frontmatter_depends_on_workplans(task) + _frontmatter_depends_on_tasks(task) + wait_fields = _task_wait_fields(task) + if t_status == "wait" and normalised_file_status not in CLOSED_WORKSTREAM_STATUSES: + # Rule 6: an owner-return wait with no workplan id yet is + # qualified by the C-25 vocabulary on the task block; a + # wait_until date is a window that clears by itself. + owner_return = bool(_BLOCKED_ON_RE.match(wait_fields["blocked_on"] or "")) + wait_until = wait_fields["wait_until"] + if ( + not task_dep_targets and not wait_fields["needs_human"] + and not owner_return and not wait_until + ): + report.add( + severity="WARN", check_id="C-39", + message=( + f"Wait task '{t_id}' in workplan '{meta.get('id', fname)}' has " + f"neither depends_on, needs_human: true, blocked_on: message-from:… " + f"nor wait_until — unqualified wait" + ), + file_path=f"{fname}#{t_id}", + file_value=t_status, + fixable=False, + ) + # CUST-WP-0075 P1: human gates belong in the decision queue. + if ( + wait_fields["needs_human"] and not wait_fields["decision_id"] + and not is_residual_flavor(meta.get("flavor")) + ): + report.add( + severity="WARN", check_id="C-39", + message=( + f"Human wait task '{t_id}' in workplan '{meta.get('id', fname)}' " + "has needs_human: true without decision_id — unrouted human gate" + ), + file_path=f"{fname}#{t_id}", + file_value="needs_human: true", + fixable=False, + ) + # C-40: rule 3 for a wait window — the date has passed (UTC today). + if wait_until and _date.fromisoformat(wait_until) < datetime.now(timezone.utc).date(): + report.add( + severity="WARN", check_id="C-40", + message=( + f"blocker satisfied, task still wait: '{t_id}' in workplan " + f"'{meta.get('id', fname)}' — wait_until {wait_until} has passed" + ), + file_path=f"{fname}#{t_id}", + file_value=wait_until, + fixable=False, + ) + # C-40: rule 3 — every depends_on target terminal. A target the + # hub cannot resolve is skipped, which also withholds the warning. + resolved_targets = [ + (target, _resolve_dependency_target(target)) for target in task_dep_targets + ] + if resolved_targets and all( + hub_status is not None + and _dependency_target_satisfied(target, hub_status) + for target, (_hub_id, hub_status) in resolved_targets + ): + report.add( + severity="WARN", check_id="C-40", + message=( + f"blocker satisfied, task still wait: '{t_id}' in workplan " + f"'{meta.get('id', fname)}' — every depends_on target is terminal " + f"({', '.join(task_dep_targets)})" + ), + file_path=f"{fname}#{t_id}", + file_value=", ".join(task_dep_targets), + fixable=False, + ) + # C-40: rule 4 — the tracked decision has been resolved. + if wait_fields["decision_id"]: + decision = _resolve_decision(wait_fields["decision_id"]) + if decision is not None and str(decision.get("status") or "") == "resolved": + report.add( + severity="WARN", check_id="C-40", + message=( + f"blocker satisfied, task still wait: '{t_id}' in workplan " + f"'{meta.get('id', fname)}' — decision " + f"{wait_fields['decision_id']} is resolved" + ), + file_path=f"{fname}#{t_id}", + file_value=wait_fields["decision_id"], + fixable=False, + ) + + if ( + t_sh_id + and task_dep_targets + and normalised_file_status not in CLOSED_WORKSTREAM_STATUSES + and t_status not in ("done", "cancel") + ): + # C-20 for task blocks: the waiting task is the from side. + # Closed workplans and closed tasks are not indexed — their + # ordering is history, not a live commitment. + for target in task_dep_targets: + target_hub_id, _target_status = _resolve_dependency_target(target) + is_task_target = bool(_TASK_RECORD_ID_RE.search(target)) + if not target_hub_id: + report.add( + severity="WARN", + check_id="C-20", + message=( + f"{'Task' if is_task_target else 'Workplan'} dependency target " + f"'{target}' of task '{t_id}' is not linked to State Hub" + ), + file_path=f"{fname}#{t_id}", + file_value=target, + fixable=False, + ) + continue + rel = "starts_after" if is_task_target else "blocks" + dep_key = (t_sh_id, "task" if is_task_target else "workstream", target_hub_id, rel) + if dep_key not in existing_dep_keys: + report.add( + severity="WARN", + check_id="C-20", + message=( + f"Missing DB dependency edge: task '{t_id}' " + f"{'starts after task' if is_task_target else 'depends on workplan'} {target}" + ), + file_path=f"{fname}#{t_id}", + db_id=ws_id, + file_value=target, + fixable=True, + _fix_context={ + "from_workstream_id": ws_id, + "from_task_id": t_sh_id, + "to_task_id" if is_task_target else "to_workstream_id": target_hub_id, + "relationship_type": rel, + "description": f"{t_id} depends_on {target}", + }, + ) + + if t_sh_id: + file_task_sh_ids.add(t_sh_id) + db_task = db_tasks_by_id.get(t_sh_id) + if db_task is None: + if t_id and t_sh_id == _derived_work_record_uuid(t_id): + report.add( + severity="WARN", + check_id="C-11", + message=f"Derived task '{t_id}' is absent from this projection", + file_path=f"{fname}#{t_id}", + db_id=t_sh_id, + fixable=True, + _fix_context={ + "ws_id": ws_id, + "ws_status": ws.get("status", ""), + "task": task, + "wp_file": str(wp_file), + "meta": meta, + "body": body, + "desired_task_id": t_sh_id, + }, + ) + else: + report.add( + severity="FAIL", check_id="C-03", + message=f"state_hub_task_id {t_sh_id[:8]}… not found in DB", + file_path=f"{fname}#{t_id}", + db_id=t_sh_id, + fixable=False, + ) + continue + # C-10 / C-15: task status drift. ADR-001: the file originates + # work. Same-rank wait/progress is not "DB ahead" — writing + # wait over progress/todo is the CFED/RMGR-WP-0004-T02 bug. + # C-15 remains a report-only review signal when the DB is + # terminal and the file is not. + db_t_status = normalise_task_status(db_task.get("status", "todo")) + if t_status and db_t_status and t_status != db_t_status: + db_rank = STATUS_ORDER.get(db_t_status, 0) + file_rank = STATUS_ORDER.get(t_status, 0) + db_terminal = db_t_status in {"done", "cancel"} + file_terminal = t_status in {"done", "cancel"} + if db_terminal and not file_terminal: + report.add( + severity="WARN", check_id="C-15", + message=( + f"DB task '{t_id}' is terminal " + f"(db={db_t_status!r}, file={t_status!r}) " + f"— report only; file wins (ADR-001 / RMGR-WP-0005-T06)" + ), + file_path=f"{fname}#{t_id}", + db_id=t_sh_id, + file_value=t_status, + db_value=db_t_status, + fixable=False, + ) + # File always wins, including same-rank wait ↔ progress. + report.add( + severity="WARN", check_id="C-10", + message=( + f"Task status drift '{t_id}': " + f"file={t_status!r} db={db_t_status!r} (file wins)" + ), + file_path=f"{fname}#{t_id}", + db_id=t_sh_id, + file_value=t_status, + db_value=db_t_status, + fixable=True, + _fix_context={ + "task_id": t_sh_id, + "status": t_status, + # A wait needs its qualifiers in the same PATCH. + "wait_fields": wait_fields if t_status == "wait" else {}, + }, + ) + file_description = task.get("description") + if isinstance(file_description, str): + file_description = file_description.strip() or None + else: + file_description = None + db_description = db_task.get("description") + if isinstance(db_description, str): + db_description = db_description.strip() or None + else: + db_description = None + if file_description and file_description != db_description: + report.add( + severity="WARN", check_id="C-22", + message=f"Task description drift '{t_id}': file content differs from DB (file wins)", + file_path=f"{fname}#{t_id}", + db_id=t_sh_id, + file_value=file_description[:120], + db_value=(db_description or "")[:120], + fixable=True, + _fix_context={"task_id": t_sh_id, "description": file_description}, + ) + file_task_flavor = normalize_flavor(task.get("flavor")) + if file_task_flavor is not None and file_task_flavor not in WORK_RECORD_FLAVORS: + report.add( + severity="WARN", + check_id="C-36", + message=( + f"Unknown work-record flavor {file_task_flavor!r} on task '{t_id}'" + ), + file_path=f"{fname}#{t_id}", + db_id=t_sh_id, + file_value=file_task_flavor, + fixable=False, + ) + file_task_flavor = None + db_task_flavor = normalize_flavor(db_task.get("flavor")) + if file_task_flavor != db_task_flavor: + report.add( + severity="WARN", + check_id="C-38", + message=( + f"Task flavor drift '{t_id}': " + f"file={file_task_flavor!r} db={db_task_flavor!r} (file wins)" + ), + file_path=f"{fname}#{t_id}", + db_id=t_sh_id, + file_value=file_task_flavor, + db_value=db_task_flavor, + fixable=True, + _fix_context={"task_id": t_sh_id, "flavor": file_task_flavor}, + ) + # C-41: wait qualifiers — only where the file speaks, so hub-set + # flags on blocks that never carried the keys are left alone. + if any(key in task for key in _TASK_WAIT_FIELD_KEYS): + db_wait_fields = { + "needs_human": bool(db_task.get("needs_human")), + "blocking_reason": str(db_task.get("blocking_reason") or "").strip() or None, + "decision_id": str(db_task.get("decision_id") or "").strip() or None, + "blocked_on": str(db_task.get("blocked_on") or "").strip() or None, + "wait_until": str(db_task.get("wait_until") or "").strip() or None, + } + file_wait_fields = {key: wait_fields[key] for key in _TASK_WAIT_FIELD_KEYS} + if file_wait_fields != db_wait_fields: + report.add( + severity="WARN", check_id="C-41", + message=( + f"Task wait-qualifier drift '{t_id}': " + f"file={file_wait_fields!r} db={db_wait_fields!r} (file wins)" + ), + file_path=f"{fname}#{t_id}", + db_id=t_sh_id, + file_value=json.dumps(file_wait_fields, sort_keys=True), + db_value=json.dumps(db_wait_fields, sort_keys=True), + fixable=True, + _fix_context={"task_id": t_sh_id, "wait_fields": wait_fields}, + ) + elif t_id: + # C-11: task exists in file but not linked to DB + ws_status = ws.get("status", "") + report.add( + severity="WARN", check_id="C-11", + message=f"Task '{t_id}' has no state_hub_task_id", + file_path=f"{fname}#{t_id}", + fixable=True, + _fix_context={ + "ws_id": ws_id, + "ws_status": ws_status, + "task": task, + "wp_file": str(wp_file), + "meta": meta, + "body": body, + }, + ) + + # C-12: DB tasks with no file backing + if isinstance(db_tasks, list): + ws_status = ws.get("status", "") + ws_finished = normalise_workstream_status(ws_status) in CLOSED_WORKSTREAM_STATUSES + for db_t in db_tasks: + if db_t["id"] not in file_task_sh_ids: + db_t_status = normalise_task_status(db_t.get("status", "todo")) + open_task = db_t_status not in TERMINAL_TASK_STATUSES + # Terminal orphans (cancel/done) are historical duplicates after + # file linkage or explicit retirement. The public task DELETE route + # is a cancel operation, so these are not further actionable. + if not open_task: + continue + # Auto-cancel fixable when workstream is finished and task is open. + fixable = ws_finished and open_task + report.add( + severity="WARN", check_id="C-12", + message=( + f"DB task '{db_t.get('title', '')}' " + f"(id={db_t['id'][:8]}…, status={db_t_status}) " + f"in workstream '{ws.get('slug')}' has no file backing" + ), + db_id=db_t["id"], + fixable=fixable, + _fix_context={ + "task_id": db_t["id"], + "ws_finished": ws_finished, + }, + ) + + # C-13: all DB tasks done but workstream still active — worker forgot to close + db_status = ws.get("status", "") + if normalise_workstream_status(db_status) == "active" and isinstance(db_tasks, list) and db_tasks: + non_terminal = [ + t for t in db_tasks + if normalise_task_status(t.get("status", "todo")) not in TERMINAL_TASK_STATUSES + ] + if not non_terminal: + report.add( + severity="WARN", check_id="C-13", + message=( + f"All {len(db_tasks)} DB tasks for '{ws.get('slug')}' are " + f"done/cancel but workstream status is still 'active' — " + f"worker likely forgot update_workstream_status()" + ), + file_path=fname, + db_id=ws_id, + db_value="active", + fixable=True, + _fix_context={ + "ws_id": ws_id, + "field": "status", + "value": "finished", + }, + ) + + # C-07 / C-08: orphan DB workstreams (have repo_id=this_repo but no backing file) + _check_orphan_db(api_base, repo_id, set(file_ws_ids.keys()), report, active_file_ws_ids) + + # C-14: ghost duplicate — active workstream on same topic with no repo_id whose + # title matches a file-backed workstream. Root cause: create_workstream() called + # before the workplan file was written; fix-consistency then created a second + # workstream from the file, leaving the first as an invisible orphan. + _check_ghost_duplicates(api_base, workplan_infos, file_ws_ids, report) + + _check_blocked_unblock_sweep(api_base, repo_slug, repo_dir, workplan_infos, report) + if report.source_revision is not None: + # The tree must not have moved while the files were read and C-25 ran. + if scan_source_revision(repo_dir) != (report.source_revision, None): + report.source_revision, report.source_revision_reason = None, "changed_during_scan" + _check_inbox_hygiene(api_base, repo_slug, report) + _check_unlinked_supersession(api_base, set(file_ws_ids), report) + _check_workplan_prefixes(repo_dir, repo_slug, workplan_infos, report) + _check_workplan_id_collisions(api_base, repo_slug, repo_dir, workplan_infos, report) + _check_scope_freshness(repo_dir, workplan_infos, report) + + _sync_workplan_bindings(api_base, repo_slug, workplan_infos, repo_dir, report) + + return report + + +def _sync_workplan_bindings( + api_base: str, + repo_slug: str, + workplan_infos: list[tuple[Path, dict, str]], + repo_dir: Path, + report: ConsistencyReport, +) -> None: + if repo_slug == "railiance-cluster": + scanned_paths = {workplan_display_path(repo_dir, path) for path, _, _ in workplan_infos} + missing = [entry["path"] for entry in FROZEN_CLUSTER_BINDINGS + if entry["path"] not in scanned_paths] + if missing: + report.add(severity="FAIL", check_id="C-44", + message="Frozen binding batch refused: bound source missing from scan", + fixable=False) + return + bindings: list[dict[str, Any]] = [] + for wp_file, meta, _ in workplan_infos: + ws_id = str(meta.get("state_hub_workstream_id", "")).strip().strip('"') + archived = wp_file.parent.name == "archived" + file_status = normalise_workstream_status(str(meta.get("status", "")).strip()) + binding = { + "workplan_id": ws_id, "filename": wp_file.name, + "relative_path": workplan_display_path(repo_dir, wp_file), + "repo_slug": repo_slug, "archived": archived, + "status": file_status or None, + } + try: + frozen = frozen_cluster_binding(binding) + if frozen is not None: + # Verify exact source before omitting the immutable binding. + # This is safe even against a receiver predating the API guard. + from hashlib import sha256 + if (wp_file.is_symlink() or not wp_file.is_file() + or sha256(wp_file.read_bytes()).hexdigest() != frozen["sha256"]): + raise ValueError("frozen historical source missing or changed") + continue + except ValueError as exc: + report.add(severity="FAIL", check_id="C-44", + message=f"Frozen binding batch refused: {exc}", + file_path=str(wp_file), fixable=False) + return # Never send a partial batch after a frozen-source refusal. + if not ws_id: + continue + bindings.append(binding) + if not bindings: + return + result = _api_put(api_base, "/workplans/index/bindings", {"bindings": bindings}) + if isinstance(result, dict) and "_error" in result: + report.fixes_applied.append(f"bindings WARN: {result['_error']}") + elif isinstance(result, dict): + report.fixes_applied.append( + f"bindings: synced {result.get('updated', 0)}/{result.get('received', len(bindings))}" + ) + + +RETIRED_SLUG_MARK = "@retired-" # mirrors api.services.forge_projection + + +def _is_projection_retired(ws: dict) -> bool: + """True for a row forge projection retired (field, or the slug stamp on older APIs).""" + return bool(ws.get("projection_retired_at")) or RETIRED_SLUG_MARK in (ws.get("slug") or "") + + +def _check_orphan_db( + api_base: str, + repo_id: str, + file_ws_ids: set[str], + report: ConsistencyReport, + active_file_ws_ids: set[str] | None = None, +) -> None: + """Flag DB workstreams with repo_id=this_repo that have no backing workplan file. + + A row retired by forge projection is a tombstone: retirement stamps the + slug with ``@retired-YYYYMMDD`` and sets ``projection_retired_at`` but + deliberately leaves status alone, so a row retired while blocked stays + blocked. Its canonical successor is file-backed under a new id. Such rows + are reported as C-08 INFO, never C-07 FAIL — the repo has nothing to fix, + and closing them by hand would rewrite history the retirement preserved. + """ + active_file_ws_ids = active_file_ws_ids or file_ws_ids + all_ws = _api_get(api_base, "/workplans", {"repo_id": repo_id}) + if not isinstance(all_ws, list): + return + for ws in all_ws: + ws_id = ws["id"] + ws_status = ws.get("status", "") + normalised_status = normalise_workstream_status(ws_status) + if normalised_status not in CLOSED_WORKSTREAM_STATUSES and ws_id in active_file_ws_ids: + continue + if normalised_status in CLOSED_WORKSTREAM_STATUSES and ws_id in file_ws_ids: + continue + ws_slug = ws.get("slug", "") + if _is_projection_retired(ws) and ws_id not in file_ws_ids: + report.add( + severity="INFO", check_id="C-08", + message=( + f"Retired DB workstream '{ws_slug}' " + f"(id={ws_id[:8]}…, status={ws_status}) has no backing workplan file " + f"— projection tombstone, not live work" + ), + db_id=ws_id, + fixable=False, + ) + continue + if normalised_status not in CLOSED_WORKSTREAM_STATUSES: + report.add( + severity="FAIL", check_id="C-07", + message=( + f"Non-closed DB workstream '{ws_slug}' (id={ws_id[:8]}…) " + f"has no backing workplan file — ADR-001 violation" + ), + db_id=ws_id, + fixable=False, + ) + elif normalised_status in CLOSED_WORKSTREAM_STATUSES: + report.add( + severity="INFO", check_id="C-08", + message=( + f"Closed DB workstream '{ws_slug}' " + f"(id={ws_id[:8]}…, status={ws_status}) has no backing workplan file" + ), + db_id=ws_id, + fixable=False, + ) + + +def _check_ghost_duplicates( + api_base: str, + workplan_infos: list[tuple], + file_ws_ids: dict[str, tuple], + report: ConsistencyReport, +) -> None: + """C-14: detect active workstreams with repo_id=null whose title matches a + file-backed workstream — these are ghosts created by premature create_workstream() + calls before the workplan file existed. + """ + # Build lookup: normalised title → file-backed workstream id + file_titles: dict[str, str] = {} + for _, meta, _ in workplan_infos: + ws_id = str(meta.get("state_hub_workstream_id", "")).strip().strip('"') + title = str(meta.get("title", "")).strip().lower() + if title and ws_id: + file_titles[title] = ws_id + + if not file_titles: + return + + # Gather topic_ids from all file-backed workstreams so we can query by topic + topic_ids: set[str] = set() + for ws_id in file_ws_ids: + ws = _api_get(api_base, f"/workplans/{ws_id}") + if ws and ws.get("topic_id"): + topic_ids.add(ws["topic_id"]) + + for topic_id in topic_ids: + topic_ws: list[dict] = [] + for status in OPEN_WORKSTREAM_STATUSES: + status_rows = _api_get(api_base, "/workplans", {"topic_id": topic_id, "status": status}) + if isinstance(status_rows, list): + topic_ws.extend(status_rows) + for ws in topic_ws: + ws_id = ws["id"] + if ws_id in file_ws_ids: + continue # legitimately linked + if ws.get("repo_id"): + continue # C-07 covers repo-scoped orphans + ws_title = ws.get("title", "").strip().lower() + if ws_title in file_titles: + file_backed_id = file_titles[ws_title] + report.add( + severity="WARN", + check_id="C-14", + message=( + f"Ghost duplicate: active workstream '{ws.get('slug')}' " + f"(id={ws_id[:8]}…, repo_id=null) has same title as " + f"file-backed workstream {file_backed_id[:8]}… — " + f"likely created via create_workstream() before workplan file existed; " + f"archive it" + ), + db_id=ws_id, + fixable=False, + ) + + +# --------------------------------------------------------------------------- +# Git sync (T02–T04: pull gate, writeback, push seal) — see repo_sync.py +# --------------------------------------------------------------------------- +# repo_sync.py owns the push-seal invariant and all git lifecycle primitives. +# The aliases below keep internal call sites and existing tests unchanged. + +from repo_sync import ( # noqa: E402 (import after top-level imports intentional) + count_local_ahead as _detect_ahead_of_remote, + count_remote_ahead as _count_remote_ahead, + pull_ff as _git_pull, + push_ff as _git_push, +) + + +def _detect_behind_remote(repo_path: str) -> bool: + """True if remote has commits the local repo lacks (C-16 predicate). + + Delegates to repo_sync.count_remote_ahead, which fetches before counting. + Returns False on any error so C-16 is never spuriously triggered. + """ + return _count_remote_ahead(repo_path) > 0 + + +def _patch_task_status_in_file( + file_path: Path, task_block_id: str, new_status: str +) -> bool: + """Update the ``status:`` field inside a task block identified by its id. + + Only modifies lines inside a ```task … ``` fenced block whose ``id:`` + matches *task_block_id*. Returns True if the file was changed. + """ + text = file_path.read_text(encoding="utf-8") + + def _replace(m: re.Match) -> str: + # m.group(0) is the full ```task...``` block including fences. + # m.group(1) is the inner YAML content (no fences). + full_block = m.group(0) + inner = m.group(1).strip() + task_meta = _parse_yaml_block(inner) + if str(task_meta.get("id", "")).strip() != task_block_id: + return full_block + # Replace the status line only, leave everything else untouched. + return re.sub( + r"^(status:\s*)\S+", + rf"\g<1>{new_status}", + full_block, + flags=re.MULTILINE, + ) + + new_text = _TASK_BLOCK_RE.sub(_replace, text) + if new_text != text: + file_path.write_text(new_text, encoding="utf-8") + return True + return False + + +def _git_identity_config() -> list[str]: + import os as _os + + name = _os.environ.get("GIT_SYNC_USER_NAME", "custodian-sync") + email = _os.environ.get("GIT_SYNC_USER_EMAIL", "custodian-sync@railiance.local") + return ["-c", f"user.name={name}", "-c", f"user.email={email}"] + + +def _git_commit_writeback( + repo_path: str, + file_path: Path, + changes: list[str], + *, + subject: str = "chore(consistency): sync task status from DB [auto]", +) -> bool: + """Stage *file_path* and commit with a standard writeback message. + + Returns True on success, False on any error (errors are logged to stderr + but do not abort the consistency run). + """ + from datetime import date as _date + summary = "\n".join(f" - {c}" for c in changes) + msg = ( + f"{subject}\n\n" + f"Updated by fix-consistency on {_date.today().isoformat()}:\n" + f"{summary}" + ) + import os as _os + # Pass GIT_CUSTODIAN_SYNC=1 so the post-commit hook can detect it is + # running from within a sync pass and exit early, preventing re-entrancy. + sync_env = {**_os.environ, "GIT_CUSTODIAN_SYNC": "1"} + try: + subprocess.run( + ["git", "-C", repo_path, "add", str(file_path)], + check=True, capture_output=True, env=sync_env, + ) + subprocess.run( + ["git", "-C", repo_path, *_git_identity_config(), "commit", "-m", msg], + check=True, capture_output=True, env=sync_env, + ) + return True + except subprocess.CalledProcessError as e: + print( + f"WARN: git commit failed for writeback: {e.stderr.decode().strip()}", + file=sys.stderr, + ) + return False + + +# --------------------------------------------------------------------------- +# Worker orientation brief (.custodian-brief.md) +# --------------------------------------------------------------------------- + +def default_wp_prefix(repo_slug: str) -> str: + first = repo_slug.split("-", 1)[0].upper() + return f"{first}-WP" + + +def infer_wp_prefix(repo_path: Path, repo_slug: str) -> str: + """Prefer established on-disk workplan prefixes over first-token derivation.""" + counts: Counter[str] = Counter() + workplans_dir = repo_path / "workplans" + if workplans_dir.is_dir(): + for workplan in iter_workplan_files(workplans_dir): + if workplan.name.startswith("ADHOC"): + continue + try: + text = workplan.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + id_match = _WP_ID_PREFIX_RE.search(text) or _WP_ID_BARE_RE.search(text) + if id_match: + counts[id_match.group(1).upper()] += 1 + continue + file_match = _WP_FILE_PREFIX_RE.match(workplan.name) or _WP_FILE_BARE_RE.match(workplan.name) + if file_match: + counts[file_match.group(1).upper()] += 1 + if not counts: + return default_wp_prefix(repo_slug) + return counts.most_common(1)[0][0] + + +def parse_blocked_on(value: str) -> str | None: + if not value: + return None + match = _BLOCKED_ON_RE.match(value.strip()) + return match.group("agent").strip() if match else None + + +def _parse_message_timestamp(value: Any) -> datetime | None: + if not value: + return None + if isinstance(value, datetime): + return value if value.tzinfo else value.replace(tzinfo=timezone.utc) + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +def _summarise_inbox_message(msg: dict[str, Any], *, now: datetime) -> dict[str, Any]: + created = _parse_message_timestamp(msg.get("created_at")) + age_days = (now - created).days if created else None + return { + "id": str(msg.get("id", ""))[:8], + "from_agent": msg.get("from_agent", ""), + "subject": msg.get("subject", ""), + "age_days": age_days, + } + + +def collect_inbox_hygiene(api_base: str, repo_slug: str) -> dict[str, Any]: + messages = _api_get( + api_base, + "/messages", + {"to_agent": repo_slug, "unread_only": True, "limit": 100}, + ) or [] + if not isinstance(messages, list): + return { + "stale_unread": [], + "missing_thread": [], + "work_requests_unpromoted": [], + "stale_unread_count": 0, + } + + now = datetime.now(timezone.utc) + stale_cutoff = now - timedelta(days=STALE_UNREAD_DAYS) + thread_cutoff = now - timedelta(days=1) + + stale_unread: list[dict[str, Any]] = [] + missing_thread: list[dict[str, Any]] = [] + work_requests_unpromoted: list[dict[str, Any]] = [] + + for msg in messages: + created = _parse_message_timestamp(msg.get("created_at")) + if created is None: + continue + summary = _summarise_inbox_message(msg, now=now) + if created < stale_cutoff: + stale_unread.append(summary) + if not msg.get("thread_id") and created < thread_cutoff: + missing_thread.append(summary) + body = f"{msg.get('subject', '')} {msg.get('body', '')}" + if _WORK_REQUEST_RE.search(body) and created < stale_cutoff: + work_requests_unpromoted.append(summary) + + return { + "stale_unread": stale_unread, + "missing_thread": missing_thread, + "work_requests_unpromoted": work_requests_unpromoted, + "stale_unread_count": len(stale_unread), + } + + +_FULL_SHA = re.compile(r"^[0-9a-f]{40}$") + + +def scan_source_revision(repo_dir: Path) -> tuple[str | None, str | None]: + """Return ``(sha, None)`` for a clean ``workplans/`` tree, else ``(None, reason)``. + + ``sha`` is the full 40-hex HEAD commit. Any change under ``workplans/`` + (modified, staged, untracked or ignored) means the scanned files are not + that commit, so no revision is claimed. Reasons: ``not_a_git_tree``, + ``no_head_commit``, ``unsupported_object_format``, ``dirty_workplans``, + ``git_status_failed``. + """ + def git(*args: str) -> str | None: + try: + return subprocess.run( + ["git", *args], cwd=repo_dir, capture_output=True, text=True, + timeout=30, check=True, + ).stdout + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError): + return None + + if not Path(repo_dir).is_dir() or git("rev-parse", "--is-inside-work-tree") is None: + return None, "not_a_git_tree" + head = git("rev-parse", "--verify", "--quiet", "HEAD^{commit}") + if head is None or not head.strip(): + return None, "no_head_commit" + sha = head.strip() + if not _FULL_SHA.fullmatch(sha): + return None, "unsupported_object_format" + status = git("status", "--porcelain", "--untracked-files=all", "--ignored", + "--", "workplans") + if status is None: + return None, "git_status_failed" + if status.strip(): + return None, "dirty_workplans" + return sha, None + + +C25_PAGE_SIZE = 200 +C25_MAX_PAGES = 50 + + +def _collect_counterpart_messages( + api_base: str, repo_slug: str, counterpart: str +) -> tuple[list[dict], bool]: + """Page through every reply from ``counterpart`` (STATE-WP-0094-T03). + + Returns ``(messages, complete)``. Messages are deduplicated by id, so a + reply shifting between pages while new mail arrives is counted once and the + finding's message ids stay stable. ``complete`` is False when the hub + ignored ``offset`` (a page repeated only known ids) or the page bound was + hit; the caller reports that instead of silently capping. + """ + seen: dict[str, dict] = {} + anonymous: list[dict] = [] + for page in range(C25_MAX_PAGES): + rows = _api_get(api_base, "/messages", { + "to_agent": repo_slug, "from_agent": counterpart, + "unread_only": False, "limit": C25_PAGE_SIZE, "offset": page * C25_PAGE_SIZE, + }) or [] + if not isinstance(rows, list): + rows = [] + new = 0 + for row in rows: + if not isinstance(row, dict): + continue + message_id = str(row.get("id") or "") + if not message_id: + anonymous.append(row) + new += 1 + elif message_id not in seen: + seen[message_id] = row + new += 1 + if len(rows) < C25_PAGE_SIZE: + return list(seen.values()) + anonymous, True + if new == 0: + return list(seen.values()) + anonymous, False + return list(seen.values()) + anonymous, False + + +def _check_blocked_unblock_sweep( + api_base: str, + repo_slug: str, + repo_dir: Path, + workplan_infos: list[tuple[Path, dict, str]], + report: ConsistencyReport, +) -> None: + for wp_file, meta, _ in workplan_infos: + file_status = normalise_workstream_status(str(meta.get("status", "")).strip()) + if file_status != "blocked": + continue + blocked_on = str(meta.get("blocked_on", "")).strip() + counterpart = parse_blocked_on(blocked_on) + if not counterpart: + continue + messages = _api_get( + api_base, + "/messages", + { + "to_agent": repo_slug, + "from_agent": counterpart, + "unread_only": True, + "limit": 20, + }, + ) or [] + if not isinstance(messages, list) or not messages: + continue + fname = workplan_display_path(repo_dir, wp_file) + wp_id = str(meta.get("id", wp_file.stem)).strip() + report.add( + severity="WARN", + check_id="C-25", + message=( + f"Blocked workplan '{wp_id}' waiting on {blocked_on!r} has " + f"{len(messages)} unread message(s) from {counterpart} — " + "blocker may have cleared" + ), + file_path=fname, + file_value=blocked_on, + fixable=False, + ) + + + # Task-level returns can clear a wait even after somebody read the reply. + # Preserve the legacy workplan check above; task findings carry stable IDs + # for an owner runner to deduplicate without changing task status here. + messages_by_agent: dict[str, list[dict]] = {} + for wp_file, meta, body in workplan_infos: + if ( + normalise_workstream_status(str(meta.get("status", ""))) in CLOSED_WORKSTREAM_STATUSES + or is_residual_flavor(meta.get("flavor")) + ): + continue + for task in get_tasks_from_workplan(meta, body): + if normalise_task_status(task.get("status", "todo")) != "wait": + continue + counterpart = parse_blocked_on(str(task.get("blocked_on") or "")) + if not counterpart: + continue + if counterpart not in messages_by_agent: + rows, complete = _collect_counterpart_messages(api_base, repo_slug, counterpart) + messages_by_agent[counterpart] = rows + if not complete: + report.add( + severity="WARN", check_id="C-25", + message=(f"Owner-return reply collection from {counterpart} is incomplete " + f"({len(rows)} message(s) read; hub ignored paging or the " + f"{C25_MAX_PAGES}-page bound was reached) — returns may be missing"), + file_path=workplan_display_path(repo_dir, wp_file), + file_value=f"message-from:{counterpart}", + fixable=False, + ) + # Dated notes are the available acknowledgement watermark. Date-only + # notes cover the whole UTC day, avoiding same-day re-notification. + note_dates = [] + for value in re.findall(r"(?m)^\s*(?:[-*]\s+)?(?:\*\*)?(\d{4}-\d{2}-\d{2})\b", str(task.get("description") or "")): + parsed = _parse_message_timestamp(value) + if parsed: + note_dates.append(parsed + timedelta(days=1)) + explicit_updated = _parse_message_timestamp(task.get("updated")) + cutoff = max(note_dates + ([explicit_updated] if explicit_updated else []), default=None) + hits = [] + for message in messages_by_agent[counterpart]: + if message.get("from_agent") != counterpart or message.get("archived_at"): + continue + created = _parse_message_timestamp(message.get("created_at")) + if created and (created >= cutoff if cutoff else not message.get("read_at")): + hits.append(message) + if not hits: + continue + hits.sort(key=lambda message: str(message.get("created_at", ""))) + task_id = str(task.get("id", "")).strip() + message_ids = [str(message["id"]) for message in hits if message.get("id")] + report.add( + severity="WARN", check_id="C-25", + message=(f"Owner-return task '{task_id}' waiting on {counterpart} has " + f"{len(hits)} reply/replies after its last dated note " + f"(or unread when undated) — owner action needed: {', '.join(message_ids)}"), + file_path=f"{workplan_display_path(repo_dir, wp_file)}#{task_id}", + file_value=str(task.get("blocked_on")), + fixable=False, + _fix_context={"task_record_id": task_id, "message_ids": message_ids}, + ) + + +_SUPERSEDES_RE = re.compile( + r"\bsupersedes\s+(?:decision\s+)?([0-9a-f]{8}(?:-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})?)\b", + re.IGNORECASE, +) + + +def find_unlinked_supersessions( + decisions: list[dict], workplan_ids: set[str] +) -> list[tuple[dict, dict]]: + """(citing, cited) pairs where a resolved decision of these workplans says + "supersedes " but the cited decision is not superseded_by it. "Superseded + by" is the other direction and is not matched; ambiguous prefixes are skipped.""" + pairs: list[tuple[dict, dict]] = [] + for citing in decisions: + if str(citing.get("status") or "") != "resolved": + continue + if str(citing.get("workplan_id") or "") not in workplan_ids: + continue + text = " ".join(str(citing.get(k) or "") for k in ("title", "rationale", "description")) + for ref in dict.fromkeys(m.lower() for m in _SUPERSEDES_RE.findall(text)): + matches = [d for d in decisions if str(d.get("id") or "").lower().startswith(ref)] + if len(matches) != 1 or matches[0] is citing: + continue + cited = matches[0] + if ( + str(cited.get("status") or "") != "superseded" + or str(cited.get("superseded_by") or "") != str(citing["id"]) + ): + pairs.append((citing, cited)) + return pairs + + +def _check_unlinked_supersession( + api_base: str, workplan_ids: set[str], report: ConsistencyReport +) -> None: + workplan_ids = {str(w) for w in workplan_ids if w} + if not workplan_ids: + return + decisions = _api_get(api_base, "/decisions/") + if not isinstance(decisions, list): + return + for citing, cited in find_unlinked_supersessions(decisions, workplan_ids): + report.add( + severity="WARN", + check_id="C-43", + message=( + f"Decision {citing['id']} supersedes {cited['id']}, but the cited " + f"decision is status={cited.get('status')!r} " + f"superseded_by={cited.get('superseded_by')!r}; the decider should " + f"set status=superseded and superseded_by" + ), + db_id=str(cited["id"]), + fixable=False, + ) + + +def _check_inbox_hygiene(api_base: str, repo_slug: str, report: ConsistencyReport) -> None: + hygiene = collect_inbox_hygiene(api_base, repo_slug) + if hygiene["stale_unread_count"]: + preview = ", ".join( + f"{m['from_agent']}:{m['id']}" for m in hygiene["stale_unread"][:5] + ) + extra = "" if hygiene["stale_unread_count"] <= 5 else ", ..." + report.add( + severity="WARN", + check_id="C-28", + message=( + f"{hygiene['stale_unread_count']} unread inbox message(s) older than " + f"{STALE_UNREAD_DAYS} day(s): {preview}{extra}" + ), + fixable=False, + ) + if hygiene["missing_thread"]: + preview = ", ".join( + f"{m['from_agent']}:{m['id']}" for m in hygiene["missing_thread"][:5] + ) + report.add( + severity="WARN", + check_id="C-28", + message=( + f"{len(hygiene['missing_thread'])} unread message(s) lack thread_id " + f"for supersession tracking: {preview}" + ), + fixable=False, + ) + for msg in hygiene["work_requests_unpromoted"]: + report.add( + severity="WARN", + check_id="C-29", + message=( + f"Unread work request from {msg['from_agent']} ({msg['id']}) may need " + f"a workplan file: {msg['subject'][:120]}" + ), + fixable=False, + ) + + +def _check_workplan_prefixes( + repo_dir: Path, + repo_slug: str, + workplan_infos: list[tuple[Path, dict, str]], + report: ConsistencyReport, +) -> None: + canonical = infer_wp_prefix(repo_dir, repo_slug) + for wp_file, meta, _ in workplan_infos: + if wp_file.name.startswith("ADHOC"): + continue + wp_id = str(meta.get("id", "")).strip() + if not wp_id: + continue + match = _WP_ID_PREFIX_RE.match(wp_id) or _WP_ID_BARE_RE.match(wp_id) + if not match: + continue + prefix = match.group(1).upper() + if prefix == canonical.upper(): + continue + report.add( + severity="WARN", + check_id="C-26", + message=( + f"Workplan id '{wp_id}' uses prefix '{prefix}' but repo canonical " + f"prefix is '{canonical}' — new plans must conform" + ), + file_path=workplan_display_path(repo_dir, wp_file), + file_value=wp_id, + db_value=canonical, + fixable=False, + ) + + +def _scan_repo_workplan_ids(repos: list[dict[str, Any]]) -> dict[str, list[tuple[str, str]]]: + id_map: dict[str, list[tuple[str, str]]] = {} + for repo in repos: + slug = repo["slug"] + path = resolve_repo_path(repo) + if not path or not Path(path).is_dir(): + continue + workplans_dir = Path(path) / "workplans" + if not workplans_dir.is_dir(): + continue + for wp_file in iter_workplan_files(workplans_dir): + if wp_file.name.startswith("ADHOC"): + continue + try: + text = wp_file.read_text(encoding="utf-8") + except OSError: + continue + if not text.startswith("---"): + continue + meta, _ = parse_frontmatter(text) + if not meta or meta.get("_parse_error"): + continue + wp_id = str(meta.get("id", "")).strip() + if wp_id: + id_map.setdefault(wp_id, []).append((slug, wp_file.name)) + return id_map + + +def _check_workplan_id_collisions( + api_base: str, + repo_slug: str, + repo_dir: Path, + workplan_infos: list[tuple[Path, dict, str]], + report: ConsistencyReport, +) -> None: + repos = _api_get(api_base, "/repos") or [] + if not isinstance(repos, list): + return + id_map = _scan_repo_workplan_ids(repos) + for wp_file, meta, _ in workplan_infos: + wp_id = str(meta.get("id", "")).strip() + if not wp_id: + continue + locations = id_map.get(wp_id, []) + if len(locations) <= 1: + continue + report.add( + severity="WARN", + check_id="C-27", + message=( + f"Workplan id '{wp_id}' collides across repos: " + + ", ".join(f"{slug}/{fname}" for slug, fname in locations) + ), + file_path=workplan_display_path(repo_dir, wp_file), + file_value=wp_id, + fixable=False, + ) + + +def _scope_current_state_lines(repo_dir: Path) -> dict[str, str]: + scope_path = repo_dir / "SCOPE.md" + if not scope_path.exists(): + return {} + text = scope_path.read_text(encoding="utf-8", errors="replace") + match = re.search(r"## Current State\s*\n(.*?)(?:\n## |\Z)", text, re.DOTALL) + if not match: + return {} + result: dict[str, str] = {} + for line in match.group(1).splitlines(): + stripped = line.strip() + if not stripped.startswith("-"): + continue + parts = stripped[1:].split(":", 1) + if len(parts) != 2: + continue + result[parts[0].strip().lower()] = parts[1].strip() + return result + + +def _repo_has_open_workplans(workplan_infos: list[tuple[Path, dict, str]]) -> bool: + for wp_file, meta, _ in workplan_infos: + if wp_file.parent.name == "archived": + continue + status = normalise_workstream_status(str(meta.get("status", "")).strip()) + if status in _OPEN_WORKPLAN_STATUSES: + return True + return False + + +def _check_scope_freshness( + repo_dir: Path, + workplan_infos: list[tuple[Path, dict, str]], + report: ConsistencyReport, +) -> None: + scope = _scope_current_state_lines(repo_dir) + if not scope: + return + has_open_workplans = _repo_has_open_workplans(workplan_infos) + scope_status = scope.get("status", "").lower() + warnings: list[str] = [] + if "active" in scope_status and not has_open_workplans: + warnings.append( + "SCOPE.md Current State says active but no open workplans remain — may be stale" + ) + if has_open_workplans and any( + token in scope_status for token in ("finished", "archived", "idle", "dormant") + ): + warnings.append( + "SCOPE.md Current State does not reflect active workplans — may be stale" + ) + implementation = scope.get("implementation", "").lower() + if has_open_workplans and implementation and "not yet started" in implementation: + warnings.append( + "SCOPE.md mentions work not yet started but active workplans exist — may be stale" + ) + for warning in warnings: + report.add( + severity="WARN", + check_id="C-30", + message=warning, + file_path="SCOPE.md", + fixable=False, + ) + + +_BRIEF_HEADER = "" +_TASK_STATUS_ICON = {"done": "✓", "cancel": "✗", "progress": "►", "wait": "!", "todo": "·"} +_OPEN_STATUSES = set(OPEN_TASK_STATUSES) + + +def _commit_working_memory_notes(repo_slug: str, repo_path: str) -> bool: + """Stage and commit activity-core daily-triage working-memory notes. + + Only the-custodian carries the fleet working-memory sink. Notes are + repo-relative under memory/working/ and are written by activity-core on + Railiance via hostPath into the clone checkout. + """ + if repo_slug != "the-custodian": + return False + + wm_dir = Path(repo_path) / "memory" / "working" + if not wm_dir.is_dir(): + return False + + notes = sorted(wm_dir.glob("daily-triage-*.md")) + if not notes: + return False + + import os as _os + + sync_env = {**_os.environ, "GIT_CUSTODIAN_SYNC": "1"} + rel_paths = [str(note.relative_to(repo_path)) for note in notes] + try: + subprocess.run( + ["git", "-C", repo_path, "add", "--", *rel_paths], + check=True, + capture_output=True, + env=sync_env, + ) + staged = subprocess.run( + ["git", "-C", repo_path, "diff", "--cached", "--quiet"], + capture_output=True, + env=sync_env, + ) + if staged.returncode == 0: + return False + from datetime import date as _date + + msg = ( + "chore(working-memory): sync daily-triage notes from activity-core " + f"[auto {_date.today().isoformat()}]" + ) + subprocess.run( + ["git", "-C", repo_path, *_git_identity_config(), "commit", "-m", msg], + check=True, + capture_output=True, + env=sync_env, + ) + return True + except subprocess.CalledProcessError as e: + print( + f"WARN: working-memory commit failed: {e.stderr.decode().strip()}", + file=sys.stderr, + ) + return False + + +def _write_custodian_brief(api_base: str, repo_slug: str, repo_path: str) -> bool: + """Generate .custodian-brief.md at the repo root and git-commit if changed. + + The brief gives any agent — including subagents without MCP access and + workers on remote machines — instant orientation without a live hub + connection. Returns True if the file was written (content changed). + """ + import datetime as _dt + from datetime import timezone as _tz + + repo = _api_get(api_base, f"/repos/{repo_slug}") + if not repo: + return False + + repo_id: str = repo.get("id", "") + domain_slug = str(repo.get("domain_slug") or "") + + # The repository projection owns domain identity. Retain the workplan/topic + # lookup only for compatibility with older API responses that omitted it. + workstreams: list[dict] = [] + for status in OPEN_WORKSTREAM_STATUSES: + rows = _api_get(api_base, "/workplans", {"repo_id": repo_id, "status": status}) or [] + if isinstance(rows, list): + workstreams.extend(rows) + _ws_for_domain = workstreams if workstreams else [] + if not _ws_for_domain: + all_ws = _api_get(api_base, "/workplans", {"repo_id": repo_id}) or [] + _ws_for_domain = all_ws if isinstance(all_ws, list) else [] + if not domain_slug and _ws_for_domain: + topic = _api_get(api_base, f"/topics/{_ws_for_domain[0].get('topic_id', '')}") + if topic: + domain_slug = topic.get("domain_slug", "") + + # Active repo goal (first active one if multiple) + goal_text = "" + goals = _api_get(api_base, "/repo-goals", {"repo_slug": repo_slug}) or [] + if isinstance(goals, list): + active_goals = [g for g in goals if g.get("status") == "active"] + if active_goals: + g = active_goals[0] + goal_text = g.get("title", "") or g.get("description", "") + + now_utc = _dt.datetime.now(_tz.utc) + ts = now_utc.strftime("%Y-%m-%d %H:%M UTC") + + lines = [ + _BRIEF_HEADER, + f"# Custodian Brief — {repo_slug}", + "", + f"**Domain:** {domain_slug or '(unknown)'} ", + f"**Last synced:** {ts} ", + "**State Hub:** http://127.0.0.1:8000 *(adjust if running on a remote machine)*", + "", + ] + + if goal_text: + lines += ["## Current Goal", "", goal_text, ""] + + if isinstance(workstreams, list) and workstreams: + lines.append("## Active Workstreams") + for ws in workstreams: + ws_title = ws.get("title", ws.get("slug", "?")) + ws_id = ws["id"] + tasks = _api_get(api_base, "/tasks", {"workplan_id": ws_id}) or [] + if not isinstance(tasks, list): + tasks = [] + + done = sum( + 1 for t in tasks + if normalise_task_status(t.get("status", "todo")) in TERMINAL_TASK_STATUSES + ) + total = len(tasks) + pct = f"{done}/{total}" if total else "no tasks" + + open_tasks = [ + t for t in tasks + if normalise_task_status(t.get("status", "todo")) in _OPEN_STATUSES + ] + # Show wait first, then progress, then todo (cap at 5). + priority_order = {"wait": 0, "progress": 1, "todo": 2} + open_tasks.sort( + key=lambda t: priority_order.get( + normalise_task_status(t.get("status", "todo")), 9 + ) + ) + + lines += [ + "", + f"### {ws_title}", + f"Progress: {pct} done | workplan_id: `{ws_id}`", + ] + + if open_tasks: + lines.append("") + lines.append("**Open tasks:**") + for t in open_tasks[:7]: + status = normalise_task_status(t.get("status", "todo")) + icon = _TASK_STATUS_ICON.get(status, "·") + title = t.get("title", t["id"]) + tid = t["id"] + blocker = t.get("blocking_reason", "") + task_line = f"- {icon} {title} `{tid[:8]}`" + if status == "wait" and blocker: + task_line += f"\n *(wait: {blocker})*" + lines.append(task_line) + if len(open_tasks) > 7: + lines.append(f"- … and {len(open_tasks) - 7} more open tasks") + else: + lines += ["## Active Workstreams", "", "*(none — repo may need first-session setup)*"] + + hygiene = collect_inbox_hygiene(api_base, repo_slug) + if hygiene["stale_unread_count"] or hygiene["missing_thread"] or hygiene["work_requests_unpromoted"]: + lines += ["", "## Inbox Hygiene", ""] + if hygiene["stale_unread_count"]: + lines.append( + f"**Stale unread:** {hygiene['stale_unread_count']} message(s) older than " + f"{STALE_UNREAD_DAYS} day(s) — triage at session start." + ) + if hygiene["missing_thread"]: + lines.append( + f"**Missing thread_id:** {len(hygiene['missing_thread'])} unread message(s) " + "lack supersession chains." + ) + for msg in hygiene["work_requests_unpromoted"][:3]: + lines.append( + f"- ! {msg['from_agent']}: {msg['subject'][:100]} `{msg['id']}`" + ) + + blocked_warnings: list[str] = [] + workplans_dir = Path(repo_path) / "workplans" + if workplans_dir.is_dir(): + for wp_file in iter_workplan_files(workplans_dir): + try: + text = wp_file.read_text(encoding="utf-8") + except OSError: + continue + if not text.startswith("---"): + continue + meta, _ = parse_frontmatter(text) + if not meta or meta.get("_parse_error"): + continue + if normalise_workstream_status(str(meta.get("status", "")).strip()) != "blocked": + continue + blocked_on = str(meta.get("blocked_on", "")).strip() + counterpart = parse_blocked_on(blocked_on) + if not counterpart: + continue + messages = _api_get( + api_base, + "/messages", + { + "to_agent": repo_slug, + "from_agent": counterpart, + "unread_only": True, + "limit": 5, + }, + ) or [] + if isinstance(messages, list) and messages: + wp_id = str(meta.get("id", wp_file.stem)).strip() + blocked_warnings.append( + f"- ! **{wp_id}** — blocker may have cleared: " + f"{len(messages)} unread message(s) from `{counterpart}`" + ) + if blocked_warnings: + lines += ["", "## Blocked Workplans", ""] + lines.extend(blocked_warnings) + + scope_warnings: list[str] = [] + scope = _scope_current_state_lines(Path(repo_path)) + has_open_workplans = False + if workplans_dir.is_dir(): + for wp_file in iter_workplan_files(workplans_dir): + try: + text = wp_file.read_text(encoding="utf-8") + except OSError: + continue + if not text.startswith("---"): + continue + meta, _ = parse_frontmatter(text) + if not meta or meta.get("_parse_error"): + continue + if wp_file.parent.name == "archived": + continue + status = normalise_workstream_status(str(meta.get("status", "")).strip()) + if status in _OPEN_WORKPLAN_STATUSES: + has_open_workplans = True + break + if scope: + scope_status = scope.get("status", "").lower() + if "active" in scope_status and not has_open_workplans: + scope_warnings.append( + "SCOPE.md says active but no open workplans remain — refresh Current State." + ) + if has_open_workplans and any( + token in scope_status for token in ("finished", "archived", "idle", "dormant") + ): + scope_warnings.append( + "SCOPE.md Current State does not reflect active workplans — may be stale." + ) + if scope_warnings: + lines += ["", "## SCOPE Freshness", ""] + lines.extend(f"- {warning}" for warning in scope_warnings) + + lines += [ + "", + "---", + "## MCP Orientation (when available)", + "", + "If the state-hub MCP server is reachable, call:", + f"`get_domain_summary(\"{domain_slug}\")`", + "This provides richer cross-domain context.", + "If the MCP call fails, use this file as your orientation source.", + ] + + content = "\n".join(lines) + "\n" + + brief_path = Path(repo_path) / ".custodian-brief.md" + existing = brief_path.read_text(encoding="utf-8") if brief_path.exists() else "" + + # Strip the timestamp line before comparing to avoid spurious writes + def _strip_ts(text: str) -> str: + return "\n".join( + ln for ln in text.splitlines() + if not ln.startswith("**Last synced:**") + ) + + if _strip_ts(content) == _strip_ts(existing): + return False # no meaningful change + + brief_path.write_text(content, encoding="utf-8") + + # Commit the brief so remote workers can pull it + _git_commit_writeback( + repo_path, + brief_path, + [f"update .custodian-brief.md for {repo_slug}"], + ) + return True + + +# --------------------------------------------------------------------------- +# Fix engine +# --------------------------------------------------------------------------- + +def fix_repo( + api_base: str, + repo_slug: str, + repo_path_override: str | None = None, + no_writeback: bool = False, + bootstrap_empty_projection: bool = False, +) -> ConsistencyReport: + """Run checks then apply all auto-fixable issues. Returns updated report.""" + report = check_repo( + api_base, + repo_slug, + repo_path_override, + bootstrap_empty_projection=bootstrap_empty_projection, + ) + if any(i.check_id in {"C-00", "C-36"} for i in report.failures): + return report + + # RMGR-WP-0002: optional repo-manager reconcile proxy (index rebuild) for pilots + try: + from api.services.repo_manager_dual_run import reconcile_for_repo, rm_reconcile + + if reconcile_for_repo(repo_slug) and report.repo_path: + rm_rec = rm_reconcile(repo_path=report.repo_path, repo_slug=repo_slug) + if rm_rec.get("ok") or rm_rec.get("exit_code") == 0: + report.fixes_applied.append( + f"RM dual-run reconcile ok: index={rm_rec.get('index_path') or 'written'}" + ) + else: + report.fixes_applied.append( + f"RM dual-run reconcile note: {rm_rec.get('error') or rm_rec}" + ) + except Exception as rec_exc: # noqa: BLE001 + report.fixes_applied.append(f"RM dual-run reconcile skipped: {rec_exc}") + + # Auto-register this machine's path in host_paths so future runs work + # without --repo-path. Idempotent: skipped when already correct. + repo_path = report.repo_path + if repo_path: + repo_record = _api_get(api_base, f"/repos/{repo_slug}") + if repo_record: + hostname = _sweep_hostname() + if (repo_record.get("host_paths") or {}).get(hostname) != repo_path: + result = _api_post( + api_base, f"/repos/{repo_slug}/paths", + {"host": hostname, "path": repo_path}, + ) + if result and "_error" not in result: + report.fixes_applied.append( + f"host_paths[{hostname}] → {repo_path}" + ) + + # T02 — pull gate: warn and skip all write operations when local repo is + # behind its remote tracking branch. + if repo_path and _detect_behind_remote(repo_path): + report.add( + severity="WARN", check_id="C-16", + message=( + f"Repo '{repo_slug}' is behind its remote tracking branch — " + f"pull before fixing to avoid clobbering remote progress. " + f"Run: git -C {repo_path} pull --ff-only" + ), + fixable=False, + ) + report.fixes_applied.append( + "C-16: all write operations skipped — local repo is behind remote" + ) + return report + + # C-17: backlog guard — if local has unpushed commits from a prior failed push, + # try to push them before making more. Skipping writes prevents runaway divergence. + if repo_path: + ahead = _detect_ahead_of_remote(repo_path) + if ahead > 0: + push_ok, push_msg = _git_push(repo_path) + if not push_ok: + report.add( + severity="WARN", check_id="C-17", + message=( + f"Repo '{repo_slug}' has {ahead} unpushed commit(s) and push " + f"failed ({push_msg}) — skipping writes to prevent runaway divergence" + ), + fixable=False, + ) + report.fixes_applied.append( + "C-17: all write operations skipped — unpushed commits, push failed" + ) + return report + # Push succeeded — local is now in sync; proceed normally + report.fixes_applied.append(f"C-17 cleared: pushed {ahead} backlogged commit(s)") + + fixable = [i for i in report.issues if i.fixable] + + for issue in fixable: + ctx = issue._fix_context + try: + if issue.check_id in ("C-04", "C-05", "C-13", "C-19"): + ws_id = ctx["ws_id"] + result = _api_patch(api_base, f"/workplans/{ws_id}", + {ctx["field"]: ctx["value"]}) + if result is not None and "_error" not in result: + report.fixes_applied.append( + f"{issue.check_id} fixed: workstream {ws_id[:8]}… " + f"{ctx['field']} → {ctx['value']!r}" + ) + elif result is not None: + report.fixes_applied.append( + f"{issue.check_id} FAILED: workstream {ws_id[:8]}… " + f"{ctx['field']} → {ctx['value']!r}: {result['_error']}" + ) + + elif issue.check_id == "C-23": + ws_id = ctx["ws_id"] + target_status = ctx["target_status"] + result = _api_patch(api_base, f"/workplans/{ws_id}", {"status": target_status}) + if result is not None and "_error" not in result: + report.fixes_applied.append( + f"C-23 fixed: workstream {ws_id[:8]}… status → {target_status!r}" + ) + elif result is not None: + report.fixes_applied.append( + f"C-23 FAILED: workstream {ws_id[:8]}… " + f"status → {target_status!r}: {result['_error']}" + ) + + if no_writeback: + report.fixes_applied.append( + f"C-23 skipped file repair (--no-writeback): {Path(ctx['wp_file']).name}" + ) + else: + wp_file = Path(ctx["wp_file"]) + old_status = ctx["file_status"] + if _patch_frontmatter_field(wp_file, "status", target_status): + committed = _git_commit_writeback( + repo_path, + wp_file, + [f"workplan status: {old_status} → {target_status}"], + subject="chore(consistency): renormalize lifecycle state [auto]", + ) + suffix = " (committed)" if committed else " (file patched, commit failed)" + report.fixes_applied.append( + f"C-23 fixed: {wp_file.name} status " + f"{old_status} → {target_status}{suffix}" + ) + else: + report.fixes_applied.append( + f"C-23 SKIP: {wp_file.name} already has status {target_status!r}" + ) + + elif issue.check_id == "C-06": + wp_file = Path(ctx["wp_file"]) + meta = ctx["meta"] + domain = ctx["domain"] + repo_market_domain = str(ctx.get("repo_market_domain") or "").strip() + repo_id_val = ctx["repo_id"] + body = ctx.get("body", "") + wp_id = str(meta.get("id", "")).strip() + desired_ws_id = str( + ctx.get("desired_ws_id") or _derived_work_record_uuid(wp_id) + ) + title = str(meta.get("title", "")).strip() + status = str(meta.get("status", "active")).strip() + status = normalise_workstream_status(status) + if status not in VALID_WP_STATUSES: + status = "active" + + # Find topic_id — workplan frontmatter may still use legacy + # coordination slugs (e.g. custodian); map to market domain first. + topic_domain = resolve_topic_domain_slug( + domain, + repo_market_domain=repo_market_domain or None, + ) + topics = _api_get(api_base, "/topics") + topic_id = None + if isinstance(topics, list): + for t in topics: + if t.get("domain_slug") == topic_domain: + topic_id = t["id"] + break + if topic_id is None: + report.fixes_applied.append( + f"C-06 SKIP {wp_id}: no topic found for domain " + f"'{topic_domain}' (workplan domain={domain!r})" + ) + continue + + base_slug = re.sub(r"[^a-z0-9-]", "-", wp_id.lower()).strip("-") or "workplan" + repo_slug_part = re.sub( + r"[^a-z0-9-]", "-", str(ctx.get("repo_slug") or "").lower() + ).strip("-") + slug_candidates = [base_slug] + repo_qualified_slug = base_slug + if repo_slug_part and not base_slug.startswith(f"{repo_slug_part}-"): + repo_qualified_slug = f"{repo_slug_part}-{base_slug}" + slug_candidates.append(repo_qualified_slug) + for suffix in range(2, 21): + slug_candidates.append(f"{repo_qualified_slug}-{suffix}") + + ws_data = None + last_error = None + for slug in slug_candidates: + existing = _api_get(api_base, "/workplans", {"slug": slug}, return_error=True) + if isinstance(existing, dict) and "_error" in existing: + last_error = existing["_error"] + continue + if isinstance(existing, list) and existing: + existing_same_repo = next( + (w for w in existing if w.get("repo_id") == repo_id_val), + None, + ) + if ( + existing_same_repo + and existing_same_repo.get("title") == (title or wp_id) + and existing_same_repo.get("id") == desired_ws_id + ): + ws_data = existing_same_repo + report.fixes_applied.append( + f"C-06 reusing existing workstream {ws_data['id'][:8]}... for {wp_id}" + ) + break + last_error = ( + f"slug {slug!r} already belongs to a different workplan identity" + ) + continue + + create_flavor = normalize_flavor(meta.get("flavor")) + if create_flavor is not None and create_flavor not in WORK_RECORD_FLAVORS: + create_flavor = None + create_reason = normalize_promotion_reason(meta.get("flavor_promotion_reason")) + if create_reason is not None and create_reason not in FLAVOR_PROMOTION_REASONS: + create_reason = None + create_from = normalize_flavor(meta.get("flavor_promoted_from")) + if create_from is not None and create_from not in WORK_RECORD_FLAVORS: + create_from = None + ws_data = _api_post(api_base, "/workplans", { + "id": desired_ws_id, + "topic_id": topic_id, + "repo_id": repo_id_val, + "slug": slug, + "title": title or wp_id, + "status": status, + "owner": str(meta.get("owner", "")).strip() or None, + "planning_priority": str(meta.get("planning_priority", "")).strip() or None, + "planning_order": _as_int_or_none(meta.get("planning_order")), + "flavor": create_flavor, + "flavor_promotion_reason": create_reason, + "flavor_promoted_from": create_from, + }) + if ws_data is None or (isinstance(ws_data, dict) and "_error" in ws_data): + last_error = ws_data.get("_error") if isinstance(ws_data, dict) else "no response" + ws_data = None + continue + break + + if ws_data is None: + report.fixes_applied.append( + f"C-06 FAIL {wp_id}: could not create workstream in DB: {last_error or 'no usable slug'}" + ) + continue + + new_ws_id = ws_data["id"] + if new_ws_id != desired_ws_id: + report.fixes_applied.append( + f"C-06 FAIL {wp_id}: projection returned unexpected UUID {new_ws_id}" + ) + continue + if not meta.get("state_hub_workstream_id"): + _add_frontmatter_field(wp_file, "state_hub_workstream_id", new_ws_id) + report.fixes_applied.append( + f"C-06 fixed: created workstream {new_ws_id[:8]}… " + f"for {wp_id}, wrote ID to {wp_file.name}" + ) + + # Create tasks and inject IDs + tasks = get_tasks_from_workplan(meta, body) + for task in tasks: + if task.get("_parse_error"): + continue + t_id = str(task.get("id", "")).strip() + if not t_id: + continue + t_status = normalise_task_status(task.get("status", "todo")) + t_priority = str(task.get("priority", "medium")).strip() + if t_priority not in VALID_TASK_PRIORITIES: + t_priority = "medium" + raw_task_id = task.get("state_hub_task_id") + desired_task_id = ( + str(raw_task_id).strip().strip('"') + if raw_task_id not in (None, "", "~", "null", "None", "none") + else _derived_work_record_uuid(t_id) + ) + task_flavor = normalize_flavor(task.get("flavor")) + if task_flavor is not None and task_flavor not in WORK_RECORD_FLAVORS: + task_flavor = None + t_data = _api_post(api_base, "/tasks", { + "id": desired_task_id, + "workplan_id": new_ws_id, + "title": str(task.get("title", t_id)).strip() or t_id, + "description": task.get("description") or None, + "status": t_status, + "priority": t_priority, + "assignee": task.get("assignee") or None, + "flavor": task_flavor, + **_task_wait_fields(task), + }) + if t_data and "_error" not in t_data: + t_db_id = t_data["id"] + if t_db_id != desired_task_id: + report.fixes_applied.append( + f" ! task {t_id} returned unexpected UUID {t_db_id}" + ) + continue + if not raw_task_id: + injected = _inject_task_id_into_block( + wp_file, "state_hub_task_id", t_db_id, t_id + ) + if not injected: + _inject_task_id_frontmatter_list(wp_file, t_db_id, t_id) + report.fixes_applied.append(f" + task {t_id} → {t_db_id[:8]}…") + elif t_data: + report.fixes_applied.append( + f" ! task {t_id} not created: {t_data.get('_error', t_data)}" + ) + + elif issue.check_id == "C-32": + md_path = ctx["md_path"] + kind = ctx["kind"] + rid = ctx["rid"] + meta = ctx["meta"] + id_field = _WORK_RECORD_ID_FIELD_BY_KIND[kind] + title = str(meta.get("title") or rid) + desired_record_id = ctx.get("desired_record_id") + + repo_record = _api_get(api_base, f"/repos/{repo_slug}") + if not repo_record: + report.fixes_applied.append( + f"C-32 SKIP {rid}: could not look up repo '{repo_slug}'" + ) + continue + created = None + if kind == "intake": + payload = { + "id": desired_record_id, + "title": title, + "repo_id": repo_record.get("id"), + "description": meta.get("description"), + "lane": meta.get("lane", "green"), + "origin": meta.get("origin"), + "origin_ref": meta.get("origin_ref"), + "source_repo_path": str(md_path.relative_to(Path(report.repo_path))), + } + created = _api_post(api_base, "/intakes", payload) + elif kind == "decision": + repo_market_domain = str(repo_record.get("domain_slug") or "").strip() + topic_domain = resolve_topic_domain_slug( + repo_market_domain, repo_market_domain=repo_market_domain or None + ) + topics = _api_get(api_base, "/topics") + topic_id = None + if isinstance(topics, list): + for t in topics: + if t.get("domain_slug") == topic_domain: + topic_id = t["id"] + break + if topic_id is None: + report.fixes_applied.append( + f"C-32 SKIP {rid}: no topic found for domain '{topic_domain}'" + ) + continue + payload = { + "id": desired_record_id, + "title": title, + "topic_id": topic_id, + "decision_type": normalise_decision_type(meta), + "description": meta.get("description"), + "rationale": meta.get("agent_recommendation") or meta.get("rationale"), + } + created = _api_post(api_base, "/decisions", payload) + + if not created or "_error" in created: + report.fixes_applied.append( + f"C-32 FAIL {rid}: could not create {kind} in DB: " + f"{(created or {}).get('_error', 'no response')}" + ) + continue + + new_id = created["id"] + if desired_record_id and new_id != desired_record_id: + report.fixes_applied.append( + f"C-32 FAIL {rid}: projection returned unexpected UUID {new_id}" + ) + continue + if desired_record_id: + report.fixes_applied.append( + f"C-32 fixed: created {kind} {new_id[:8]}… for {rid} " + "with the authoritative ID" + ) + continue + if _inject_yaml_block_field(md_path, id_field, new_id, rid): + report.fixes_applied.append( + f"C-32 fixed: created {kind} {new_id[:8]}… for {rid}, " + f"wrote {id_field} to {md_path.name}" + ) + else: + report.fixes_applied.append( + f"C-32 WARN {rid}: created {kind} {new_id[:8]}… but " + f"failed to write {id_field} back to {md_path.name}" + ) + + elif issue.check_id == "C-33": + ctx["index_file"].write_text(ctx["content"], encoding="utf-8") + report.fixes_applied.append( + f"C-33 fixed: regenerated {ctx['index_file'].name}" + ) + + elif issue.check_id == "C-09": + ws_id = ctx["ws_id"] + correct_repo_id = ctx["correct_repo_id"] + result = _api_patch(api_base, f"/workplans/{ws_id}", + {"repo_id": correct_repo_id}) + if result is not None: + report.fixes_applied.append( + f"C-09 fixed: workstream {ws_id[:8]}… " + f"repo_id → {correct_repo_id[:8]}…" + ) + + elif issue.check_id == "C-20": + from_workstream_id = ctx["from_workstream_id"] + body = { + "to_workstream_id": ctx.get("to_workstream_id"), + "to_task_id": ctx.get("to_task_id"), + "relationship_type": ctx["relationship_type"], + } + if ctx.get("from_task_id"): + # A hub without migration e8f9a0b1c2d3 ignores the field + # and would store a workplan-level edge that later + # duplicates the task-scoped one. Skip until deployed. + if not _hub_accepts_from_task_id(api_base): + report.fixes_applied.append( + f"C-20 SKIP: hub at {api_base} does not accept " + f"from_task_id yet (task '{ctx.get('description', '')}')" + ) + continue + body["from_task_id"] = ctx["from_task_id"] + body["description"] = ctx.get("description") + result = _api_post(api_base, f"/workplans/{from_workstream_id}/dependencies", body) + if result is not None and "_error" not in result: + target = ctx.get("to_workstream_id") or ctx.get("to_task_id") + report.fixes_applied.append( + f"C-20 fixed: dependency {from_workstream_id[:8]}… " + f"{ctx['relationship_type']} → {target[:8]}…" + ) + elif result is not None: + report.fixes_applied.append( + f"C-20 FAILED: {result['_error']}" + ) + + elif issue.check_id == "C-10": + task_id = ctx["task_id"] + status = ctx["status"] + result = _api_patch(api_base, f"/tasks/{task_id}", + {"status": status, "suppress_token_event": True, + **ctx.get("wait_fields", {})}) + if result is not None and "_error" not in result: + report.fixes_applied.append( + f"C-10 fixed: task {task_id[:8]}… status → {status!r}" + ) + elif result is not None: + report.fixes_applied.append( + f"C-10 FAILED: task {task_id[:8]}… status → {status!r}: {result['_error']}" + ) + + elif issue.check_id == "C-11": + ws_id = ctx["ws_id"] + ws_status = ctx.get("ws_status", "") + task = ctx["task"] + wp_file = Path(ctx["wp_file"]) + meta = ctx["meta"] + body = ctx.get("body", "") + t_id = str(task.get("id", "")).strip() + # Skip creating tasks for finished workstreams — the workstream is + # done/archived so unlinked tasks are stale file artefacts, not gaps. + if normalise_workstream_status(ws_status) in CLOSED_WORKSTREAM_STATUSES: + report.fixes_applied.append( + f"C-11 skipped: task '{t_id}' in {ws_status} workstream — not created" + ) + else: + t_status = normalise_task_status(task.get("status", "todo")) + t_priority = str(task.get("priority", "medium")).strip() + if t_priority not in VALID_TASK_PRIORITIES: + t_priority = "medium" + desired_task_id = str( + ctx.get("desired_task_id") or _derived_work_record_uuid(t_id) + ) + t_data = _api_post(api_base, "/tasks", { + "id": desired_task_id, + "workplan_id": ws_id, + "title": str(task.get("title", t_id)).strip() or t_id, + "description": task.get("description") or None, + "status": t_status, + "priority": t_priority, + "assignee": task.get("assignee") or None, + **_task_wait_fields(task), + }) + if t_data: + t_db_id = t_data["id"] + if t_db_id != desired_task_id: + report.fixes_applied.append( + f"C-11 FAIL: task '{t_id}' returned unexpected UUID {t_db_id}" + ) + continue + if not task.get("state_hub_task_id"): + injected = _inject_task_id_into_block( + wp_file, "state_hub_task_id", t_db_id, t_id + ) + if not injected: + _inject_task_id_frontmatter_list(wp_file, t_db_id, t_id) + report.fixes_applied.append( + f"C-11 fixed: task '{t_id}' → {t_db_id[:8]}…" + ) + + elif issue.check_id == "C-12": + task_id = ctx["task_id"] + if ctx.get("ws_finished"): + result = _api_patch(api_base, f"/tasks/{task_id}", {"status": "cancel"}) + if result is not None: + report.fixes_applied.append( + f"C-12 fixed: orphan task {task_id[:8]}… canceled (workstream finished)" + ) + + elif issue.check_id == "C-22": + task_id = ctx["task_id"] + description = ctx["description"] + result = _api_patch(api_base, f"/tasks/{task_id}", {"description": description}) + if result is not None and "_error" not in result: + report.fixes_applied.append( + f"C-22 fixed: task {task_id[:8]}… description updated" + ) + elif result is not None: + report.fixes_applied.append( + f"C-22 FAILED: task {task_id[:8]}… description update: {result['_error']}" + ) + + elif issue.check_id == "C-38": + task_id = ctx["task_id"] + flavor = ctx.get("flavor") + result = _api_patch(api_base, f"/tasks/{task_id}", {"flavor": flavor}) + if result is not None and "_error" not in result: + report.fixes_applied.append( + f"C-38 fixed: task {task_id[:8]}… flavor → {flavor!r}" + ) + elif result is not None: + report.fixes_applied.append( + f"C-38 FAILED: task {task_id[:8]}… flavor → {flavor!r}: {result['_error']}" + ) + + elif issue.check_id == "C-41": + task_id = ctx["task_id"] + wait_fields = ctx["wait_fields"] + result = _api_patch(api_base, f"/tasks/{task_id}", wait_fields) + if result is not None and "_error" not in result: + report.fixes_applied.append( + f"C-41 fixed: task {task_id[:8]}… wait qualifiers synced" + ) + elif result is not None: + report.fixes_applied.append( + f"C-41 FAILED: task {task_id[:8]}… wait qualifiers: {result['_error']}" + ) + + elif issue.check_id == "C-15": + # T03 — writeback: DB is ahead of file — patch file to match DB. + if no_writeback: + report.fixes_applied.append( + f"C-15 skipped (--no-writeback): task '{ctx['task_block_id']}' " + f"db={ctx['db_status']!r}" + ) + else: + wp_file = Path(ctx["wp_file"]) + task_block_id = ctx["task_block_id"] + db_status = ctx["db_status"] + old_status = issue.file_value + # RMGR-WP-0002 dual-run: prefer repo-manager writeback when flagged + rm_done = False + try: + from api.services.repo_manager_dual_run import ( + try_writeback_for_task, + writeback_for_repo, + ) + + if writeback_for_repo(repo_slug) and repo_path: + # Prefer hub UUID when available for robust matching + task_ref = ctx.get("task_id") or task_block_id + rm_result = try_writeback_for_task( + repo_path=repo_path, + repo_slug=repo_slug, + task_id=str(task_ref), + status=db_status, + reason="state-hub C-15 dual-run writeback", + ) + if rm_result and rm_result.get("status") == "applied": + git_sha = (rm_result.get("evidence") or {}).get("git_sha") + suffix = f" (repo-manager git_sha={git_sha})" if git_sha else " (repo-manager)" + report.fixes_applied.append( + f"C-15 fixed: task '{task_block_id}' " + f"{old_status} → {db_status}{suffix}" + ) + rm_done = True + except Exception as rm_exc: # noqa: BLE001 + report.fixes_applied.append( + f"C-15 dual-run fallback to native: {rm_exc}" + ) + if not rm_done: + if _patch_task_status_in_file(wp_file, task_block_id, db_status): + committed = _git_commit_writeback( + repo_path, + wp_file, + [f"{task_block_id}: {old_status} → {db_status}"], + ) + try: + from api.services.repo_manager_dual_run import record_mutation + + record_mutation( + source="state-hub", + kind="task_status_writeback_c15", + repo_slug=repo_slug, + detail={ + "task_block_id": task_block_id, + "status": db_status, + "committed": committed, + }, + ) + except Exception: + pass + suffix = " (committed)" if committed else " (file patched, commit failed)" + report.fixes_applied.append( + f"C-15 fixed: task '{task_block_id}' " + f"{old_status} → {db_status}{suffix}" + ) + else: + report.fixes_applied.append( + f"C-15 SKIP: could not locate task block '{task_block_id}' " + f"in {wp_file.name}" + ) + + except Exception as e: + report.fixes_applied.append(f"{issue.check_id} ERROR: {e}") + + # Record that a sync run happened for this repo + from datetime import timezone as _tz + import datetime as _dt + now_iso = _dt.datetime.now(_tz.utc).isoformat() + _api_patch(api_base, f"/repos/{repo_slug}", {"last_state_synced_at": now_iso}) + + # Write the worker orientation brief (.custodian-brief.md) + if repo_path: + brief_written = _write_custodian_brief(api_base, repo_slug, repo_path) + if brief_written: + report.fixes_applied.append("brief: .custodian-brief.md updated") + + if repo_path: + wm_committed = _commit_working_memory_notes(repo_slug, repo_path) + if wm_committed: + report.fixes_applied.append("working-memory: daily-triage notes committed") + + # Push all commits made this run (writebacks + brief) to close the loop. + # This ensures the next run starts with local == remote, making the + # timer idempotent and preventing commit pile-up. + if repo_path: + push_ok, push_msg = _git_push(repo_path) + if push_ok: + report.fixes_applied.append(f"push: {push_msg}") + else: + report.fixes_applied.append(f"push WARN: {push_msg}") + + return report + + +# Check IDs that are known-background noise in multi-machine setups: +# C-08 = finished/archived DB workstream with no file (pre-ADR-001 legacy) +# These alone do not warrant a pull+fix cycle. +_BACKGROUND_CHECKS: frozenset[str] = frozenset({"C-08"}) + + +def _report_needs_action( + report: ConsistencyReport, behind_remote: bool, ahead_of_remote: int = 0 +) -> bool: + """Return True if the repo warrants a pull+fix cycle. + + A repo is considered clean (no action needed) when: + - It is not behind its remote tracking branch, AND + - It has no unpushed local commits (from a prior failed push), AND + - It has no FAIL issues, AND + - Every WARN/INFO issue is in the background-noise set (C-08). + """ + if behind_remote or ahead_of_remote > 0: + return True + if report.assessment_failures or report.automation_errors: + return True + actionable_warns = [ + i for i in report.warnings + report.infos + if i.check_id not in _BACKGROUND_CHECKS + ] + return bool(actionable_warns) + + +def fix_all_remote( + api_base: str, + no_writeback: bool = False, + max_seconds: int = DEFAULT_REMOTE_ALL_MAX_SECONDS, +) -> list[ConsistencyReport]: + """Pull-then-fix all registered repos that need attention. + + For each repo with a resolvable local path on this machine: + 1. Skip if the path does not exist (repo lives on another machine). + 2. Run check_repo() and detect_behind_remote() — read-only. + 3. If the repo is clean (no actionable issues, not behind remote): skip. + 4. Otherwise: git pull --ff-only, then fix_repo(). + + Returns one ConsistencyReport per repo that was *not* skipped as clean. + Repos skipped as clean are printed to stdout but not included in the return value. + """ + repos = _api_get(api_base, "/repos") + if not isinstance(repos, list): + print("ERROR: Could not fetch repos from state-hub API", file=sys.stderr) + return None + + started = time.monotonic() + reports: list[ConsistencyReport] = [] + skipped_clean: list[str] = [] + skipped_missing: list[str] = [] + skipped_budget: list[str] = [] + + for repo in repos: + slug = repo["slug"] + if max_seconds > 0 and time.monotonic() - started > max_seconds: + skipped_budget.append(slug) + skipped_budget.extend(r.get("slug", "?") for r in repos[repos.index(repo) + 1:]) + break + + # Resolve path using the same priority as check_repo + path = resolve_repo_path(repo) + if not path or not Path(path).is_dir(): + skipped_missing.append(slug) + continue + + # Read-only pass: detect issues and remote staleness. + # _detect_behind_remote does a fetch, so _detect_ahead_of_remote + # after it sees an up-to-date tracking branch. + pre_report = check_repo(api_base, slug) + behind = _detect_behind_remote(path) + ahead = _detect_ahead_of_remote(path) + + if not _report_needs_action(pre_report, behind, ahead): + skipped_clean.append(slug) + continue + + # Pull before fixing + pull_ok, pull_msg = _git_pull(path) + pull_tag = f"pull: {pull_msg}" + if not pull_ok: + pull_tag = f"pull WARN: {pull_msg} — proceeding anyway" + + report = fix_repo(api_base, slug, no_writeback=no_writeback) + report.fixes_applied.insert(0, pull_tag) + reports.append(report) + + # Print clean/missing summary lines before the detailed reports + if skipped_clean: + print(f" CLEAN (skipped): {', '.join(skipped_clean)}") + if skipped_missing: + print(f" NOT ON THIS HOST (skipped): {', '.join(skipped_missing)}") + if skipped_budget: + print( + f" BUDGET EXHAUSTED after {max_seconds}s (skipped): " + f"{', '.join(skipped_budget)}" + ) + if skipped_clean or skipped_missing or skipped_budget: + print() + + return reports + + +def archive_closed_workplans( + repo_path: str, + completion_date: str | None = None, + workplan: str | None = None, +) -> list[str]: + """Move closed root workplans into workplans/archived/ with YYMMDD prefix. + + Only root-level files whose frontmatter status normalises to finished or + archived are moved. Files with any open task blocks are left in place. + """ + repo_dir = Path(repo_path) + workplans_dir = repo_dir / "workplans" + archived_dir = workplans_dir / "archived" + if not workplans_dir.is_dir(): + return [] + + date_prefix = completion_date or datetime.now().strftime("%y%m%d") + archived_dir.mkdir(exist_ok=True) + moved: list[str] = [] + + for wp_file in sorted(workplans_dir.glob("*.md")): + text = wp_file.read_text(encoding="utf-8") + if not text.startswith("---"): + continue + meta, body = parse_frontmatter(text) + if not meta or meta.get("_parse_error"): + continue + if workplan: + wanted = workplan.removesuffix(".md") + if wanted not in {str(meta.get("id", "")), wp_file.stem, wp_file.name}: + continue + status = normalise_workstream_status(str(meta.get("status", "")).strip()) + if status not in CLOSED_WORKSTREAM_STATUSES: + continue + tasks = get_tasks_from_workplan(meta, body) + open_tasks = [ + t for t in tasks + if normalise_task_status(t.get("status", "todo")) not in TERMINAL_TASK_STATUSES + ] + if open_tasks: + continue + + target = archived_dir / f"{date_prefix}-{canonical_workplan_filename(wp_file)}" + if target.exists(): + raise FileExistsError(f"Archived workplan already exists: {target}") + wp_file.rename(target) + moved.append(f"{wp_file.relative_to(repo_dir)} -> {target.relative_to(repo_dir)}") + + return moved + + +# --------------------------------------------------------------------------- +# Output / rendering +# --------------------------------------------------------------------------- + +def render_renormalization_guide() -> str: + lines = ["Lifecycle Renormalization Rules", "=" * 33] + for rule in RENORMALIZATION_RULES: + lines.extend( + [ + "", + f"{rule.check_id}: {rule.invariant}", + f" Detect: {rule.detection}", + f" Repair: {rule.repair}", + f" Tests: {rule.test_anchor}", + ] + ) + lines.extend(["", "Add The Next Guard"]) + for idx, item in enumerate(RENORMALIZATION_NEXT_GUARD_CHECKLIST, 1): + lines.append(f" {idx}. {item}") + return "\n".join(lines) + + +def render_text(report: ConsistencyReport, show_info: bool = True) -> str: + SEP = "=" * 66 + lines = [ + f"ADR-001 Consistency Report", + f"Repo: {report.repo_slug}", + f"Path: {report.repo_path}", + SEP, + ] + + error_section = report.automation_errors + fail_section = report.assessment_failures + if error_section: + lines.append(f"\n AUTOMATION ERRORS ({len(error_section)}):") + for i in error_section: + loc = f" [{i.file_path}]" if i.file_path else "" + lines.append(f" {i.check_id}{loc}") + lines.append(f" {i.message}") + + if fail_section: + lines.append(f"\n ASSESSMENT FAILURES ({len(fail_section)}):") + for i in fail_section: + loc = f" [{i.file_path}]" if i.file_path else "" + fix_tag = " [fixable]" if i.fixable else "" + lines.append(f" {i.check_id}{loc}{fix_tag}") + lines.append(f" {i.message}") + if i.file_value or i.db_value: + lines.append(f" file={i.file_value!r} db={i.db_value!r}") + + for sev in ("WARN", "INFO"): + section = [i for i in report.issues if i.severity == sev] + if not section or (sev == "INFO" and not show_info): + continue + lines.append(f"\n {sev}S ({len(section)}):") + for i in section: + loc = f" [{i.file_path}]" if i.file_path else "" + fix_tag = " [fixable]" if i.fixable else "" + lines.append(f" {i.check_id}{loc}{fix_tag}") + lines.append(f" {i.message}") + if i.file_value or i.db_value: + lines.append(f" file={i.file_value!r} db={i.db_value!r}") + + if report.fixes_applied: + lines.append(f"\n FIXES APPLIED ({len(report.fixes_applied)}):") + for f in report.fixes_applied: + lines.append(f" {f}") + + lines.append(f"\n{SEP}") + n_err = len(report.automation_errors) + n_fail = len(report.assessment_failures) + n_warn = len(report.warnings) + n_info = len(report.infos) + lines.append( + f" {n_err} automation-error | {n_fail} assessment-fail | " + f"{n_warn} warn | {n_info} info" + ) + if n_err: + lines.append(" RESULT: ✗ AUTOMATION ERROR") + elif n_fail: + lines.append(" RESULT: ✗ ASSESSMENT FAIL (follow-up needed)") + elif n_warn: + lines.append(" RESULT: ✓ PASS (with warnings)") + else: + lines.append(" RESULT: ✓ PASS") + lines.append(SEP) + return "\n".join(lines) + + +def report_to_dict(report: ConsistencyReport) -> dict: + revision = { + "source_revision": report.source_revision, + "source_revision_reason": report.source_revision_reason, + } + return { + "repo_slug": report.repo_slug, + "repo_path": report.repo_path, + **revision, + "issues": [ + { + "severity": i.severity, + "check_id": i.check_id, + "message": i.message, + "file_path": i.file_path, + "db_id": i.db_id, + "file_value": i.file_value, + "db_value": i.db_value, + "fixable": i.fixable, + **({"repo_slug": report.repo_slug, **revision} if i.check_id == "C-25" else {}), + **({"owner_return": i._fix_context} if i.check_id == "C-25" and i._fix_context.get("task_record_id") else {}), + } + for i in report.issues + ], + "fixes_applied": report.fixes_applied, + "summary": { + "fail": len(report.assessment_failures), + "automation_error": len(report.automation_errors), + "warn": len(report.warnings), + "info": len(report.infos), + }, + "result": ( + "error" if report.automation_errors else + "fail" if report.assessment_failures else + "warn" if report.warnings else + "pass" + ), + } + + +def consistency_exit_code(reports: list[ConsistencyReport], *, remote_all: bool = False) -> int: + """Return the strict CLI exit code for consistency reports.""" + any_automation_error = any(r.automation_errors for r in reports) + any_assessment_fail = any(r.assessment_failures for r in reports) + any_warn = any(r.warnings for r in reports) + if remote_all: + return 1 if any_automation_error else 0 + if any_automation_error or any_assessment_fail: + return 1 + return 2 if any_warn else 0 + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser( + description="ADR-001 consistency checker — bidirectional file↔DB validation", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--repo", metavar="SLUG", + help="Registered repo slug (e.g. the-custodian)") + group.add_argument("--all", action="store_true", + help="Run checks against all repos with a resolvable path on this host") + group.add_argument("--here", metavar="PATH", nargs="?", const="", + help="Infer repo slug from git remote URL at PATH (default: CWD)") + group.add_argument("--renormalization-guide", action="store_true", + help="Print lifecycle renormalization rules and the add-next-guard checklist") + parser.add_argument("--fix", action="store_true", + help="Apply auto-fixable issues (status drift, repo mismatch, etc.)") + parser.add_argument("--remote", action="store_true", + help="Pull each repo before fixing; when used with --all, skips repos " + "that are already clean (no actionable issues, not behind remote). " + "Implies --fix.") + parser.add_argument("--max-seconds", type=int, default=DEFAULT_REMOTE_ALL_MAX_SECONDS, + help="Wall-clock budget for --remote --all before remaining repos are skipped " + f"(default: {DEFAULT_REMOTE_ALL_MAX_SECONDS}; 0 disables)") + parser.add_argument("--no-writeback", action="store_true", dest="no_writeback", + help="Disable DB→file status writeback (C-15) while keeping other fixes") + parser.add_argument( + "--bootstrap-empty-projection", + action="store_true", + help=( + "Rebuild authoritative legacy UUIDs only after proving the target " + "repository has zero projected workplans" + ), + ) + parser.add_argument("--archive-closed", action="store_true", + help="Move closed root workplans to workplans/archived/YYMMDD-*.md") + parser.add_argument("--archive-workplan", metavar="ID_OR_FILE", default=None, + help="When archiving, only move the matching workplan id or filename") + parser.add_argument("--archive-date", metavar="YYMMDD", default=None, + help="Completion date prefix for --archive-closed (default: today)") + parser.add_argument("--repo-path", metavar="PATH", default=None, + help="Override the local repo path (useful when the DB has a different " + "machine's path). Takes priority over host_paths and local_path.") + parser.add_argument("--api-base", default="http://127.0.0.1:8000", + help="State Hub API base URL") + parser.add_argument("--json", action="store_true", dest="as_json", + help="Output JSON instead of human-readable text") + args = parser.parse_args() + + if args.renormalization_guide: + if args.as_json: + print(json.dumps({ + "rules": [ + { + "check_id": rule.check_id, + "invariant": rule.invariant, + "detection": rule.detection, + "repair": rule.repair, + "test_anchor": rule.test_anchor, + } + for rule in RENORMALIZATION_RULES + ], + "add_next_guard": list(RENORMALIZATION_NEXT_GUARD_CHECKLIST), + }, indent=2)) + else: + print(render_renormalization_guide()) + sys.exit(0) + + import os as _os + no_wb = getattr(args, "no_writeback", False) + do_fix = args.fix or args.remote + if args.bootstrap_empty_projection and (not do_fix or args.all or args.remote): + parser.error( + "--bootstrap-empty-projection requires --fix for one local repository " + "and cannot be combined with --all or --remote" + ) + + # --here: infer slug from git remote URL, then run as single-repo check/fix + if args.here is not None: + search_path = args.here or _os.getcwd() + inferred = _infer_slug_from_path(args.api_base, search_path) + if inferred is None: + print( + f"ERROR: No registered repo with a matching remote_url found at '{search_path}'.", + file=sys.stderr, + ) + print( + " Register the repo first: POST /repos/ with remote_url set.", + file=sys.stderr, + ) + sys.exit(1) + inferred_slug, git_root = inferred + print(f" Detected: {inferred_slug} ({git_root})") + if do_fix: + reports = [ + fix_repo( + args.api_base, + inferred_slug, + git_root, + no_writeback=no_wb, + bootstrap_empty_projection=args.bootstrap_empty_projection, + ) + ] + else: + reports = [check_repo(args.api_base, inferred_slug, git_root)] + if args.archive_closed: + moved = archive_closed_workplans(git_root, args.archive_date, args.archive_workplan) + reports[0].fixes_applied.extend(f"archive: {m}" for m in moved) + # --remote --all: smart pull+fix across all repos + elif args.remote and args.all: + with run_lock("consistency-remote-all") as acquired: + if not acquired: + print( + "SKIP: another fix-consistency-remote --all run is already active", + file=sys.stderr, + ) + sys.exit(0) + reports = fix_all_remote( + args.api_base, + no_writeback=no_wb, + max_seconds=args.max_seconds, + ) + if reports is None: + sys.exit(1) + if not reports: + sys.exit(0) + else: + # Resolve repo list + hostname = _sweep_hostname() + repo_slugs: list[str] = [] + if args.all: + repos = _api_get(args.api_base, "/repos") + if not isinstance(repos, list): + print("ERROR: Could not fetch repos from state-hub API", file=sys.stderr) + sys.exit(1) + repo_slugs = [ + r["slug"] for r in repos + if r.get("local_path") or (r.get("host_paths") or {}).get(hostname) + ] + if not repo_slugs: + print( + f"No repos with a path registered for host '{hostname}'.", + file=sys.stderr, + ) + sys.exit(0) + else: + repo_slugs = [args.repo] + + # --repo-path only applies to single-repo runs; silently ignored with --all + path_override = args.repo_path if not args.all else None + + if args.remote: + # Single-repo remote: pull first, then fix + slug = repo_slugs[0] + repo = _api_get(args.api_base, f"/repos/{slug}") + path = resolve_repo_path(repo or {}, path_override) if repo else (path_override or "") + if path: + pull_ok, pull_msg = _git_pull(path) + prefix = "pull" if pull_ok else "pull WARN" + print(f" {prefix}: {pull_msg}") + report = fix_repo(args.api_base, slug, path_override, no_writeback=no_wb) + reports = [report] + elif do_fix: + reports = [ + fix_repo( + args.api_base, + slug, + path_override, + no_writeback=no_wb, + bootstrap_empty_projection=args.bootstrap_empty_projection, + ) + for slug in repo_slugs + ] + else: + reports = [check_repo(args.api_base, slug, path_override) for slug in repo_slugs] + + if args.archive_closed: + for report in reports: + moved = archive_closed_workplans(report.repo_path, args.archive_date, args.archive_workplan) + report.fixes_applied.extend(f"archive: {m}" for m in moved) + + if args.as_json: + output = ( + report_to_dict(reports[0]) + if len(reports) == 1 + else [report_to_dict(r) for r in reports] + ) + print(json.dumps(output, indent=2)) + else: + for report in reports: + print(render_text(report)) + print() + + sys.exit(consistency_exit_code(reports, remote_all=args.remote and args.all)) + + +if __name__ == "__main__": + main() diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/scripts/sync_classification_allowed.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/scripts/sync_classification_allowed.py new file mode 100644 index 00000000..322770f5 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/owner-followthrough-r3/state-hub/scripts/sync_classification_allowed.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Sync the canon classification allowed-values into the chart (CUST-WP-0067-T09). + +The State Hub validates repository classification against +``the-custodian/canon/standards/repo-classification.allowed.yaml``. That file is +canon and must stay authoritative, but a container has no the-custodian +checkout, so a copy has to travel with the release. + +A copy that nobody checks is a copy that silently drifts. This script owns the +copy: ``--check`` fails when it diverges from canon, so the release can refuse +to ship a stale vocabulary rather than validate against yesterday's rules. + +Usage: + python scripts/sync_classification_allowed.py # write the copy + python scripts/sync_classification_allowed.py --check # verify, exit 1 on drift +""" +from __future__ import annotations + +import argparse +import difflib +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +CHART_COPY = ( + REPO_ROOT + / "deploy/railiance/apps/charts/state-hub/files/repo-classification.allowed.yaml" +) +CANON_CANDIDATES = ( + Path("/home/worsch/the-custodian/canon/standards/repo-classification.allowed.yaml"), + Path("/home/tegwick/the-custodian/canon/standards/repo-classification.allowed.yaml"), + REPO_ROOT.parent / "the-custodian/canon/standards/repo-classification.allowed.yaml", +) + +HEADER = ( + "# GENERATED — do not edit.\n" + "# Synced from the-custodian/canon/standards/repo-classification.allowed.yaml\n" + "# by scripts/sync_classification_allowed.py (CUST-WP-0067-T09).\n" + "# Canon is authoritative; this copy exists only so the container has one.\n" +) + + +def find_canon() -> Path: + for candidate in CANON_CANDIDATES: + if candidate.is_file(): + return candidate + raise SystemExit( + "ERROR: canon allowed-values not found. Looked in:\n " + + "\n ".join(str(c) for c in CANON_CANDIDATES) + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="Verify the chart copy matches canon; exit 1 on drift", + ) + args = parser.parse_args() + + canon = find_canon() + desired = HEADER + canon.read_text(encoding="utf-8") + + if args.check: + if not CHART_COPY.is_file(): + print(f"DRIFT: {CHART_COPY} is missing; run without --check to create it") + return 1 + current = CHART_COPY.read_text(encoding="utf-8") + if current != desired: + print(f"DRIFT: chart copy differs from canon ({canon})") + sys.stdout.writelines( + difflib.unified_diff( + current.splitlines(keepends=True), + desired.splitlines(keepends=True), + fromfile="chart copy", + tofile="canon", + ) + ) + return 1 + print(f"OK: chart copy matches canon ({canon})") + return 0 + + CHART_COPY.parent.mkdir(parents=True, exist_ok=True) + CHART_COPY.write_text(desired, encoding="utf-8") + print(f"Synced {canon} -> {CHART_COPY}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())