diff --git a/docs/release/backup-producer-custody-validation-2026-10-31.json b/docs/release/backup-producer-custody-validation-2026-10-31.json new file mode 100644 index 0000000..9b9f7d9 --- /dev/null +++ b/docs/release/backup-producer-custody-validation-2026-10-31.json @@ -0,0 +1,62 @@ +{ + "schema": "coulomb.backup-producer-custody-validation.v1", + "completed_at": "2026-10-31T11:45:14.257216+00:00", + "status": "source_corrections_verified", + "consumer_task": "CSOC-WP-0012-T13", + "owner_task": "RAPPS-WP-0015-T06", + "owner_source_commit": "904dcb3ce32c446fe4236d3e2c5e42b409a5fa7b", + "files": [ + { + "repo": "coulomb-social", + "path": "scripts/source_date_supervisor.py", + "sha256": "d9bb0443bbbe45131d73b2d11017a67f33ee888454c79f4da0ebb0d663dd1b66" + }, + { + "repo": "coulomb-social", + "path": "tests/test_source_date_supervisor.py", + "sha256": "d2d76c4027ff3965e0dcc5e973b958858131b6919b6bca045d16c018a715b1b1" + }, + { + "repo": "railiance-apps", + "path": "tools/coulomb_image_release_node.py", + "sha256": "f6d05a9b1b055ade5195160c7793c8f1a1d72ae83b9dbd7e5fdd3cea0adb288f" + }, + { + "repo": "railiance-apps", + "path": "tests/test_coulomb_image_release.py", + "sha256": "b879132c1509ec8d35745a4ed7fb6343591580252472ad5c8610185a4806c05b" + } + ], + "gates": [ + { + "command": "make check", + "tests": 524, + "failures": 0, + "errors": 0, + "skipped": 0, + "junit_sha256": "dfa960ea3608f89d06fa7a1a492f78e1fe4ba0ff8467c26117dbf55a528d7021" + }, + { + "command": "pytest -q tests/test_coulomb_image_release.py", + "tests": 24, + "failures": 0, + "errors": 0, + "skipped": 0, + "junit_sha256": "e789fadcfe9148ead8783c692c789d7ecad7642f96da0071f01a4f43bf4ebb8c" + } + ], + "focused_supervisor_cases": 34, + "product_gate_warnings": 69, + "log_sha256": "bb1324684e86fc97221d80881d83055569d4f4300dea272d33d0464a2f725056", + "proof_limits": [ + "Kubernetes lifecycle and root/helper ownership transitions are synthetic adapters.", + "Real temporary-file descriptor, content, inode, mode and lock checks pass.", + "No production execution or native future-operation return is claimed." + ], + "historical_custody_repair_pins_unchanged": true, + "pending_historical_repair_digest": "sha256:fa2763d1dcbe92fe638a29c9a7a7d51c03d30849c4232f2d43f81ef705bae611", + "production_changed": false, + "frozen_historical_capsules_changed": false, + "timers_activated": false, + "emails_sent": false +} diff --git a/docs/release/goal-progress-audit-2026-10-31.md b/docs/release/goal-progress-audit-2026-10-31.md index 5137dac..a0592fa 100644 --- a/docs/release/goal-progress-audit-2026-10-31.md +++ b/docs/release/goal-progress-audit-2026-10-31.md @@ -48,7 +48,7 @@ extend calendar time without proportionate engineering effort. rehearsals pass. Neither preparation nor automated success supplies a productive transfer verdict. -The latest source gate is `make check`: 475 passed, zero failures/errors/skips, +At the first tracking-audit checkpoint, `make check` reported 475 passed, zero failures/errors/skips, including required browser journeys. Fifteen cases cover the source installer library. No source code changed during this tracking audit; its source hashes and result envelope are in `backup-source-installer-validation-2026-10-31.json`. @@ -123,3 +123,19 @@ The shared State Hub C-06 creation path currently chooses the first topic matchi a domain instead of the declared topic slug. That code is not changed by this projection correction; future newly registered plans need an explicit topic readback until the owner fixes that selection. + +## Producer-custody follow-through + +Subsequent installer preflight identified eight historical metadata exceptions. +The exact repair is prepared with passing native preflight, but awaits its attended +execution. A separate completed source task, WP-0012-T13, now prevents recurrence +in current release and date-operation producers; its release-owner return is +RAPPS-WP-0015-T06. Evidence: backup-producer-custody-validation-2026-10-31.json. +This makes the source checklist **12 of 31 done**. The extra task exposes a discovered +correction, not a new user-facing capability or a revised effort percentage. + +WP-0012-T07 remains wait for the reviewed recurring route execution. T08 preparation +is complete and waits on T04 historical custody/capacity and T07. T04 remains in +progress for those native returns and historical-copy disposition. The original +roughly 40% remaining estimate is unchanged: these source fixes do not establish +recurring operation, productive acceptance or full remaining-cohort transfer. diff --git a/scripts/source_date_supervisor.py b/scripts/source_date_supervisor.py index e59954f..326c965 100644 --- a/scripts/source_date_supervisor.py +++ b/scripts/source_date_supervisor.py @@ -248,7 +248,7 @@ class Node: return point def prepare_root(self, root): - root.mkdir(mode=0o700, parents=True, exist_ok=True) + root.mkdir(mode=0o700, exist_ok=True) info = root.lstat() if (not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o077 or any(p.is_symlink() for p in root.parents)): @@ -285,6 +285,68 @@ class Node: sync_directory(path) sync_directory(path.parent) + @contextmanager + def helper_evidence_files(self, path): + """Open only the closed helper's known evidence; preserve crash recovery.""" + with ExitStack() as stack: + def opened(name, flags, parent=None): + fd = os.open(name, flags | os.O_NOFOLLOW, dir_fd=parent) + stack.callback(os.close, fd) + return fd + + directory = opened("/", os.O_RDONLY | os.O_DIRECTORY) + for component in path.parts[1:]: + directory = opened(component, os.O_RDONLY | os.O_DIRECTORY, directory) + info = os.fstat(directory) + if (info.st_uid not in {os.geteuid(), 10001} + or info.st_gid not in {os.getegid(), 10001} + or stat.S_IMODE(info.st_mode) != 0o700): + raise Refused("private_helper_evidence_required") + names = sorted(os.listdir(directory)) + if set(names) - {"execute.lock", "journal.json", "result.json"}: + raise Refused("unexpected_helper_evidence") + files, evidence = {}, {} + for name in names: + fd = opened(name, os.O_RDONLY | os.O_NONBLOCK, directory) + info = os.fstat(fd) + if (not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 + or info.st_uid not in {os.geteuid(), 10001} + or info.st_gid not in {os.getegid(), 10001} + or stat.S_IMODE(info.st_mode) != 0o600): + raise Refused("private_helper_evidence_required") + if name == "execute.lock": + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + digest = hashlib.sha256() + while chunk := os.read(fd, 1024 * 1024): + digest.update(chunk) + after = os.fstat(fd) + if any(getattr(info, key) != getattr(after, key) for key in ( + "st_dev", "st_ino", "st_mode", "st_uid", "st_gid", "st_nlink", + "st_size", "st_mtime_ns", "st_ctime_ns")): + raise Refused("helper_evidence_changed") + files[name] = fd + evidence[name] = {"sha256": digest.hexdigest(), "bytes": info.st_size, + "device": info.st_dev, "inode": info.st_ino, + "mtime_ns": info.st_mtime_ns} + yield directory, files, evidence + + def helper_evidence(self, path): + with self.helper_evidence_files(path) as (_, _, evidence): + return evidence + + def seal_helper_evidence(self, path, expected): + # Mixed helper/root custody is allowed only to resume this recorded step. + with self.helper_evidence_files(path) as (directory, files, evidence): + if evidence != expected: + raise Refused("helper_evidence_changed") + for fd in [*files.values(), directory]: + os.fchown(fd, os.geteuid(), os.getegid()) + os.fsync(fd) + info = os.fstat(fd) + if (info.st_uid, info.st_gid) != (os.geteuid(), os.getegid()): + raise Refused("helper_evidence_custody_unconfirmed") + sync_directory(path.parent) + def read_state(self, path): return json.loads(private_file(path)) @@ -492,7 +554,27 @@ class Supervisor: self.node.command(["kubectl", "rollout", "status", f"deployment/{NS}", "-n", NS, "--timeout=180s"]) self.deployment(1) - self.record("completed", application_resumed=True) + return self.complete() + + def complete(self, **changes): + """Seal evidence only after resume, when helper recovery is no longer needed.""" + self.deployment(1) + path = self.directory / "private" + for pod in self.node.doc("get", "pods", "-A")["items"]: + if pod["metadata"]["name"] == self.helper: + raise Refused("helper_still_present_at_closeout") + for volume in pod["spec"].get("volumes", []): + if "hostPath" in volume: + mount = Path(volume["hostPath"]["path"]) + if mount == path or mount in path.parents or path in mount.parents: + raise Refused("helper_evidence_still_mounted") + evidence = self.state.get("helper_evidence") + if evidence is None: + evidence = self.node.helper_evidence(path) + self.record("custody_closeout_started", application_resumed=True, + helper_evidence=evidence, **changes) + self.node.seal_helper_evidence(path, evidence) + self.record("completed", helper_evidence_root_custody_verified=True) return self.state def execute(self): @@ -559,19 +641,23 @@ class Supervisor: if (self.state.get("schema") != "coulomb.supervised-date-attempt.v1" or self.state["proposal_digest"] != self.capsule.digest): raise Refused("original_attempt_required") + if self.state["phase"] == "completed": + raise Refused("completed_attempt_must_not_be_replayed") self.identity() dep = self.node.doc("get", "deployment", NS, "-n", NS) if dep["spec"]["replicas"] == 1: # A lost response after scale-up must never undo later writes. - if (self.state.get("failed_phase", self.state["phase"]) != "resume_started" + phase = (self.state["failed_phase"] if self.state["phase"] == "requires_review" + else self.state["phase"]) + if (phase not in { + "resume_started", "custody_closeout_started"} or "expected_final" not in self.state): raise Refused("running_application_requires_owner_review") self.deployment(1) self.node.command(["kubectl", "rollout", "status", f"deployment/{NS}", "-n", NS, "--timeout=180s"]) - self.record("completed", application_resumed=True, - resumed_reply_reconciled=True, post_resume_writes_preserved=True) - return self.state + return self.complete(resumed_reply_reconciled=True, + post_resume_writes_preserved=True) self.deployment(0) writers = self.writers() if not writers: diff --git a/tests/test_source_date_supervisor.py b/tests/test_source_date_supervisor.py index ebe26b7..395b59b 100644 --- a/tests/test_source_date_supervisor.py +++ b/tests/test_source_date_supervisor.py @@ -3,6 +3,8 @@ import copy import importlib.util import json +import os +import stat from pathlib import Path import pytest @@ -205,6 +207,7 @@ def test_success_captures_fences_applies_checks_and_resumes_once(prepared): assert result["phase"] == "completed" and result["correction_status"] == "completed" assert node.capture_count == node.apply_count == 1 assert node.dep["spec"]["replicas"] == 1 and not node.helper + assert result["helper_evidence_root_custody_verified"] with pytest.raises(supervisor.Refused, match="prior_attempt"): operation.execute() assert node.apply_count == 1 @@ -365,3 +368,179 @@ def test_observer_prepares_without_capturing_or_refuses_missing_release( assert capsule.baseline == BASE and capsule.target == operation.target assert proof["baseline_rechecked"] and not proof["live_changes"] assert node.capture_count == node.apply_count == 0 and node.dep["spec"]["replicas"] == 1 + + +def test_closeout_failure_recovers_only_custody_and_preserves_new_writes(prepared, monkeypatch): + operation, node, _, _ = prepared + seal = node.seal_helper_evidence + + def lost_closeout(path, expected): + assert node.dep['spec']['replicas'] == 1 and node.helper is None + seal(path, expected) + raise supervisor.Refused('synthetic lost custody response') + + monkeypatch.setattr(node, 'seal_helper_evidence', lost_closeout) + with pytest.raises(supervisor.Refused, match='lost custody'): + operation.execute() + assert operation.state['failed_phase'] == 'custody_closeout_started' + node.fp = {**AFTER, 'rows': 115} + before = list(node.calls) + monkeypatch.setattr(node, 'seal_helper_evidence', seal) + result = operation.recover() + assert result['phase'] == 'completed' and result['helper_evidence_root_custody_verified'] + assert node.fp['rows'] == 115 and node.capture_count == node.apply_count == 1 + assert all(call[:2] not in (['kubectl', 'create'], ['kubectl', 'scale'], + ['kubectl', 'exec']) for call in node.calls[len(before):]) + with pytest.raises(supervisor.Refused, match='completed_attempt'): + operation.recover() + + +def test_uncertain_helper_evidence_is_not_sealed_before_resume(prepared, monkeypatch): + from unittest.mock import Mock + + operation, node, _, _ = prepared + seal = Mock() + monkeypatch.setattr(node, 'seal_helper_evidence', seal) + node.fail = 'apply_pending' + with pytest.raises(supervisor.Refused): + operation.execute() + seal.assert_not_called() + assert node.helper and node.dep['spec']['replicas'] == 0 + + +def evidence_directory(tmp_path): + private = tmp_path / 'private' + private.mkdir(mode=0o700) + for name, data in {'execute.lock': b'', 'journal.json': b'synthetic private journal', + 'result.json': b'synthetic result'}.items(): + file = private / name + file.write_bytes(data) + file.chmod(0o600) + return private + + +def test_evidence_sealing_preserves_bytes_paths_and_modes(tmp_path): + private = evidence_directory(tmp_path) + node = supervisor.Node() + before = node.helper_evidence(private) + node.seal_helper_evidence(private, before) + assert node.helper_evidence(private) == before + for path in [private, *private.iterdir()]: + info = path.stat() + assert (info.st_uid, info.st_gid) == (os.geteuid(), os.getegid()) + assert stat.S_IMODE(info.st_mode) == (0o700 if path.is_dir() else 0o600) + + +@pytest.mark.parametrize('problem', ['changed_bytes', 'extra', 'symlink', 'hardlink', + 'broad', 'busy', 'fifo']) +def test_evidence_closeout_refuses_drift_before_any_chown(tmp_path, monkeypatch, problem): + from unittest.mock import Mock + + private = evidence_directory(tmp_path) + node = supervisor.Node() + before = node.helper_evidence(private) + item = private / 'result.json' + lock = None + if problem == 'changed_bytes': + item.write_bytes(b'changed') + elif problem == 'extra': + (private / 'unexpected.json').write_bytes(b'unexpected') + elif problem in {'symlink', 'hardlink', 'fifo'}: + item.unlink() + if problem == 'symlink': + item.symlink_to(private / 'journal.json') + elif problem == 'hardlink': + os.link(private / 'journal.json', item) + else: + os.mkfifo(item, 0o600) + elif problem == 'broad': + item.chmod(0o644) + else: + lock = (private / 'execute.lock').open() + supervisor.fcntl.flock(lock, supervisor.fcntl.LOCK_EX | supervisor.fcntl.LOCK_NB) + chown = Mock() + monkeypatch.setattr(supervisor.os, 'fchown', chown) + try: + with pytest.raises((supervisor.Refused, OSError)): + node.seal_helper_evidence(private, before) + chown.assert_not_called() + finally: + if lock: + lock.close() + + +@pytest.mark.parametrize('overlap', ['helper', 'ancestor', 'child']) +def test_closeout_refuses_existing_helper_or_host_mount(prepared, monkeypatch, overlap): + from unittest.mock import Mock + + operation, node, _, _ = prepared + operation.execute() + original = node.doc + evidence_path = operation.directory / 'private' + pod = {'metadata': {'name': operation.helper if overlap == 'helper' else 'foreign'}, + 'spec': {'volumes': []}} + if overlap != 'helper': + path = evidence_path.parent if overlap == 'ancestor' else evidence_path / 'journal.json' + pod['spec']['volumes'] = [{'hostPath': {'path': str(path)}}] + + def documents(*args): + result = original(*args) + if args[:2] == ('get', 'pods'): + result['items'].append(pod) + return result + + monkeypatch.setattr(node, 'doc', documents) + seal = Mock() + monkeypatch.setattr(node, 'seal_helper_evidence', seal) + with pytest.raises(supervisor.Refused, match='still_'): + operation.complete() + seal.assert_not_called() + + +def test_partial_helper_ownership_transfer_resumes_from_fingerprints(tmp_path, monkeypatch): + from types import SimpleNamespace + + private = evidence_directory(tmp_path) + node = supervisor.Node() + real_stat = os.fstat + owners = {p.stat().st_ino: (10001, 10001) for p in [private, *private.iterdir()]} + changed = [] + + def observed(fd): + info = real_stat(fd) + values = {name: getattr(info, name) for name in dir(info) if name.startswith('st_')} + values['st_uid'], values['st_gid'] = owners.get(info.st_ino, (info.st_uid, info.st_gid)) + return SimpleNamespace(**values) + + def transfer(fd, uid, gid): + inode = real_stat(fd).st_ino + owners[inode] = (uid, gid) + changed.append(inode) + if len(changed) == 1: + raise OSError('synthetic interruption after first ownership change') + + monkeypatch.setattr(supervisor.os, 'fstat', observed) + monkeypatch.setattr(supervisor.os, 'fchown', transfer) + expected = node.helper_evidence(private) + with pytest.raises(OSError, match='interruption'): + node.seal_helper_evidence(private, expected) + assert len(set(owners.values())) == 2 + node.seal_helper_evidence(private, expected) + assert set(owners.values()) == {(os.geteuid(), os.getegid())} + assert node.helper_evidence(private) == expected + + +def test_second_lost_reply_during_recovery_uses_latest_phase(prepared): + operation, node, _, _ = prepared + node.fail = 'apply_reply' + with pytest.raises(supervisor.Refused): + operation.execute() + node.fail = 'resume_reply' + with pytest.raises(supervisor.Refused): + operation.recover() + assert node.dep['spec']['replicas'] == 1 + assert operation.state['phase'] == 'resume_started' + node.fp = {**AFTER, 'rows': 116} + result = operation.recover() + assert result['phase'] == 'completed' and result['post_resume_writes_preserved'] + assert node.fp['rows'] == 116 and node.capture_count == node.apply_count == 1 diff --git a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md index fa1c6d7..bc6f7f9 100644 --- a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md +++ b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md @@ -581,12 +581,11 @@ guard afterward. Nineteen dedicated cases cover refusal, real locks and rollback No historical ownership/mode change has executed. Receipt/body hashes were computed natively; no private journal body was printed or sent to inference. -Before recurring service acceptance, also prevent future producers from -reintroducing the mismatch: release capsule intermediate directories need private -creation, and completed date helper evidence needs an explicit custody closeout. -Keep frozen historical capsules unchanged; implement and test current producer -sources through their existing owners. This remains live T04 work, not a prose-only -residual. The prepared metadata repair is not full retention/capacity acceptance. +The future-producer correction is now completed source task T13, with the release +owner return RAPPS-WP-0015-T06. It prevents new intermediate-directory and completed +helper ownership exceptions. T04 still owns the exact historical eight-entry repair, +retention/capacity admission and historical-copy disposition. Source correction does +not execute that repair or establish native recurring-service acceptance. ## T05 — Verify both approved backup alert destinations @@ -701,3 +700,32 @@ Remaining T03 service-contract §6: resolve external/unattributed data, obtain p ## Completion Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records. + +## T13 — Correct backup evidence producers and recovery closeout + +```task +id: CSOC-WP-0012-T13 +status: done +priority: high +needs_human: false +``` + +Bounded source correction split from T04 after native installer preflight found +historical custody exceptions. The current railiance-apps image-release controller +creates every capsule directory privately and refuses unsafe existing parents. +RAPPS-WP-0015-T06 owns that source return; the historical storage capsule is retained. +The current date supervisor leaves helper evidence usable until application resume, +then records file fingerprints and transfers the closed evidence to root custody. +Interrupted closeout resumes against those fingerprints without another capture, +apply, helper creation or scale operation. A stale failure-phase bug during a second +recovery interruption is also corrected; completed attempts refuse replay. + +Validation: docs/release/backup-producer-custody-validation-2026-10-31.json. +Focused release and supervisor cases cover nested modes, unsafe parents, active +mounts/locks, changed or unexpected files, partial ownership transfer, and productive +writes after resume. Root ownership transitions use a synthetic metadata adapter; +ordinary descriptor/file preservation checks use real temporary files. Full make +check includes protected browser journeys. No production operation, frozen historical +capsule or pending repair/installation/route packet was changed. Future reviewed +capsules carry these sources; the existing eight-entry repair remains pending T04, +and native recurring evidence remains T09. This task closes source prevention only.