diff --git a/docs/release/backup-completed-evidence-custody-preflight-2026-10-31.json b/docs/release/backup-completed-evidence-custody-preflight-2026-10-31.json new file mode 100644 index 0000000..b731343 --- /dev/null +++ b/docs/release/backup-completed-evidence-custody-preflight-2026-10-31.json @@ -0,0 +1,146 @@ +{ + "schema": "coulomb.completed-backup-evidence-custody-preflight.v1", + "observed_at": "2026-10-31T11:20:38.284348+00:00", + "targets": [ + { + "relative_path": "releases", + "kind": "directory", + "mode": "0o755", + "uid": 0, + "gid": 0, + "nlink": 4, + "bytes": 4096, + "inode": 543834, + "device": 2049, + "mtime_ns": 1793430825511133103, + "sha256": "3190439b978096a16c28aa0dd6b372735594808217b59a01dadbbe744d4110c8", + "hash_kind": "directory-entry-names" + }, + { + "relative_path": "releases/9f1daf6c5fb56b2b4fc9817f28f5ad090c884b0295f7d6f4469e42d48fa8d59f/capsule/charts", + "kind": "directory", + "mode": "0o755", + "uid": 0, + "gid": 0, + "nlink": 3, + "bytes": 4096, + "inode": 549330, + "device": 2049, + "mtime_ns": 1793430825511133103, + "sha256": "6b8295661773cb5820c305091679383b09fb0d7dcdde4412eeae48bfabab28b5", + "hash_kind": "directory-entry-names" + }, + { + "relative_path": "releases/9f1daf6c5fb56b2b4fc9817f28f5ad090c884b0295f7d6f4469e42d48fa8d59f/capsule/release", + "kind": "directory", + "mode": "0o755", + "uid": 0, + "gid": 0, + "nlink": 3, + "bytes": 4096, + "inode": 549341, + "device": 2049, + "mtime_ns": 1793430825512133101, + "sha256": "25a35515bc386a0c0a1a91b5fa3c966fffad64e2ce1d8bde35959bc164b11d79", + "hash_kind": "directory-entry-names" + }, + { + "relative_path": "releases/5fd7ccd8f68057346367d81efb82b48280a16dd31d87683a5bd83e60e3d1837e/capsule/charts", + "kind": "directory", + "mode": "0o755", + "uid": 0, + "gid": 0, + "nlink": 3, + "bytes": 4096, + "inode": 549219, + "device": 2049, + "mtime_ns": 1793406468579820277, + "sha256": "6b8295661773cb5820c305091679383b09fb0d7dcdde4412eeae48bfabab28b5", + "hash_kind": "directory-entry-names" + }, + { + "relative_path": "date-operations/47829c7ac2e6fd2c81fde1c914aec7c6e433098fa66985db6cfb9b34069942dc/private", + "kind": "directory", + "mode": "0o700", + "uid": 10001, + "gid": 10001, + "nlink": 2, + "bytes": 4096, + "inode": 549263, + "device": 2049, + "mtime_ns": 1793412565567095985, + "sha256": "384eaf76c1a05ea25236c7a0d7bee390638b68fc89186604def37028364992fb", + "hash_kind": "directory-entry-names" + }, + { + "relative_path": "date-operations/47829c7ac2e6fd2c81fde1c914aec7c6e433098fa66985db6cfb9b34069942dc/private/result.json", + "kind": "regular", + "mode": "0o600", + "uid": 10001, + "gid": 10001, + "nlink": 1, + "bytes": 455, + "inode": 549311, + "device": 2049, + "mtime_ns": 1793412565564096058, + "sha256": "859e5a80442ad77b5eff61b68e5062a167d51f5db1b6c2d1ee064db977beb022", + "hash_kind": "file-bytes" + }, + { + "relative_path": "date-operations/47829c7ac2e6fd2c81fde1c914aec7c6e433098fa66985db6cfb9b34069942dc/private/journal.json", + "kind": "regular", + "mode": "0o600", + "uid": 10001, + "gid": 10001, + "nlink": 1, + "bytes": 12930, + "inode": 549315, + "device": 2049, + "mtime_ns": 1793412565062108322, + "sha256": "fef62ad9317066d5d82eccbea0317eceb2f883e07f50935104e47113855fe0dd", + "hash_kind": "file-bytes" + }, + { + "relative_path": "date-operations/47829c7ac2e6fd2c81fde1c914aec7c6e433098fa66985db6cfb9b34069942dc/private/execute.lock", + "kind": "regular", + "mode": "0o600", + "uid": 10001, + "gid": 10001, + "nlink": 1, + "bytes": 0, + "inode": 549310, + "device": 2049, + "mtime_ns": 1793412563167160722, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "hash_kind": "file-bytes" + } + ], + "receipts": [ + { + "relative_path": "releases/5fd7ccd8f68057346367d81efb82b48280a16dd31d87683a5bd83e60e3d1837e/receipt.json", + "sha256": "cc3de724f8bcd6b19b5d9fd7a3ea93ab342c9865da0f0cb29a011f8d80358534", + "terminal_status": "storage_candidate_deployed_verified", + "pid": 2724118, + "pid_absent": true + }, + { + "relative_path": "releases/9f1daf6c5fb56b2b4fc9817f28f5ad090c884b0295f7d6f4469e42d48fa8d59f/receipt.json", + "sha256": "7c8a953f55b38ee86266d279cb571b5d6795e8afb7dd6780ce1d052d435fbcca", + "terminal_status": "image_candidate_deployed_verified", + "pid": 3639216, + "pid_absent": true + }, + { + "relative_path": "date-operations/47829c7ac2e6fd2c81fde1c914aec7c6e433098fa66985db6cfb9b34069942dc/receipt.json", + "sha256": "851b0d39ec26804882703ba4792a3439f2821c5ce5c5fbe945934f6575ddb00e", + "terminal_status": "completed", + "pid": 2951460, + "pid_absent": true + } + ], + "historical_helper_absent": true, + "state_hostpath_mounts": [], + "locks_acquired_and_released": true, + "modified": false, + "private_bytes_printed": false +} diff --git a/docs/release/backup-completed-evidence-custody-review-2026-10-31.md b/docs/release/backup-completed-evidence-custody-review-2026-10-31.md new file mode 100644 index 0000000..651eaeb --- /dev/null +++ b/docs/release/backup-completed-evidence-custody-review-2026-10-31.md @@ -0,0 +1,117 @@ +# Completed backup evidence custody repair + +This is the exact proposed repair for the eight historical metadata exceptions +recorded in CSOC-WP-0012-T04. Native preflight passes. **Nothing has been changed.** + +Proposal: +`sha256:fa2763d1dcbe92fe638a29c9a7a7d51c03d30849c4232f2d43f81ef705bae611`. + +## Exact change + +Within `/var/lib/coulomb-social-backup/`, change: + +| Entries | Before | After | +| --- | --- | --- | +| `releases` and the three enumerated historical capsule directories | root:root0755 | root:root0700 | +| Completed date operation's `private` directory |10001:10001,0700 | root:root0700 | +| Its `result.json`, `journal.json`, `execute.lock` |10001:10001,0600 | root:root0600 | + +The complete eight-path list, inode/device/mtime, modes/owners and content or +directory-entry hashes are in +`backup-completed-evidence-custody-preflight-2026-10-31.json`. The date operation +is47829c7a; the historical releases are5fd7ccd8 and9f1daf6c. No path is moved, +file body rewritten, backup removed, application paused or timer activated. + +The UID10001 entries were the temporary helper's execution journal. Its source +requires that owner while operating. This proposal transfers the **completed** +records to root custody only after exact terminal receipt, absent process/helper, +unmounted state and operation-lock checks. It does not generalize the change to +other helper directories. The original ownership remains in the repair journal +for explicit restoration if an authorized later review requires it. + +The strict backup capacity guard is unchanged. Successful execution also runs its +existing state-usage/custody check over the entire state tree. This is neither a +blanket chmod/chown nor an exception that omits historical data from accounting. + +## Founder terminal command + +Run this from the workstation after reviewing the scope above: + +```bash +python3 /home/worsch/coulomb-social/scripts/seal-backup-evidence.py \ + --execute --expect-digest sha256:fa2763d1dcbe92fe638a29c9a7a7d51c03d30849c4232f2d43f81ef705bae611 +``` + +At the prompt, enter: + +```text +SEAL sha256:fa2763d1dcbe92fe638a29c9a7a7d51c03d30849c4232f2d43f81ef705bae611 +``` + +Expected status: `completed_evidence_private_root_custody_verified`, eight entries +verified and `postflight.whole_state_custody_guard_passed: true`. Paste the result +back for independent readback. The private durable receipt is: + +`/var/lib/coulomb-social-backup-custody/fa2763d1dcbe92fe638a29c9a7a7d51c03d30849c4232f2d43f81ef705bae611/receipt.json`. + +The [supervision decision](/home/worsch/the-custodian/docs/agent-autonomy-decision.md) +requires: β€œThe supervisor approves that proposal or performs it through the +privileged execution path.” This is a new, exact historical ownership/mode change; +it is not covered by approval of an SMTP recipient or recurring schedule. The +entry point requires the founder terminal and exact digest. No new credential +grant is requested. + +Offline review uses the same command without flags. Read-only native preflight +uses `--preflight`. Any source, capacity-guard, metadata or receipt change changes +the proposal or refuses the preflight; do not bypass a changed digest. + +## Verification, interruption and rollback + +The operation takes the date supervisor, backup, snapshot and helper execute +locks without replacing/truncating them. It verifies the pinned cluster, inactive +timers, three unchanged terminal receipts and absent historical PIDs. It refuses +the original helper by name/UID or any pod hostPath overlapping the state tree. +All eight fingerprints are checked before mutation. Each metadata change is +journaled and fsynced before descriptor-based chown/chmod. Symlink ancestors, +hardlinks, changed bytes/children, replaced inodes and unknown paths refuse. + +Afterward, all eight complete fingerprints must match the expected ownership/ +mode changes with unchanged hashes, sizes, paths, device/inodes and mtimes. +Native process/helper/timer checks repeat and the unchanged whole-state custody +guard must pass before success is recorded. Receipts contain metadata/hashes, +not historical journal bodies. The new repair journal is private and outside the +backup state tree so it cannot alter the reviewed historical directory listings. + +On lost connection or failure, inspect the retained receipt and native process +state first. A partially completed seal is not replayed: the attempt directory +refuses another installation. The explicit rollback entry point is the same +command with `--rollback` instead of `--execute`, and confirmation +`ROLLBACK `. It restores only the recorded original metadata, +after the same terminal/inactive/lock checks, and refuses content or inode drift. +Earlier receipts remain nested in the rollback journal. Returning the old metadata +also restores the original capacity-guard refusal; rollback is not backup readiness. + +Cost: no new service, storage purchase, provider request or runtime allocation; +existing-node execution and operator time are unpriced. Risk is loss of the retired +helper UID's access if its supposedly completed operation were still needed. +Terminal/helper/lock checks and explicit metadata rollback bound that risk; +production file content and database writes are not part of this operation. + +## Evidence and remaining scope + +Native metadata/fence observation and the final read-only preflight pass. The +validation companion `backup-completed-evidence-custody-validation-2026-10-31.json` +records509 passing full-gate tests, zero failures/errors/skips and source hashes. +Nineteen dedicated cases exercise real metadata/lock IO, preserved bytes/paths, +explicit/partial rollback, interference refusal, source/digest boundaries, +nonterminal refusal, and native receipt/process/helper/mount/cluster/timer guards. + +This resolves only the current custody mismatch after attended success. T04 still +owns retention/capacity admission and historical-copy disposition. Preventing +future release/date producers from reintroducing these metadata mismatches is also +live T04 work before recurring service acceptance: nested release directory +creation currently uses default intermediate modes, and date helper journals +retain the execution UID after completion. Do not silently change frozen capsules. +T07 still needs the existing route execution; T08 then needs inactive installation +and independent readback. No native backup-operation or migration acceptance is +inferred from this repair. diff --git a/docs/release/backup-completed-evidence-custody-validation-2026-10-31.json b/docs/release/backup-completed-evidence-custody-validation-2026-10-31.json new file mode 100644 index 0000000..d5d2408 --- /dev/null +++ b/docs/release/backup-completed-evidence-custody-validation-2026-10-31.json @@ -0,0 +1,55 @@ +{ + "schema": "coulomb.completed-evidence-custody-validation.v1", + "completed_at": "2026-10-31T11:31:08.384781+00:00", + "proposal": { + "status": "prepared", + "proposal_digest": "sha256:fa2763d1dcbe92fe638a29c9a7a7d51c03d30849c4232f2d43f81ef705bae611", + "entries": 8, + "permission_or_ownership_changed": false + }, + "native_preflight": { + "status": "completed_evidence_seal_preflight_verified", + "proposal_digest": "sha256:fa2763d1dcbe92fe638a29c9a7a7d51c03d30849c4232f2d43f81ef705bae611", + "entries": 8, + "modified": false, + "historical_operations_terminal": true + }, + "files": [ + { + "path": "scripts/backup_custody.py", + "sha256": "4fcb3a347b2d0029866d077a2554776d2a86e32d95d22d5b7a6296481c8b2b80" + }, + { + "path": "scripts/seal-backup-evidence.py", + "sha256": "c2c33f90ae5736f1a5b96f6298330fa82cfdf8cfab48e7bdd197da693b2e645d" + }, + { + "path": "scripts/backup_capacity.py", + "sha256": "cb699a79e67b129239b5aa5d6f3b0d4ffaa9773a8b073227ad17fdecb5b9d893" + }, + { + "path": "tests/test_backup_custody.py", + "sha256": "9dcde8abf077eaafa08028e0a6017c4309115444f2f5d1c8c4dbabd841517f73" + }, + { + "path": "docs/release/backup-completed-evidence-custody-preflight-2026-10-31.json", + "sha256": "96a5ab46fea2bbc00b574e66ee416665e3b619282f1b72f42b5387b8bf8f46f3" + } + ], + "full_gate": { + "command": "make check", + "tests": 509, + "failures": 0, + "errors": 0, + "skipped": 0, + "warnings": 69, + "junit_sha256": "6355aed63340b021b9f4163d6a1572e15d7d8a18d6614bcffc8896e9c94354ef", + "log_sha256": "93603f9df00147a2cdc8138e03dc629f2a7ef877ac2ef1281b06bab5825064bc" + }, + "dedicated_custody_tests": 19, + "ownership_changed": false, + "permissions_changed": false, + "private_journal_bodies_printed": false, + "historical_data_deleted": false, + "timers_activated": false +} diff --git a/scripts/backup_custody.py b/scripts/backup_custody.py new file mode 100644 index 0000000..e1ccb5e --- /dev/null +++ b/scripts/backup_custody.py @@ -0,0 +1,385 @@ +"""Seal only eight completed historical evidence entries; preserve bytes and paths.""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import os +import stat +import subprocess +import tempfile +import types +from contextlib import contextmanager +from datetime import UTC, datetime +from pathlib import Path + +STATE = Path("/var/lib/coulomb-social-backup") +JOURNALS = Path("/var/lib/coulomb-social-backup-custody") +DATE = "47829c7ac2e6fd2c81fde1c914aec7c6e433098fa66985db6cfb9b34069942dc" +IMAGE = "9f1daf6c5fb56b2b4fc9817f28f5ad090c884b0295f7d6f4469e42d48fa8d59f" +STORAGE = "5fd7ccd8f68057346367d81efb82b48280a16dd31d87683a5bd83e60e3d1837e" +PRIVATE = "date-operations/" + DATE + "/private" +PATHS = ( + "releases", + "releases/" + IMAGE + "/capsule/charts", + "releases/" + IMAGE + "/capsule/release", + "releases/" + STORAGE + "/capsule/charts", + PRIVATE, + PRIVATE + "/result.json", + PRIVATE + "/journal.json", + PRIVATE + "/execute.lock", +) +CLUSTER = "a553c742-0115-43d4-99a4-a5ca56fe0786" + + +class Refused(ValueError): + pass + + +def require(ok, reason): + if not ok: + raise Refused(reason) + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +@contextmanager +def opened(relative): + parts = Path(relative).parts + require( + not Path(relative).is_absolute() and ".." not in parts and parts, "invalid_relative_path" + ) + handles = [] + try: + fd = os.open(STATE, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + handles.append(fd) + info = os.fstat(fd) + require(info.st_uid == os.geteuid() and not info.st_mode & 0o077, "private_state_required") + for index, part in enumerate(parts): + if index: + info = os.fstat(fd) + require( + info.st_uid in (os.geteuid(), 10001) and not info.st_mode & 0o022, + "unsafe_ancestor", + ) + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK + if index < len(parts) - 1: + flags |= os.O_DIRECTORY + fd = os.open(part, flags, dir_fd=fd) + handles.append(fd) + yield fd + finally: + for fd in reversed(handles): + os.close(fd) + + +def fingerprint(relative): + with opened(relative) as fd: + info = os.fstat(fd) + directory = stat.S_ISDIR(info.st_mode) + require(directory or stat.S_ISREG(info.st_mode) and info.st_nlink == 1, "unsupported_entry") + if directory: + raw = json.dumps(sorted(os.listdir(fd)), separators=(",", ":")).encode() + else: + require(info.st_size <= 1024 * 1024, "oversized_evidence") + raw = b"" + while chunk := os.read(fd, 65536): + raw += chunk + require(len(raw) <= 1024 * 1024, "growing_evidence") + after = os.fstat(fd) + require( + (info.st_mtime_ns, info.st_size, info.st_uid, info.st_gid, info.st_mode) + == (after.st_mtime_ns, after.st_size, after.st_uid, after.st_gid, after.st_mode), + "evidence_changed_during_read", + ) + return dict( + relative_path=relative, + kind="directory" if directory else "regular", + mode=oct(stat.S_IMODE(info.st_mode)), + uid=info.st_uid, + gid=info.st_gid, + nlink=info.st_nlink, + bytes=info.st_size, + inode=info.st_ino, + device=info.st_dev, + mtime_ns=info.st_mtime_ns, + sha256=sha(raw), + hash_kind="directory-entry-names" if directory else "file-bytes", + ) + + +@contextmanager +def locked(): + from contextlib import ExitStack + + with ExitStack() as stack: + for path in ( + "date-operations/supervisor.lock", + "backup.lock", + "snapshot.lock", + PRIVATE + "/execute.lock", + ): + fd = stack.enter_context(opened(path)) + info = os.fstat(fd) + require( + stat.S_ISREG(info.st_mode) + and info.st_nlink == 1 + and info.st_uid in (os.geteuid(), 10001) + and not info.st_mode & 0o077, + "unsafe_operation_lock", + ) + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + yield + + +def native_fence(plan): + for timer in ("coulomb-social-paired-backup.timer", "coulomb-social-backup-freshness.timer"): + result = subprocess.run( + ["systemctl", "show", timer, "--property=ActiveState,UnitFileState"], + capture_output=True, + timeout=15, + ) + require(result.returncode == 0, "timer_observation_failed") + values = dict( + line.split("=", 1) for line in result.stdout.decode().splitlines() if "=" in line + ) + require( + values.get("ActiveState") == "inactive" + and values.get("UnitFileState", "") in ("", "disabled", "static"), + "timers_must_remain_inactive", + ) + + def kube(*args): + result = subprocess.run( + ["k3s", "kubectl", *args, "-o", "json"], capture_output=True, timeout=30 + ) + require( + result.returncode == 0 and len(result.stdout) < 16 * 1024 * 1024, "native_read_failed" + ) + return json.loads(result.stdout) + + require(kube("get", "namespace", "kube-system")["metadata"]["uid"] == CLUSTER, "wrong_cluster") + for entry in plan["receipts"]: + with opened(entry["relative_path"]) as fd: + info = os.fstat(fd) + require( + stat.S_ISREG(info.st_mode) + and info.st_nlink == 1 + and info.st_uid == 0 + and not info.st_mode & 0o077 + and info.st_size < 65536, + "unsafe_receipt", + ) + raw = os.read(fd, 65537) + require(sha(raw) == entry["sha256"], "historical_receipt_changed") + receipt = json.loads(raw) + require( + receipt["pid"] == entry["pid"] and not Path("/proc", str(entry["pid"])).exists(), + "historical_process_present", + ) + status = ( + receipt.get("phase") + if entry["relative_path"].startswith("date-") + else receipt.get("status") + ) + require(status == entry["terminal_status"], "historical_operation_not_terminal") + pods = kube("get", "pods", "-A")["items"] + for pod in pods: + require( + pod["metadata"]["name"] != "csoc-date-" + DATE[:20] + and pod["metadata"]["uid"] != "60c3b803-69d3-45da-9b09-19a6c0b274da", + "historical_helper_present", + ) + for volume in pod["spec"].get("volumes", []): + path = volume.get("hostPath", {}).get("path") + if path: + require( + not ( + path.startswith(str(STATE)) or str(STATE).startswith(path.rstrip("/") + "/") + ), + "state_hostpath_mount_present", + ) + + +def desired(before): + return dict(before, uid=0, gid=0, mode="0o700" if before["kind"] == "directory" else "0o600") + + +def save(directory, value): + fd, name = tempfile.mkstemp(prefix=".receipt-", dir=directory) + with os.fdopen(fd, "wb") as stream: + stream.write(canonical(value) + b"\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(name, directory / "receipt.json") + fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def change(row, before, after): + # Descriptor operations cannot follow a replacement final symlink. + require(fingerprint(row["relative_path"]) == before, "entry_changed_before_metadata_write") + with opened(row["relative_path"]) as fd: + info = os.fstat(fd) + require((info.st_ino, info.st_dev) == (before["inode"], before["device"]), "entry_replaced") + if (before["uid"], before["gid"]) != (after["uid"], after["gid"]): + os.fchown(fd, after["uid"], after["gid"]) + if before["mode"] != after["mode"]: + os.fchmod(fd, int(after["mode"], 8)) + os.fsync(fd) + require(fingerprint(row["relative_path"]) == after, "metadata_postflight_differs") + + +def transact(plan, directory, *, rollback=False, postflight=lambda: None): + rows = plan["targets"] + current = {row["relative_path"]: fingerprint(row["relative_path"]) for row in rows} + for row in rows: + allowed = (row, desired(row)) if rollback else (row,) + require(current[row["relative_path"]] in allowed, "evidence_drift_requires_review") + record = dict( + schema="coulomb.completed-evidence-custody.v1", + proposal_digest=plan["proposal_digest"], + status="rollback_started" if rollback else "seal_started", + started_at=datetime.now(UTC).isoformat(), + changes=[], + bytes_modified=False, + paths_moved=False, + data_deleted=False, + capture_executed=False, + timers_activated=False, + ) + if rollback: + fd = os.open(directory / "receipt.json", os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen(fd, "rb") as stream: + info = os.fstat(stream.fileno()) + require( + stat.S_ISREG(info.st_mode) + and info.st_nlink == 1 + and info.st_uid == os.geteuid() + and not info.st_mode & 0o077 + and info.st_size < 1024 * 1024, + "private_prior_receipt_required", + ) + prior = json.load(stream) + require(prior["proposal_digest"] == plan["proposal_digest"], "rollback_proposal_differs") + # Retain the earlier evidence rather than replace its outcome history. + record["prior_receipt"] = prior + else: + directory.mkdir(mode=0o700, parents=True, exist_ok=False) + save(directory, record) + try: + for row in reversed(rows) if rollback else rows: + before = current[row["relative_path"]] + after = row if rollback else desired(row) + if before == after: + continue + record["changes"].append(dict(path=row["relative_path"], before=before, after=after)) + save(directory, record) + change(row, before, after) + for row in rows: + require( + fingerprint(row["relative_path"]) == (row if rollback else desired(row)), + "whole_scope_postflight_differs", + ) + record["postflight"] = postflight() + record.update( + status="original_metadata_restored_verified" + if rollback + else "completed_evidence_private_root_custody_verified", + verified_entries=len(rows), + ) + except Exception as error: + record.update(status="requires_inspection", failure_type=type(error).__name__) + raise + finally: + record["updated_at"] = datetime.now(UTC).isoformat() + save(directory, record) + return record + + +def run(plan, action, capacity_source=None): + require(__debug__ and os.geteuid() == 0, "attended_root_required") + require(action in ("preflight", "seal", "rollback"), "unknown_action") + scope = dict(plan) + value = scope.pop("proposal_digest") + require(value == "sha256:" + sha(canonical(scope)), "proposal_digest_differs") + require( + tuple(row["relative_path"] for row in plan["targets"]) == PATHS, + "exact_eight_paths_required", + ) + require(len(plan["receipts"]) == 3, "three_completed_receipts_required") + require( + tuple(entry["relative_path"] for entry in plan["receipts"]) + == ( + "releases/" + STORAGE + "/receipt.json", + "releases/" + IMAGE + "/receipt.json", + "date-operations/" + DATE + "/receipt.json", + ), + "exact_historical_receipts_required", + ) + for row in plan["targets"]: + require( + row["uid"] == row["gid"] == (10001 if row["relative_path"].startswith(PRIVATE) else 0), + "original_ownership_differs", + ) + require( + isinstance(capacity_source, str) + and sha(capacity_source.encode()) == plan["capacity_source_sha256"], + "capacity_source_differs", + ) + capacity = types.ModuleType("reviewed_backup_capacity") + exec(compile(capacity_source, "", "exec"), capacity.__dict__) + directory = JOURNALS / plan["proposal_digest"].removeprefix("sha256:") + require(directory.name.isalnum() and len(directory.name) == 64, "invalid_proposal_digest") + for parent in (JOURNALS, JOURNALS.parent): + if parent.exists() or parent.is_symlink(): + info = parent.lstat() + require( + stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and not info.st_mode & 0o022, + "unsafe_journal_parent", + ) + if parent == JOURNALS: + require(not info.st_mode & 0o077, "private_journal_root_required") + with locked(): + native_fence(plan) + if action == "preflight": + require( + all(fingerprint(row["relative_path"]) == row for row in plan["targets"]), + "evidence_drift_requires_review", + ) + require(not directory.exists(), "existing_attempt_requires_inspection") + return dict( + status="completed_evidence_seal_preflight_verified", + proposal_digest=plan["proposal_digest"], + entries=8, + modified=False, + historical_operations_terminal=True, + ) + if action == "rollback": + info = directory.lstat() + require( + stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and not info.st_mode & 0o077, + "private_attempt_required", + ) + else: + JOURNALS.mkdir(mode=0o700, exist_ok=True) + + def postflight(): + native_fence(plan) + result = {"historical_operations_still_terminal": True} + if action == "seal": + result["state_bytes"] = capacity.state_usage(STATE) + result["whole_state_custody_guard_passed"] = True + return result + + return transact(plan, directory, rollback=action == "rollback", postflight=postflight) diff --git a/scripts/seal-backup-evidence.py b/scripts/seal-backup-evidence.py new file mode 100644 index 0000000..fbca2eb --- /dev/null +++ b/scripts/seal-backup-evidence.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Review or attend sealing of eight completed backup evidence metadata entries.""" + +import argparse +import hashlib +import json +import shlex +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OBSERVATION = ROOT / "docs/release/backup-completed-evidence-custody-preflight-2026-10-31.json" + + +def prepare(): + if not __debug__: + raise ValueError("optimized_python_refused") + observed = json.loads(OBSERVATION.read_text()) + source = (ROOT / "scripts/backup_custody.py").read_text() + plan = dict( + schema="coulomb.completed-evidence-custody-proposal.v1", + targets=observed["targets"], + receipts=observed["receipts"], + runtime_sha256=hashlib.sha256(source.encode()).hexdigest(), + launcher_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + capacity_source_sha256=hashlib.sha256( + (ROOT / "scripts/backup_capacity.py").read_bytes() + ).hexdigest(), + ) + plan["proposal_digest"] = ( + "sha256:" + + hashlib.sha256( + json.dumps(plan, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + ) + return plan, source + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + action = parser.add_mutually_exclusive_group() + action.add_argument("--preflight", action="store_true") + action.add_argument("--execute", action="store_true") + action.add_argument("--rollback", action="store_true") + parser.add_argument("--expect-digest") + args = parser.parse_args(argv) + plan, source = prepare() + digest = plan["proposal_digest"] + if args.expect_digest and args.expect_digest != digest: + raise SystemExit("Reviewed custody proposal changed; no action") + if not (args.preflight or args.execute or args.rollback): + print( + json.dumps( + dict( + status="prepared", + proposal_digest=digest, + entries=8, + permission_or_ownership_changed=False, + ) + ) + ) + return 0 + if args.execute or args.rollback: + if not args.expect_digest or not sys.stdin.isatty(): + raise SystemExit("Founder terminal and exact digest required") + verb = "ROLLBACK" if args.rollback else "SEAL" + print( + "Restore the original metadata only." + if args.rollback + else "Seal eight completed evidence entries: root ownership, private modes; " + "bytes and paths preserved." + ) + if input("Type " + verb + " followed by the full digest: ").strip() != verb + " " + digest: + raise SystemExit("Not confirmed; no action") + payload = dict( + plan=plan, + action="rollback" if args.rollback else "seal" if args.execute else "preflight", + capacity_source=(ROOT / "scripts/backup_capacity.py").read_text(), + ) + bootstrap = ( + source + + """ +import sys +p=json.load(sys.stdin) +try: + result=run(p['plan'],p['action'],p['capacity_source']) +except Exception as error: + result=dict(status='requires_inspection',proposal_digest=p['plan']['proposal_digest'], + failure_type=type(error).__name__,retry_automatically=False) + if isinstance(error,Refused):result['reason']=str(error) +print(json.dumps(result)) +sys.exit(0 if result['status'] in ('completed_evidence_seal_preflight_verified', + 'completed_evidence_private_root_custody_verified','original_metadata_restored_verified') else 1) +""" + ) + try: + result = subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=10", + "railiance01", + "sudo -n python3 -c " + shlex.quote(bootstrap), + ], + input=json.dumps(payload).encode(), + capture_output=True, + timeout=150, + ) + value = json.loads(result.stdout) + except (ValueError, subprocess.TimeoutExpired): + print( + json.dumps( + dict( + status="remote_outcome_unconfirmed", + proposal_digest=digest, + retry_automatically=False, + ) + ) + ) + return 1 + print(json.dumps(value)) + return int(result.returncode != 0) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_backup_custody.py b/tests/test_backup_custody.py new file mode 100644 index 0000000..095c6fb --- /dev/null +++ b/tests/test_backup_custody.py @@ -0,0 +1,288 @@ +"""Historical evidence metadata sealing against disposable files only.""" + +import fcntl +import importlib.util +import json +import os +import subprocess +from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).parents[1] +spec = importlib.util.spec_from_file_location("backup_custody", ROOT / "scripts/backup_custody.py") +custody = importlib.util.module_from_spec(spec) +spec.loader.exec_module(custody) + + +@pytest.fixture +def evidence(tmp_path, monkeypatch): + state = tmp_path / "state" + state.mkdir(mode=0o700) + directory = state / "releases" + directory.mkdir(mode=0o755) + (directory / "fixture.json").write_bytes(b'{"retained":"synthetic evidence"}\n') + (directory / "fixture.json").chmod(0o600) + monkeypatch.setattr(custody, "STATE", state) + # Native entry point always selects root. Exercise real chmod/fd/hash IO as + # the test user, and verify the root chown syscall separately below. + monkeypatch.setattr( + custody, + "desired", + lambda r: dict( + r, + uid=os.getuid(), + gid=os.getgid(), + mode="0o700" if r["kind"] == "directory" else "0o600", + ), + ) + rows = [custody.fingerprint("releases"), custody.fingerprint("releases/fixture.json")] + plan = {"targets": rows, "proposal_digest": "sha256:" + "a" * 64} + return plan, tmp_path / "attempt", state + + +def test_real_metadata_seal_preserves_bytes_paths_and_explicit_rollback(evidence): + plan, directory, state = evidence + raw = (state / "releases/fixture.json").read_bytes() + result = custody.transact(plan, directory) + assert result["status"] == "completed_evidence_private_root_custody_verified" + assert result["verified_entries"] == 2 and result["bytes_modified"] is False + assert (state / "releases").stat().st_mode & 0o777 == 0o700 + assert (state / "releases/fixture.json").read_bytes() == raw + assert json.loads((directory / "receipt.json").read_text()) == result + result = custody.transact(plan, directory, rollback=True) + assert result["status"] == "original_metadata_restored_verified" + assert result["prior_receipt"]["status"] == "completed_evidence_private_root_custody_verified" + assert [custody.fingerprint(r["relative_path"]) for r in plan["targets"]] == plan["targets"] + + +def test_changed_body_refuses_before_any_metadata_write(evidence): + plan, directory, state = evidence + (state / "releases/fixture.json").write_bytes(b"changed") + with pytest.raises(custody.Refused, match="evidence_drift"): + custody.transact(plan, directory) + assert not directory.exists() + assert (state / "releases").stat().st_mode & 0o777 == 0o755 + + +def test_replaced_inode_or_child_listing_refuses(evidence): + plan, directory, state = evidence + (state / "releases/unreviewed").write_bytes(b"new") + with pytest.raises(custody.Refused): + custody.transact(plan, directory) + assert not directory.exists() + + +def test_partial_seal_has_durable_preimages_and_can_restore(evidence, monkeypatch): + plan, directory, _ = evidence + original = custody.change + + def interrupted(row, before, after): + original(row, before, after) + raise SystemExit(9) + + monkeypatch.setattr(custody, "change", interrupted) + with pytest.raises(SystemExit): + custody.transact(plan, directory) + receipt = json.loads((directory / "receipt.json").read_text()) + assert receipt["status"] == "seal_started" and len(receipt["changes"]) == 1 + assert receipt["changes"][0]["before"] == plan["targets"][0] + monkeypatch.setattr(custody, "change", original) + result = custody.transact(plan, directory, rollback=True) + assert result["status"] == "original_metadata_restored_verified" + + +def test_postflight_failure_is_not_success_and_retains_journal(evidence): + plan, directory, _ = evidence + + def fail(): + raise RuntimeError("private provider text") + + with pytest.raises(RuntimeError): + custody.transact(plan, directory, postflight=fail) + raw = (directory / "receipt.json").read_text() + assert json.loads(raw)["status"] == "requires_inspection" + assert "private provider text" not in raw + + +def test_interfering_rollback_and_duplicate_attempt_refuse(evidence): + plan, directory, state = evidence + custody.transact(plan, directory) + original = (directory / "receipt.json").read_bytes() + with pytest.raises(custody.Refused): + custody.transact(plan, directory) + assert (directory / "receipt.json").read_bytes() == original + (state / "releases/fixture.json").write_bytes(b"interference") + with pytest.raises(custody.Refused): + custody.transact(plan, directory, rollback=True) + assert (directory / "receipt.json").read_bytes() == original + + +def test_symlink_ancestor_and_final_file_refuse(evidence): + _, _, state = evidence + (state / "alias").symlink_to(state / "releases") + with pytest.raises(OSError): + custody.fingerprint("alias/fixture.json") + (state / "releases/link").symlink_to(state / "releases/fixture.json") + with pytest.raises(OSError): + custody.fingerprint("releases/link") + with pytest.raises(custody.Refused): + custody.fingerprint("../escape") + + +def test_hardlink_and_public_state_refuse(evidence): + _, _, state = evidence + os.link(state / "releases/fixture.json", state / "another") + with pytest.raises(custody.Refused): + custody.fingerprint("releases/fixture.json") + state.chmod(0o755) + with pytest.raises(custody.Refused): + custody.fingerprint("releases") + + +def test_root_ownership_change_uses_fd_and_retains_regular_bytes(evidence, monkeypatch): + plan, _, state = evidence + before = plan["targets"][1] + after = dict(before, uid=0, gid=0) + checks = iter([before, after]) + calls = [] + monkeypatch.setattr(custody, "fingerprint", lambda _: next(checks)) + + def chown(fd, uid, gid): + assert os.fstat(fd).st_ino == before["inode"] + calls.append((uid, gid)) + + monkeypatch.setattr(custody.os, "fchown", chown) + custody.change(before, before, after) + assert calls == [(0, 0)] + assert (state / "releases/fixture.json").read_bytes() == b'{"retained":"synthetic evidence"}\n' + + +def test_nonterminal_launcher_refuses_before_ssh(monkeypatch): + spec = importlib.util.spec_from_file_location( + "seal_launcher", ROOT / "scripts/seal-backup-evidence.py" + ) + launcher = importlib.util.module_from_spec(spec) + spec.loader.exec_module(launcher) + monkeypatch.setattr(launcher, "prepare", lambda: ({"proposal_digest": "fixture"}, "")) + monkeypatch.setattr(launcher.sys, "stdin", SimpleNamespace(isatty=lambda: False)) + monkeypatch.setattr(launcher.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("no SSH")) + with pytest.raises(SystemExit, match="Founder terminal"): + launcher.main(["--execute", "--expect-digest", "fixture"]) + + +def test_digest_or_path_scope_refuses_before_native_calls(monkeypatch): + fake_os = SimpleNamespace(**vars(os)) + fake_os.geteuid = lambda: 0 + monkeypatch.setattr(custody, "os", fake_os) + monkeypatch.setattr(custody, "native_fence", lambda _: pytest.fail("no native call")) + with pytest.raises(custody.Refused, match="proposal_digest_differs"): + custody.run({"proposal_digest": "sha256:" + "a" * 64}, "seal") + plan = {"targets": []} + plan["proposal_digest"] = "sha256:" + custody.sha(custody.canonical(plan)) + with pytest.raises(custody.Refused, match="exact_eight_paths_required"): + custody.run(plan, "seal") + + +@pytest.mark.parametrize("fault", [None, "receipt", "pid", "helper", "mount", "cluster", "timer"]) +def test_native_fence_requires_terminal_unchanged_receipt_absent_process_and_helper( + tmp_path, monkeypatch, fault +): + pid = os.getpid() if fault == "pid" else 999999999 + raw = json.dumps({"pid": pid, "phase": "completed"}).encode() + path = tmp_path / "receipt.json" + path.write_bytes(raw) + path.chmod(0o600) + + @contextmanager + def opened(_): + with path.open("rb") as stream: + yield stream.fileno() + + monkeypatch.setattr(custody, "opened", opened) + fake_os = SimpleNamespace(**vars(os)) + + def stat_as_native(fd): + result = os.fstat(fd) + return SimpleNamespace( + st_mode=result.st_mode, st_nlink=result.st_nlink, st_uid=0, st_size=result.st_size + ) + + fake_os.fstat = stat_as_native + monkeypatch.setattr(custody, "os", fake_os) + pods = [] + if fault in ("helper", "mount"): + pods = [ + { + "metadata": { + "name": "csoc-date-" + custody.DATE[:20] if fault == "helper" else "other", + "uid": "fixture", + }, + "spec": { + "volumes": [{"hostPath": {"path": str(custody.STATE)}}] + if fault == "mount" + else [] + }, + } + ] + + def command(args, **_): + if args[0] == "systemctl": + output = ( + b"ActiveState=active\nUnitFileState=enabled" + if fault == "timer" + else (b"ActiveState=inactive\nUnitFileState=disabled") + ) + return subprocess.CompletedProcess(args, 0, output, b"") + value = ( + {"items": pods} + if "pods" in args + else {"metadata": {"uid": "wrong" if fault == "cluster" else custody.CLUSTER}} + ) + return subprocess.CompletedProcess(args, 0, json.dumps(value).encode(), b"") + + monkeypatch.setattr(custody.subprocess, "run", command) + plan = { + "receipts": [ + { + "relative_path": "date-operations/fixture/receipt.json", + "sha256": "wrong" if fault == "receipt" else custody.sha(raw), + "pid": pid, + "terminal_status": "completed", + } + ] + } + if fault: + with pytest.raises(custody.Refused): + custody.native_fence(plan) + else: + custody.native_fence(plan) + + +def test_all_four_real_operation_locks_are_held_and_released(tmp_path, monkeypatch): + state = tmp_path / "state" + state.mkdir(mode=0o700) + monkeypatch.setattr(custody, "STATE", state) + names = ( + "date-operations/supervisor.lock", + "backup.lock", + "snapshot.lock", + custody.PRIVATE + "/execute.lock", + ) + for name in names: + p = state / name + p.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + p.write_bytes(b"retained") + p.chmod(0o600) + before = [custody.fingerprint(name) for name in names] + with custody.locked(): + for name in names: + with (state / name).open("rb") as stream: + with pytest.raises(BlockingIOError): + fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + for name in names: + with (state / name).open("rb") as stream: + fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + assert [custody.fingerprint(name) for name in names] == before diff --git a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md index c128bc9..a4c3f67 100644 --- a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md +++ b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md @@ -558,7 +558,7 @@ status: progress priority: high state_hub_task_id: "a36722a5-5f13-5728-9d58-24f030c2bb00" needs_human: true -intervention_note: "Historical private production-pair copies still need owner disposition. Native installer preflight also found eight historical release/date-evidence ownership/mode exceptions rejected by the capacity guard; see backup-state-custody-observation-2026-10-31.json. Preserve evidence; no automatic permission change or deletion. Existing custody and schedule approval are settled." +intervention_note: "Eight-entry completed-evidence custody repair is prepared and native preflight passes; founder terminal command is in docs/release/backup-completed-evidence-custody-review-2026-10-31.md, proposal fa2763d1dcbe92fe638a29c9a7a7d51c03d30849c4232f2d43f81ef705bae611. No change executed. Historical pair disposition and retention/capacity admission remain open; existing destinations and schedule are settled." ``` Remaining scope from T01. Close with owner-admitted capacity/headroom limits, a current generation/expiry plan, and explicit retention or deletion disposition for historical private production-pair copies. Execute only admitted expiry and retain native receipts. Proposed limits and a deletion candidate list are not admission. Evidence: `docs/release/backup-capacity-review-2026-10-30.md` and `generation-retention-post-date-2026-10-31.json`. No deletion has occurred. Existing RPF-WP-0053 custody/expiry ownership remains canonical. @@ -571,6 +571,23 @@ docs/release/backup-state-custody-observation-2026-10-31.json. Prepare a reviewe disposition preserving historical evidence and its ownership/access requirements; do not bypass accounting or silently change/delete it. No body was read or changed. +Prepared the exact bounded custody repair under +docs/release/backup-completed-evidence-custody-review-2026-10-31.md. Fresh native +checks bind three unchanged terminal operation receipts, absent recorded PIDs, +absent date helper and no state hostPath mounts; final preflight passes. The +founder-only entry point takes all four locks, journals descriptor-based metadata +changes, verifies unchanged bytes/paths and runs the unchanged whole-state custody +guard afterward. Nineteen dedicated cases cover refusal, real locks and rollback. +No historical ownership/mode change has executed. Receipt/body hashes were +computed natively; no private journal body was printed or sent to inference. + +Before recurring service acceptance, also prevent future producers from +reintroducing the mismatch: release capsule intermediate directories need private +creation, and completed date helper evidence needs an explicit custody closeout. +Keep frozen historical capsules unchanged; implement and test current producer +sources through their existing owners. This remains live T04 work, not a prose-only +residual. The prepared metadata repair is not full retention/capacity acceptance. + ## T05 β€” Verify both approved backup alert destinations ```task @@ -597,7 +614,7 @@ Durable independent channel attempts, retry/replay, failed/pending/stale episode ```task id: CSOC-WP-0012-T07 -status: progress +status: wait priority: high state_hub_task_id: "8c8e4810-d967-54b2-ae49-25d367eac459" needs_human: true @@ -610,9 +627,12 @@ The exact owner execution packet is ready; durable receipt was absent at the 10: ```task id: CSOC-WP-0012-T08 -status: progress +status: wait priority: high state_hub_task_id: "ca41b874-5e09-538e-9692-d31d990d679b" +depends_on: +- CSOC-WP-0012-T04 +- CSOC-WP-0012-T07 ``` The installation/rollback library and final supervised wrapper are implemented. @@ -626,6 +646,12 @@ Then obtain the attended return and independently verify installed hashes and disabled units. Review: docs/release/backup-source-installation-execution-review-2026-10-31.md; source/gate evidence: backup-supervised-installer-validation-2026-10-31.json. +Preparation is complete; task status is now wait for the T04 custody/profile +return and T07 attended route. The exact eight-entry custody repair has a passing +native preflight but has not executed. Installation and independent postflight +still prevent this task from being done. T07 likewise waits for its founder run; +neither task represents a currently running native job. + ## T09 β€” Prove native paired service and recurring failure reception ```task