diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 08c87fd..bce8109 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -88,7 +88,7 @@ | task | CSOC-WP-0011-T03 | todo | — | workplans/CSOC-WP-0011-ui-framework-architecture.md | | task | CSOC-WP-0012-T01 | progress | — | workplans/CSOC-WP-0012-durable-recovery-service-readiness.md | | task | CSOC-WP-0012-T02 | done | — | workplans/CSOC-WP-0012-durable-recovery-service-readiness.md | -| task | CSOC-WP-0012-T03 | todo | — | workplans/CSOC-WP-0012-durable-recovery-service-readiness.md | +| task | CSOC-WP-0012-T03 | progress | — | workplans/CSOC-WP-0012-durable-recovery-service-readiness.md | | task | CSOC-WP-0013-T01 | wait | — | workplans/CSOC-WP-0013-sharing-content-hardening.md | | task | CSOC-WP-0013-T02 | progress | — | workplans/CSOC-WP-0013-sharing-content-hardening.md | | task | CSOC-WP-0013-T03 | todo | — | workplans/CSOC-WP-0013-sharing-content-hardening.md | diff --git a/deploy/backup/coulomb-social-backup-freshness.service b/deploy/backup/coulomb-social-backup-freshness.service index 343f0fd..c58c8d4 100644 --- a/deploy/backup/coulomb-social-backup-freshness.service +++ b/deploy/backup/coulomb-social-backup-freshness.service @@ -1,11 +1,14 @@ [Unit] -Description=Check coulomb-social paired backup age and local encrypted hash +Description=Check coulomb-social pending captures, paired backup, mirror and capacity evidence [Service] Type=oneshot User=root -ExecStart=/usr/bin/python3 /opt/coulomb-social-backup/paired-backup.py --check-freshness +# Candidate limits: owner admission and received-alert proof precede activation. +ExecStart=/usr/bin/python3 /opt/coulomb-social-backup/check-paired-backup-health.py --mirror-receipt /var/lib/coulomb-social-backup/last-mirror.json --minimum-free-bytes 8589934592 --maximum-state-bytes 8589934592 +TimeoutStartSec=5min NoNewPrivileges=true +PrivateTmp=true ProtectSystem=strict ProtectHome=true diff --git a/docs/release/backup-operating-preflight-2026-10-30.json b/docs/release/backup-operating-preflight-2026-10-30.json new file mode 100644 index 0000000..fa46e4d --- /dev/null +++ b/docs/release/backup-operating-preflight-2026-10-30.json @@ -0,0 +1,46 @@ +{ + "limits_owner_admitted": false, + "native_checker_sha256": "e30047d5759b1f200516dad09dec2368f88da505109f601165ead192a786fde4", + "node_files_installed": false, + "notification_sent": false, + "observation": { + "backup_process_holds_lock": false, + "capture_age_seconds": 6279.229, + "free_bytes": 119716765696, + "maximum_state_bytes": 8589934592, + "minimum_free_bytes": 8589934592, + "notification_sent": false, + "observed_at": "2026-10-29T23:07:25.228631+00:00", + "operator_reception_proven": false, + "pending_count": 0, + "provider_contacted": false, + "reasons": [ + "matching_mirror_evidence_missing_or_invalid" + ], + "recorded_readback_only": true, + "schema": "coulomb.paired-backup-health.v1", + "state_bytes": 2123252, + "state_modified": false, + "status": "unhealthy" + }, + "provider_contacted": false, + "review_date": "2026-10-30", + "schema": "coulomb.backup-operating-preflight.v1", + "source_unit_verification_passed": true, + "status": "activation_requirements_incomplete", + "timers": { + "coulomb-social-backup-freshness.timer": { + "ActiveState": "inactive", + "LoadState": "not-found", + "NextElapseUSecRealtime": "", + "UnitFileState": "" + }, + "coulomb-social-paired-backup.timer": { + "ActiveState": "inactive", + "LoadState": "not-found", + "NextElapseUSecRealtime": "", + "UnitFileState": "" + } + }, + "timers_activated": false +} diff --git a/docs/release/backup-operating-preflight-2026-10-30.md b/docs/release/backup-operating-preflight-2026-10-30.md new file mode 100644 index 0000000..29ff814 --- /dev/null +++ b/docs/release/backup-operating-preflight-2026-10-30.md @@ -0,0 +1,66 @@ +# Paired backup operating preflight + +The first protected application recovery is proved. Recurring operation is not +ready to activate. This review preserves the already approved daily 03:15 UTC +schedule with five-minute jitter, 24-hour RPO, 26-hour stale threshold and +60-minute attended RTO target; it requests no repeat schedule/custody approval. + +The old hourly check inspected only last-success age and local ciphertext. It +could pass while a failed newer capture remained pending, and did not establish +matching mirror evidence. `scripts/check-paired-backup-health.py` now checks: + +- Exact Coulomb primary identity, immutable version/readback flags, original + capture time, image/recipient and both paired-store hashes. +- Local ciphertext size and hash, plus an exact corresponding mirror receipt + bound to the primary key/version/hash and dedicated recovery project/bucket. +- Pending captures even while the previous point is fresh. An active lock grants + at most 35 minutes for one in-progress capture; an abandoned pending directory, + multiple captures or a stalled lock fails. +- Private regular files/directories without symlinks, declared local state budget + and minimum free space. It reads and acquires an existing lock without creating + files. It prints only counts, times and fixed reason codes. + +The candidate freshness unit invokes this checker with an explicit last-mirror +receipt and proposed 8 GiB state budget / 8 GiB free-space reserve. These limits +are review parameters, not owner-admitted storage or spending authority. A health +check does not itself stop future uploads or enforce provider retention. + +## Native observation + +The exact checker ran through the existing admitted SSH path on railiance01, +without installing code, changing state, reading keys or contacting a provider. +Receipt: `backup-operating-preflight-2026-10-30.json`. + +At 2026-10-29 23:07 UTC the primary point was about 1.74 hours old; no pending +capture or held backup lock existed. State occupied 2,123,252 bytes and free space +was 119,716,765,696 bytes. The new check correctly failed because the worker's +`last-mirror.json` was absent. The independently verified first mirror and its +protected kit remain valid; their external receipts are not a recurring worker +pipeline. Both backup/freshness timers were not found and inactive. The old +scheduled checker and the proposed new one have not been installed by this work. + +## Required activation return + +1. Prepare and verify recurring mirror publication using only the worker's own + admitted publisher. Retry must preserve the exact primary key/version/hash; + retain each resulting public receipt independently. No private recovery key + or independent reader belongs on the worker. Promote last-mirror only after + the matching version/hash readback, and block another unattended capture + while the previous point lacks its required mirror/receipt. +2. Complete the rolling-retention/headroom contract, provider prefix expiry + compatibility and pre-capture budget/reserve guard. Preserve protected kit and + restored point outside rolling expiry. Historical private copies still await + explicit owner disposition; this change deletes none. +3. Bind the established evidence-owner receiver, then prove failed/pending/stale + signal reception and response. The checker makes no sends. Journald output, + a locally failed unit, or a queued notification is not reception proof. +4. Review exact installed source/unit hashes and first native service run before + enabling the already approved timers. Verify subsequent runs, readbacks, + Ready app, timer state and next elapse; keep production restore attended. + +Source unit verification and 35 targeted backup/monitor tests passed. Synthetic +checks cover old/foreign mirror refusal, corrupted ciphertext, absent receipts, +pending failures hidden by a fresh point, active/stalled locks, capacity and +symlink refusal. They do not prove provider availability, received alerts or +recurring operation. The existing storage release packet remains unchanged. +The full `make check` gate passed 273 tests, including mandatory browser journeys. diff --git a/docs/release/service-contract-conformance-2026-10-27.md b/docs/release/service-contract-conformance-2026-10-27.md index 3e7e466..ccd535e 100644 --- a/docs/release/service-contract-conformance-2026-10-27.md +++ b/docs/release/service-contract-conformance-2026-10-27.md @@ -9,7 +9,7 @@ evidence from the returns required to finish WP-0012/0014. | 1: shape/tenancy | Standalone repo, `coulomb_social_db`, dedicated deployment/values in railiance-apps. Space/Member tenant keys and explicit membership joins; tests deny foreign tenant/member access. Slug-keyed content has global collision refusal in source. | Deploy and verify the SQL slug constraint. Reconcile all source grants and prove complete tenant export/consolidation without ambiguous paths; global uniqueness alone does not certify a multi-client migration. | | 2: identity | OIDC/PKCE behind `apps/identity`; app-local users have unusable passwords. Accepted ADR-0001 specializes the local-account baseline. Historical Case B and personal recovery are recorded in docs/identity/smoke.md and the October 6 receipts. | Current registration Case B and source-cohort identity/grant return. Link the governing State Hub decision for the ADR specialization before claiming §8.3 deviation acceptance; the ADR alone does not establish that link. | | 3: build/release | Forgejo immutable product digest `9a646d8e…`, source `dd83c90`, values/release and smoke in the revision16 receipt; resource-only revision17 in WP-0017. Release-unit declaration in registry/release-units/coulomb-social.yaml. | Guarded storage/content release and matching source/image/values/smoke receipt. Old contract §3.2 names Gitea; current delivery uses Forgejo. Resolve the fleet successor provenance rather than silently treating that as identical. | -| 4: operations | Deployment Ready, health endpoint/runbooks, capture and isolated restore tooling. Actual historical paired restore exists. | Current ops-hub client/environment/endpoint registration evidence; admitted recurring *paired* backup, independent receipt/key/image recovery and measured complete recovery; failed/stale signal reception; post-release stage-1 CRUD/copy/transfer smoke and CSOC-IN-0004 reconciliation. Database-only backup does not cover CONTENT_ROOT. | +| 4: operations | Deployment Ready, health endpoint/runbooks. October 29 protected provider-only image/tools/pair recovery proved all tables/files and packaged app views in 30.232s, with independent escrow and retained proof. October 30 read-only native operating preflight identifies missing recurring worker mirror receipt; both timers remain absent/inactive. | Current ops-hub client/environment/endpoint registration evidence; admitted recurring *paired* backup with matching mirrored point/receipt, retention/headroom and received failed/stale signals; post-release stage-1 CRUD/copy/transfer smoke and CSOC-IN-0004 reconciliation. Provider-only recovery excludes operator-account recovery and a fresh Docker daemon. Database-only backup does not cover CONTENT_ROOT. | | 5: cost | Attribution key `resource:tenant:coulomb:coulomb-social`, dated workload/demand/labor records in docs/resource-evidence; WP-0017 records current requests 15m/176Mi. | Demonstrate the key's client × app × instance join in current finance ingestion and per-instance infrastructure cost report. Planning labor rates and CPU requests are not booked costs. | | 6: protection | Platform instance isolation, explicit read/write denial, sanitized Markdown, private images, account-local JSON export and successor-aware local exit. No credentials copied into these records. | Full client-dataset export, instance teardown plus backup expiry contract (§6.2), and schema-derived personal-data inventory. Personal export is narrower than complete client offboarding. Productive sharing/portability verdicts and retained-data disposition remain WP-0013. | diff --git a/scripts/check-paired-backup-health.py b/scripts/check-paired-backup-health.py new file mode 100644 index 0000000..1861e3a --- /dev/null +++ b/scripts/check-paired-backup-health.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Read-only backup operating check. No provider calls, sends, retries or cleanup. + +Recorded readback evidence is checked; current provider availability and operator +reception require separate native proofs. Timers remain subject to admission. +""" + +from __future__ import annotations + +import argparse +import fcntl +import hashlib +import json +import os +import re +import shutil +import stat +from datetime import UTC, datetime +from pathlib import Path + +PREFIX = "platform-pg/application-archives/coulomb-social/" +PROJECT = "06d107bc-9d84-4672-b2a0-95701cc0c07f" +MIRROR_BUCKET = "railiance-coulomb-social-archives" +NAME = r"\d{8}T\d{6}Z-[a-f0-9]{24}\.tar\.age" +BINDING = ( + "application", "captured_at", "object_key", "version_id", "ciphertext_sha256", + "ciphertext_bytes", "age_recipient", "image", "database_sha256", "content_sha256", +) + + +class Refused(ValueError): + pass + + +def regular(path): + info = path.lstat() + if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() + or info.st_mode & 0o077 or info.st_nlink != 1): + raise Refused("unsafe_state_entry") + return info + + +def read_json(path): + regular(path) + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen(fd, "rb") as stream: + data = stream.read(1024 * 1024 + 1) + if len(data) > 1024 * 1024: + raise Refused("oversized_receipt") + value = json.loads(data) + if not isinstance(value, dict): + raise Refused("invalid_receipt") + return value + + +def timestamp(value): + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise Refused("naive_capture_timestamp") + return parsed.astimezone(UTC) + + +def validate_primary(receipt): + captured = timestamp(receipt["captured_at"]) + name = receipt["local_archive"] + if (receipt.get("schema") != "platform.application-primary-archive.v1" + or receipt.get("application") != "coulomb-social" + or receipt.get("status") != "offsite_fetched_verified" + or receipt.get("immutable_create") is not True + or receipt.get("remote_retrieval_verified") is not True + or not isinstance(name, str) or not re.fullmatch(NAME, name) + or not name.startswith(captured.strftime("%Y%m%dT%H%M%SZ-")) + or receipt["object_key"] != PREFIX + captured.strftime("%Y/%m/%d/") + name + or receipt.get("destination") != "s3://railiance-platform-pg-backup/" + + receipt["object_key"] + or not isinstance(receipt.get("version_id"), str) + or not receipt["version_id"] or receipt["version_id"] == "null" + or type(receipt.get("ciphertext_bytes")) is not int + or receipt["ciphertext_bytes"] <= 0 + or receipt.get("encrypted_bytes") != receipt["ciphertext_bytes"] + or receipt.get("encrypted_sha256") != receipt.get("ciphertext_sha256") + or not isinstance(receipt.get("age_recipient"), str) + or not re.fullmatch(r"age1[a-z0-9]{58}", receipt["age_recipient"]) + or not isinstance(receipt.get("image"), str) + or not re.fullmatch(r"forgejo\.coulomb\.social/coulomb/coulomb-social@sha256:" + r"[a-f0-9]{64}", receipt["image"])): + raise Refused("invalid_primary_evidence") + for field in ("ciphertext_sha256", "database_sha256", "content_sha256"): + if (not isinstance(receipt.get(field), str) + or not re.fullmatch(r"[a-f0-9]{64}", receipt[field])): + raise Refused("invalid_primary_hash") + return captured + + +def validate_mirror(primary, mirrored): + validate_primary(mirrored) + ref = mirrored["recovery_mirror"] + if (any(mirrored[k] != primary[k] for k in BINDING) + or ref.get("project_id") != PROJECT or ref.get("bucket") != MIRROR_BUCKET + or ref.get("object_key") != "archives/" + primary["object_key"] + or ref.get("ciphertext_sha256") != primary["ciphertext_sha256"] + or ref.get("ciphertext_bytes") != primary["ciphertext_bytes"] + or ref.get("hash_readback_verified") is not True + or not isinstance(ref.get("version_id"), str) + or not ref["version_id"] or ref["version_id"] == "null"): + raise Refused("mirror_binding_differs") + + +def check(state, mirror_receipt, now, *, min_free_bytes, max_state_bytes, + disk_usage=shutil.disk_usage): + if (now.tzinfo is None or type(min_free_bytes) is not int or min_free_bytes < 0 + or type(max_state_bytes) is not int or max_state_bytes <= 0): + raise ValueError("Explicit nonnegative reserve and positive state budget required") + report = { + "schema": "coulomb.paired-backup-health.v1", "status": "unhealthy", "reasons": [], + "observed_at": now.astimezone(UTC).isoformat(), + "provider_contacted": False, "recorded_readback_only": True, + "state_modified": False, "notification_sent": False, "operator_reception_proven": False, + "minimum_free_bytes": min_free_bytes, "maximum_state_bytes": max_state_bytes, + } + reasons = report["reasons"] + try: + info = state.lstat() + if (not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() + or info.st_mode & 0o077 or state.resolve() != state.absolute()): + raise Refused("unsafe_state_directory") + total, pending = 0, [] + def walk_error(error): + raise error + + for directory, directories, files in os.walk(state, followlinks=False, onerror=walk_error): + for name in directories: + path = Path(directory) / name + entry = path.lstat() + if (not stat.S_ISDIR(entry.st_mode) or entry.st_uid != os.geteuid() + or entry.st_mode & 0o077): + raise Refused("unsafe_state_entry") + if Path(directory) == state and name.startswith("pending-"): + pending.append(entry) + for name in files: + total += regular(Path(directory) / name).st_size + report.update(state_bytes=total, free_bytes=disk_usage(state).free, + pending_count=len(pending)) + if total > max_state_bytes: + reasons.append("state_budget_exceeded") + if report["free_bytes"] < min_free_bytes: + reasons.append("insufficient_free_space") + busy = False + lock_path = state / "backup.lock" + if lock_path.exists() or lock_path.is_symlink(): + regular(lock_path) + fd = os.open(lock_path, os.O_RDONLY | os.O_NOFOLLOW) + try: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + busy = True + finally: + os.close(fd) + report["backup_process_holds_lock"] = busy + if pending: + ages = [now.timestamp() - p.st_mtime for p in pending] + if len(pending) != 1: + reasons.append("multiple_pending_captures") + elif not busy: + reasons.append("pending_requires_review") + elif min(ages) < 0 or max(ages) > 35 * 60: + reasons.append("backup_stalled") + receipt = read_json(state / "last-success.json") + captured = validate_primary(receipt) + age = (now - captured).total_seconds() + report["capture_age_seconds"] = round(age, 3) + if age < 0: + reasons.append("capture_timestamp_in_future") + elif age > 26 * 3600: + reasons.append("backup_stale") + archive = state / receipt["local_archive"] + archive_info = regular(archive) + with archive.open("rb") as stream: + digest = hashlib.file_digest(stream, "sha256").hexdigest() + if (archive_info.st_size != receipt["ciphertext_bytes"] + or digest != receipt["ciphertext_sha256"]): + reasons.append("local_ciphertext_differs") + try: + validate_mirror(receipt, read_json(mirror_receipt)) + report["matching_mirror_readback_recorded"] = True + except (OSError, ValueError, KeyError, TypeError, AttributeError, OverflowError): + reasons.append("matching_mirror_evidence_missing_or_invalid") + except (OSError, ValueError, KeyError, TypeError, AttributeError, OverflowError): + # No paths, values, receipt bodies or raw OS/provider errors enter output. + reasons.append("state_evidence_missing_or_invalid") + report["reasons"] = sorted(set(reasons)) + if not reasons: + report["status"] = "healthy" + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--state-dir", type=Path, default=Path("/var/lib/coulomb-social-backup")) + parser.add_argument("--mirror-receipt", type=Path, required=True) + parser.add_argument("--minimum-free-bytes", type=int, required=True) + parser.add_argument("--maximum-state-bytes", type=int, required=True) + args = parser.parse_args() + result = check(args.state_dir, args.mirror_receipt, datetime.now(UTC), + min_free_bytes=args.minimum_free_bytes, max_state_bytes=args.maximum_state_bytes) + print(json.dumps(result, sort_keys=True)) + return int(result["status"] != "healthy") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_backup_health.py b/tests/test_backup_health.py new file mode 100644 index 0000000..8405675 --- /dev/null +++ b/tests/test_backup_health.py @@ -0,0 +1,163 @@ +"""Operating checks must expose pending/unmirrored points without side effects.""" + +import copy +import fcntl +import hashlib +import importlib.util +import json +import os +from collections import namedtuple +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +NOW = datetime(2026, 10, 30, 1, 0, tzinfo=UTC) + + +@pytest.fixture +def monitor(): + source = Path(__file__).resolve().parents[1] / "scripts/check-paired-backup-health.py" + spec = importlib.util.spec_from_file_location("backup_health", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def save(path, value): + path.write_text(json.dumps(value)) + path.chmod(0o600) + + +@pytest.fixture +def health_state(tmp_path, monitor): + state = tmp_path / "state" + state.mkdir(mode=0o700) + name = "20261030T000000Z-" + "a" * 24 + ".tar.age" + ciphertext = b"synthetic encrypted paired point" + (state / name).write_bytes(ciphertext) + (state / name).chmod(0o600) + digest = hashlib.sha256(ciphertext).hexdigest() + primary = { + "schema": "platform.application-primary-archive.v1", "application": "coulomb-social", + "status": "offsite_fetched_verified", "captured_at": "2026-10-30T00:00:00Z", + "local_archive": name, "object_key": monitor.PREFIX + "2026/10/30/" + name, + "version_id": "primary-version", "immutable_create": True, + "remote_retrieval_verified": True, "ciphertext_sha256": digest, + "encrypted_sha256": digest, "ciphertext_bytes": len(ciphertext), + "encrypted_bytes": len(ciphertext), "database_sha256": "b" * 64, + "content_sha256": "c" * 64, "age_recipient": "age1" + "q" * 58, + "image": "forgejo.coulomb.social/coulomb/coulomb-social@sha256:" + "d" * 64, + } + primary["destination"] = "s3://railiance-platform-pg-backup/" + primary["object_key"] + mirrored = copy.deepcopy(primary) + mirrored["recovery_mirror"] = { + "project_id": monitor.PROJECT, "bucket": monitor.MIRROR_BUCKET, + "object_key": "archives/" + primary["object_key"], "version_id": "mirror-version", + "ciphertext_sha256": digest, "ciphertext_bytes": len(ciphertext), + "hash_readback_verified": True, + } + save(state / "last-success.json", primary) + save(state / "last-mirror.json", mirrored) + return state, primary, mirrored + + +def check(monitor, state, now=NOW, **kwargs): + defaults = {"min_free_bytes": 100, "max_state_bytes": 100000, + "disk_usage": lambda path: namedtuple("Disk", "free")(10000)} + defaults.update(kwargs) + return monitor.check(state, state / "last-mirror.json", now, **defaults) + + +def test_healthy_point_has_exact_mirror_and_checker_writes_nothing(monitor, health_state): + state, primary, _ = health_state + before = {str(p): (p.read_bytes(), p.stat().st_mtime_ns) for p in state.iterdir()} + result = check(monitor, state) + assert result["status"] == "healthy" and not result["reasons"] + assert result["matching_mirror_readback_recorded"] + assert not result["provider_contacted"] and not result["notification_sent"] + assert not result["operator_reception_proven"] + assert {str(p): (p.read_bytes(), p.stat().st_mtime_ns) for p in state.iterdir()} == before + assert primary["object_key"] not in json.dumps(result) + + +@pytest.mark.parametrize("age,reason", [(27, "backup_stale"), (-1, "capture_timestamp_in_future")]) +def test_capture_age_controls_rpo_not_receipt_mtime(monitor, health_state, age, reason): + state, _, _ = health_state + now = datetime(2026, 10, 30, tzinfo=UTC) + timedelta(hours=age) + assert reason in check(monitor, state, now)["reasons"] + assert check(monitor, state, datetime(2026, 10, 31, 2, tzinfo=UTC))["status"] == "healthy" + + +@pytest.mark.parametrize("field", ["version_id", "object_key", "ciphertext_sha256", "image"]) +def test_old_or_foreign_mirror_does_not_cover_new_primary(monitor, health_state, field): + state, _, mirrored = health_state + mirrored[field] = "different" + save(state / "last-mirror.json", mirrored) + assert "matching_mirror_evidence_missing_or_invalid" in check(monitor, state)["reasons"] + + +def test_missing_or_forged_readback_and_corrupted_ciphertext_fail(monitor, health_state): + state, primary, mirrored = health_state + mirrored["recovery_mirror"]["hash_readback_verified"] = False + save(state / "last-mirror.json", mirrored) + (state / primary["local_archive"]).write_bytes(b"damaged") + result = check(monitor, state) + assert set(result["reasons"]) == { + "matching_mirror_evidence_missing_or_invalid", "local_ciphertext_differs"} + (state / "last-mirror.json").unlink() + assert "matching_mirror_evidence_missing_or_invalid" in check(monitor, state)["reasons"] + + +def test_recent_success_does_not_hide_abandoned_pending_capture(monitor, health_state): + state, _, _ = health_state + pending = state / "pending-synthetic" + pending.mkdir(mode=0o700) + os.utime(pending, (NOW.timestamp(), NOW.timestamp())) + assert "pending_requires_review" in check(monitor, state)["reasons"] + + +def test_active_capture_gets_bounded_grace_but_stalled_lock_fails(monitor, health_state): + state, _, _ = health_state + pending = state / "pending-synthetic" + pending.mkdir(mode=0o700) + os.utime(pending, (NOW.timestamp(), NOW.timestamp())) + lock = state / "backup.lock" + lock.touch(mode=0o600) + with lock.open("r") as stream: + fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + assert check(monitor, state)["status"] == "healthy" + assert "backup_stalled" in check(monitor, state, NOW + timedelta(minutes=36))["reasons"] + second = state / "pending-other" + second.mkdir(mode=0o700) + assert "multiple_pending_captures" in check(monitor, state)["reasons"] + + +def test_explicit_capacity_limits_are_enforced(monitor, health_state): + state, _, _ = health_state + result = check(monitor, state, min_free_bytes=10001, max_state_bytes=1) + assert set(result["reasons"]) == {"insufficient_free_space", "state_budget_exceeded"} + + +@pytest.mark.parametrize("entry", ["receipt", "archive", "pending"]) +def test_symlinks_fail_without_following_or_printing_targets( + monitor, health_state, tmp_path, entry, +): + state, primary, _ = health_state + outside = tmp_path / "private-unrelated" + outside.write_bytes(b"UNRELATED SECRET SYNTHETIC") + path = state / {"receipt": "last-success.json", "archive": primary["local_archive"], + "pending": "pending-link"}[entry] + path.unlink(missing_ok=True) + path.symlink_to(outside) + result = check(monitor, state) + assert result["status"] == "unhealthy" + assert b"UNRELATED" not in json.dumps(result).encode() + assert outside.read_bytes() == b"UNRELATED SECRET SYNTHETIC" + + +def test_missing_invalid_or_public_state_refuses_with_redacted_reason(monitor, tmp_path): + state = tmp_path / "missing" + assert check(monitor, state)["status"] == "unhealthy" + state.mkdir(mode=0o755) + assert check(monitor, state)["reasons"] == ["state_evidence_missing_or_invalid"] diff --git a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md index 0d0fb31..22f3d58 100644 --- a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md +++ b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md @@ -8,7 +8,7 @@ status: active owner: codex topic_slug: coulomb-social created: "2026-10-08" -updated: "2026-10-29" +updated: "2026-10-30" origin: residual origin_ref: CSOC-WP-0008 state_hub_workstream_id: "0cd1f513-f26e-5707-b646-8b4bbe6369fc" @@ -169,7 +169,7 @@ remain open. No timer or production restore was activated. ```task id: CSOC-WP-0012-T03 -status: todo +status: progress priority: high state_hub_task_id: "ded6fc1d-0819-57c5-9213-e64db76bc6ed" ``` @@ -188,6 +188,20 @@ assumptions are historical; founder-approved paired schedule already exists in RPF-WP-0053 and does not need repeat schedule approval. Current findings and next returns: docs/release/acceptance-audit-2026-10-27.md. +2026-10-30: Replaced the candidate freshness unit's narrow last-success check with +a read-only operating checker covering pending/stalled captures, exact primary +and mirror receipt binding, local ciphertext integrity and explicit capacity +parameters. A fresh previous point no longer conceals an abandoned newer capture. +Native node execution (without installation) found the primary fresh, no pending +capture, 2,123,252 state bytes and 119,716,765,696 free bytes, but no matching worker +mirror receipt. Both timers remain not-found/inactive. The first independently +verified external mirror/kit is intact; recurring mirror/receipt promotion is +still missing. No provider contact, alert send, key read, node write or activation. +35 focused tests and source systemd verification pass. Required operating return +and proposed capacity limits: docs/release/backup-operating-preflight-2026-10-30.md; +native metadata/hash proof is its JSON companion. Received monitoring, retention, +pre-capture capacity guard and service-contract returns remain incomplete. + ## Completion Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records.