diff --git a/deploy/backup/coulomb-social-paired-backup.service b/deploy/backup/coulomb-social-paired-backup.service index 8affb50..ace483a 100644 --- a/deploy/backup/coulomb-social-paired-backup.service +++ b/deploy/backup/coulomb-social-paired-backup.service @@ -21,6 +21,8 @@ ExecStartPost=/opt/railiance-platform/backup-venv/bin/python /opt/coulomb-social TimeoutStartSec=30min TimeoutStopSec=5min KillMode=mixed +# Candidate operating profile, also lowered by each writer before execution. +LimitFSIZE=1G NoNewPrivileges=true PrivateTmp=true ProtectSystem=strict diff --git a/docs/release/backup-capacity-review-2026-10-30.json b/docs/release/backup-capacity-review-2026-10-30.json new file mode 100644 index 0000000..011a4f8 --- /dev/null +++ b/docs/release/backup-capacity-review-2026-10-30.json @@ -0,0 +1,50 @@ +{ + "credentials_read": false, + "installed_helpers_changed": false, + "limits_owner_admitted": false, + "provider_contacted": false, + "provider_retention_enforced": false, + "read_only_node_observation": { + "active_file_limit_bytes": 1073741824, + "bounded_file_executor": true, + "disk_reserve_bytes": 8606711808, + "filesystem_blocks_reserved": false, + "free_bytes": 118842212352, + "free_inodes": 24630813, + "limits_owner_admitted": false, + "maximum_new_file_bytes": 1073741824, + "minimum_free_bytes": 8589934592, + "minimum_free_inodes": 1024, + "phase": "capture", + "provider_contacted": false, + "reasons": [], + "retained_data_deleted": false, + "schema": "coulomb.backup-capacity.v1", + "state_budget_bytes": 8589934592, + "state_bytes": 2134016, + "state_modified": false, + "state_reserve_bytes": 7532969984, + "status": "ready" + }, + "schema": "coulomb.backup-capacity-review.v1", + "source_sha256": { + "scripts/backup_capacity.py": "cb699a79e67b129239b5aa5d6f3b0d4ffaa9773a8b073227ad17fdecb5b9d893", + "scripts/mirror-paired-backup.py": "9ceb26d9083af016c4b787907e4dce7bdf437c782fb2d15eb28d9e334e6e431c", + "scripts/paired-backup.py": "d161c742d1a6a182460a57d0a1719591ab58fc94006f55169ad3d74442a93630", + "scripts/snapshot-content-and-db.py": "f98604294fdba5f1664529bd932592a641a22bb73ec4583bbabba8a9591cf8e3" + }, + "status": "native_candidate_capacity_verified", + "timers_activated": false, + "unit_verification": { + "status": "verified", + "temporary_review_files_removed": true, + "unit_sha256": { + "coulomb-social-backup-freshness.service": "aaed68bc4bce752eb55a22c1b14bf74df306f1b0c017bd5bdfaff0f0943878d8", + "coulomb-social-backup-freshness.timer": "75d1d0721c07fdd0703910c06bb8a678396440e7d95411a37ee7de4938f3ac31", + "coulomb-social-paired-backup.service": "fe637fb31ef077be4edbbc94540fcce4ab1badde6a6a0e232bed4f0de5bf4324", + "coulomb-social-paired-backup.timer": "5e87235e66f89a64a972562d6f163f62420628fc44fb7fd8d603a96c347dfbb9" + }, + "units_installed": false, + "units_started": false + } +} diff --git a/docs/release/backup-capacity-review-2026-10-30.md b/docs/release/backup-capacity-review-2026-10-30.md new file mode 100644 index 0000000..ef25ded --- /dev/null +++ b/docs/release/backup-capacity-review-2026-10-30.md @@ -0,0 +1,77 @@ +# Pre-capture capacity and file-size review + +The candidate capture worker now checks capacity inside backup.lock before +creating a pending capture or pausing the app. The mirror worker uses the same +guard inside that lock before a readback or provider request. Refusal preserves +existing pending and retained files; no automatic retention deletion is added. + +## Candidate operating profile + +| Parameter | Proposed limit | +| --- | --- | +| Individual newly written file | 1 GiB | +| Local state budget | 8 GiB | +| Free space remaining after scratch reserve | 8 GiB | +| Minimum available inodes | 1,024 | +| Control-file/directory margin | 16 MiB | + +New capture reserves seven maximum-size files plus the control margin: paired +dump/content, tar, ciphertext, readback, retained local copy and rewritten +inventory. A sealed retry reserves three new files; existing pending files, +including earlier partial readbacks, still count in current usage. A mirror +reserves one new ciphertext readback. Each phase additionally reserves one file +outside state for the command-output sink. State usage counts the larger of +logical bytes and allocated blocks and includes directories. Symlinks, hardlinked +files, public/non-owned entries or an unreadable subtree refuse the check. + +Each executing writer lowers RLIMIT_FSIZE without raising an existing stricter +limit. The candidate unit also sets LimitFSIZE=1G for all commands and descendants. +An unbounded executor refuses before capture. A file that exceeds the limit fails +the write; it is not recorded as a successful truncated backup. The existing +snapshot finally/resume and pending-review paths remain in force. Backups larger +than this operating profile require owner review of limits/capacity rather than +silently exceeding them. + +These are proposed limits, not admitted capacity/spending authority. The guard +does not reserve filesystem blocks, implement a total-directory quota, bound +memory use or control unrelated writers. Later disk pressure can still cause a +failed capture/publication requiring its existing cleanup and pending review. +Provider generation counts, lifecycle compatibility and billing remain a separate +retention-owner return. The monitor threshold alone does not enforce retention. + +## Native and isolated verification + +`backup-capacity-review-2026-10-30.json` binds the exact sources, read-only node +observation and exact candidate-unit verification. In a disposable bounded review +process, current allocated/logical state usage was 2,134,016 bytes; free space +118,842,212,352 bytes and free inodes 24,630,813. The proposed capture scratch +reservation fits while preserving the free-space floor. Only the review process +received the 1 GiB file limit; no installed unit or host configuration changed. + +Tests prove disk, state-budget, inode and unbounded-executor refusal before child +commands, pending creation or publication. A sealed retry that lacks space leaves +its exact pending bytes unchanged. An actual child process inherits a 64 KiB test +file limit, receives EFBIG on an oversized write, runs its finally cleanup and +keeps the partial file bounded. Applying a larger requested limit afterward does +not raise the existing stricter limit. Mirror capacity failure makes no provider +call. The pinned-engine synthetic rehearsal also passes with the real candidate +file limit and the guard enabled; its receipt includes the capacity source hash. +The full `make check` gate passed all 304 tests, including mandatory browser journeys. + +Exact updated units passed systemd-analyze verification on the target using only +temporary review files, removed afterward. Existing installed capture hashes were +read back unchanged: paired worker43e119ac and snapshot helperf9860429. + +## Delivery order and remaining return + +The pending storage proposal3389 still expects the installed paired-worker hash +43e119ac. Its frozen digest is unchanged. Do not install the new operating worker +before that release is executed or its proposal is explicitly rebased/reviewed. +This source change has not replaced the installed helper or activated timers. + +The later operating delivery must include backup_capacity.py beside both writers, +the four pinned platform helpers and manifest, exact units and the owner-admitted +profile. Native mirror/receipt publication plus independent retrieval, received +failed/stale signals, compatible rolling retention and historical-copy disposition +remain required before recurring activation. Existing custody and schedule +approvals do not need to be requested again. diff --git a/docs/release/mirror-worker-operating-review-2026-10-30.md b/docs/release/mirror-worker-operating-review-2026-10-30.md index 88093f2..0e7183f 100644 --- a/docs/release/mirror-worker-operating-review-2026-10-30.md +++ b/docs/release/mirror-worker-operating-review-2026-10-30.md @@ -83,8 +83,12 @@ that exact receipt still need the reviewed operating execution. Then bind received failed/pending/stale alerts, finish rolling provider retention and the pre-capture capacity guard, resolve historical-copy disposition, install exact reviewed sources/units and prove service runs before enabling approved timers. -Local state remains bounded only by the proposed monitor threshold; this worker -does not enforce provider retention or a pre-capture budget. +The capacity follow-through is now prepared in +backup-capacity-review-2026-10-30.md: both writers check the proposed budget and +scratch reserve while holding the backup lock, with an inherited per-file limit. +The shared backup_capacity.py must accompany the operating delivery. Native +publication/receipt retrieval, owner admission and provider retention still remain +open. The guard is not a filesystem quota or a provider-expiry executor. Existing custody/schedule approvals remain valid. This preparation changes no grant, credential, provider object, installed helper, active timer or production diff --git a/docs/release/mirror-worker-rehearsal-2026-10-30.json b/docs/release/mirror-worker-rehearsal-2026-10-30.json index ed9e164..5aa8ac7 100644 --- a/docs/release/mirror-worker-rehearsal-2026-10-30.json +++ b/docs/release/mirror-worker-rehearsal-2026-10-30.json @@ -1,10 +1,11 @@ { + "capacity_sha256": "cb699a79e67b129239b5aa5d6f3b0d4ffaa9773a8b073227ad17fdecb5b9d893", "checker_sha256": "c003871fe91c5c31788fd9bfc50aef3b2a275c06c5a42c2c3165b92d5ea80db9", "credentials_read": false, - "driver_sha256": "26d60db0675b76dc0e305a7296c727eddd8ac3b280f93a48cfb4e883d5e4a8e8", + "driver_sha256": "a4ea8dd30fbcaa14018111065c836953b4e64689ef400341cf771ecad66d2e78", "fixture_sha256": { "test_backup_health.py": "6d9397ff77d00f0d17ca8a6cb1ec81f09c8859ad450b1b2a673d19914b90b6f8", - "test_backup_mirror_worker.py": "827649cbc1a2bc56c883cf45bae2d700d0e7a5886721d3917c9fc5ac57127589" + "test_backup_mirror_worker.py": "04639222bbac0721e82ff1f71cbada55492e1761106470347b93683c7a78c772" }, "node_files_changed": false, "provider_contacted": false, @@ -45,5 +46,5 @@ "status": "pinned_engine_synthetic_integration_passed", "synthetic_store": true, "timers_activated": false, - "worker_sha256": "f1209356b6ccdfcf434d5806eb104a11e1493caf6d6138986bf067116648e64a" + "worker_sha256": "9ceb26d9083af016c4b787907e4dce7bdf437c782fb2d15eb28d9e334e6e431c" } diff --git a/scripts/backup_capacity.py b/scripts/backup_capacity.py new file mode 100644 index 0000000..aa76d4a --- /dev/null +++ b/scripts/backup_capacity.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Local backup capacity checks; no deletion, provider contact or app changes. + +Candidate limits need operating admission. Estimates reserve complete pipeline +scratch, while RLIMIT_FSIZE bounds individual newly written files. This is not +a filesystem quota or a reservation against unrelated writers. +""" + +from __future__ import annotations + +import argparse +import json +import os +import resource +import shutil +import stat +from pathlib import Path + +FILE_LIMIT = 1024**3 +STATE_BUDGET = 8 * 1024**3 +MINIMUM_FREE = 8 * 1024**3 +CONTROL_MARGIN = 16 * 1024**2 +MINIMUM_INODES = 1024 + + +class Refused(ValueError): + pass + + +def active_file_limit(): + return resource.getrlimit(resource.RLIMIT_FSIZE)[0] + + +def enforce_file_limit(maximum=FILE_LIMIT): + soft, hard = resource.getrlimit(resource.RLIMIT_FSIZE) + effective = min( + [maximum] + [value for value in [soft, hard] if value != resource.RLIM_INFINITY] + ) + if effective <= 0: + raise Refused("positive_file_limit_required") + resource.setrlimit(resource.RLIMIT_FSIZE, (effective, effective)) + return effective + + +def disk_free(path): + return shutil.disk_usage(path).free + + +def free_inodes(path): + return os.statvfs(path).f_favail + + +def state_usage(root): + info = root.lstat() + if ( + not stat.S_ISDIR(info.st_mode) + or info.st_uid != os.geteuid() + or info.st_mode & 0o077 + or root.resolve() != root.absolute() + ): + raise Refused("private_state_directory_required") + + def onerror(error): + raise error + + used = max(info.st_size, info.st_blocks * 512) + for directory, directories, files in os.walk(root, followlinks=False, onerror=onerror): + for name in directories + files: + item = (Path(directory) / name).lstat() + if ( + not (stat.S_ISREG(item.st_mode) or stat.S_ISDIR(item.st_mode)) + or item.st_uid != os.geteuid() + or item.st_mode & 0o077 + or (stat.S_ISREG(item.st_mode) and item.st_nlink != 1) + ): + raise Refused("unsafe_state_entry") + used += max(item.st_size, item.st_blocks * 512) + return used + + +def reserve_plan( + state, + *, + phase="capture", + require_file_limit=True, + maximum_file=FILE_LIMIT, + state_budget=STATE_BUDGET, + minimum_free=MINIMUM_FREE, +): + if phase not in {"capture", "resume", "mirror"} or any( + type(v) is not int or v <= 0 for v in [maximum_file, state_budget, minimum_free] + ): + raise Refused("explicit_positive_capacity_profile_required") + # New capture: dump, content, tar, ciphertext, readback, retained local copy, + # and temporary rewritten inventory. Sealed retry needs three new large + # files; mirror needs only its fresh ciphertext readback. + slots = {"capture": 7, "resume": 3, "mirror": 1}[phase] + state_reserve = slots * maximum_file + CONTROL_MARGIN + # The existing command wrapper uses one anonymous output sink outside state. + disk_reserve = state_reserve + maximum_file + used, available, inodes = state_usage(state), disk_free(state), free_inodes(state) + limit = active_file_limit() + bounded = limit != resource.RLIM_INFINITY and 0 < limit <= maximum_file + reasons = [] + if used + state_reserve > state_budget: + reasons.append("local_retention_budget_required") + if available < minimum_free + disk_reserve: + reasons.append("disk_headroom_required") + if inodes < MINIMUM_INODES: + reasons.append("inode_headroom_required") + if require_file_limit and not bounded: + reasons.append("bounded_file_executor_required") + return { + "schema": "coulomb.backup-capacity.v1", + "phase": phase, + "status": "refused" if reasons else ("ready" if bounded else "reviewed_unbounded_executor"), + "reasons": reasons, + "state_bytes": used, + "state_budget_bytes": state_budget, + "state_reserve_bytes": state_reserve, + "disk_reserve_bytes": disk_reserve, + "free_bytes": available, + "minimum_free_bytes": minimum_free, + "free_inodes": inodes, + "minimum_free_inodes": MINIMUM_INODES, + "maximum_new_file_bytes": maximum_file, + "active_file_limit_bytes": limit, + "bounded_file_executor": bounded, + "state_modified": False, + "retained_data_deleted": False, + "provider_contacted": False, + "filesystem_blocks_reserved": False, + "limits_owner_admitted": False, + } + + +def require_capacity(state, *, phase="capture"): + report = reserve_plan(state, phase=phase) + if report["status"] != "ready": + raise Refused("backup_capacity_review_required") + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--state-dir", type=Path, required=True) + parser.add_argument("--phase", choices=["capture", "resume", "mirror"], default="capture") + args = parser.parse_args() + print( + json.dumps( + reserve_plan(args.state_dir, phase=args.phase, require_file_limit=False), sort_keys=True + ) + ) + + +if __name__ == "__main__": + try: + main() + except (ValueError, OSError): + raise SystemExit("Capacity review refused; inspect private state locally.") diff --git a/scripts/mirror-paired-backup.py b/scripts/mirror-paired-backup.py index da074fd..54a289f 100644 --- a/scripts/mirror-paired-backup.py +++ b/scripts/mirror-paired-backup.py @@ -21,6 +21,8 @@ import sys import tempfile from pathlib import Path +import backup_capacity as capacity + spec = importlib.util.spec_from_file_location( "csoc_backup_health", Path(__file__).with_name("check-paired-backup-health.py") ) @@ -195,6 +197,7 @@ def mirror_latest(state, engine, client): fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: raise Refused("backup_or_mirror_already_running") from None + capacity.require_capacity(state, phase="mirror") primary = health.read_json(state / "last-success.json") original = canonical(primary) health.validate_primary(primary) @@ -266,6 +269,7 @@ def main(): } if any(not value or any(c in value for c in "\r\n") for value in pair.values()): raise Refused("own_publisher_environment_required") + capacity.enforce_file_limit() client = engine.s3(pair, health.PROJECT) try: print(json.dumps(mirror_latest(args.state_dir, engine, client))) diff --git a/scripts/paired-backup.py b/scripts/paired-backup.py index e2d8b02..3dc7fa2 100644 --- a/scripts/paired-backup.py +++ b/scripts/paired-backup.py @@ -24,6 +24,8 @@ import tempfile from datetime import UTC, datetime from pathlib import Path +import backup_capacity as capacity + class Refused(RuntimeError): pass @@ -169,6 +171,11 @@ def run_backup( else: if pending: raise Refused("pending_backup_requires_review") + # Hold the writer lock across capacity check and capture. A sealed retry + # needs no new dump/tar/encryption; all previous scratch still counts. + phase = "resume" if resume and (stage / "sealed.json").is_file() else "capture" + capacity.require_capacity(state, phase=phase) + if not resume: stage = Path(tempfile.mkdtemp(prefix="pending-", dir=state)) pair = stage / "pair" helper = Path(__file__).with_name("snapshot-content-and-db.py") @@ -338,6 +345,7 @@ def main() -> int: or not args.primary_python.is_file() ): raise Refused("invalid_backup_configuration") + capacity.enforce_file_limit() # Fail before pausing the app if encryption or credential custody is absent. # Encrypt an empty input into the captured output sink to validate the # actual recipient/checksum before any capture or application pause. diff --git a/scripts/rehearse-backup-mirror.py b/scripts/rehearse-backup-mirror.py index 74bd091..99726fd 100644 --- a/scripts/rehearse-backup-mirror.py +++ b/scripts/rehearse-backup-mirror.py @@ -27,9 +27,12 @@ def fixture(directory, worker): state, primary, _ = health_state.__wrapped__(directory, worker.health) data = b"age-encryption.org/v1\n" + b"SYNTHETIC CIPHERTEXT ONLY\n" (state / primary["local_archive"]).write_bytes(data) - primary.update(ciphertext_sha256=hashlib.sha256(data).hexdigest(), - encrypted_sha256=hashlib.sha256(data).hexdigest(), - ciphertext_bytes=len(data), encrypted_bytes=len(data)) + primary.update( + ciphertext_sha256=hashlib.sha256(data).hexdigest(), + encrypted_sha256=hashlib.sha256(data).hexdigest(), + ciphertext_bytes=len(data), + encrypted_bytes=len(data), + ) (state / "last-success.json").write_text(json.dumps(primary)) (state / "last-mirror.json").unlink() return state, primary @@ -38,8 +41,9 @@ def fixture(directory, worker): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--platform-scripts", type=Path, required=True) - parser.add_argument("--runtime-manifest", type=Path, - default=ROOT / "deploy/backup/mirror-platform-runtime.json") + parser.add_argument( + "--runtime-manifest", type=Path, default=ROOT / "deploy/backup/mirror-platform-runtime.json" + ) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() if args.output.exists() or args.output.is_symlink(): @@ -49,6 +53,7 @@ def main(): worker = importlib.util.module_from_spec(spec) spec.loader.exec_module(worker) runtime = worker.load_runtime(args.platform_scripts, args.runtime_manifest) + worker.capacity.enforce_file_limit() scenarios = [] for lost_reply in [False, True]: with tempfile.TemporaryDirectory(prefix="csoc-mirror-engine-") as directory: @@ -70,42 +75,63 @@ def main(): ref = mirrored["recovery_mirror"] held = client.objects[ref["receipt_object_key"]]["Body"] assert hashlib.sha256(held).hexdigest() == ref["receipt_sha256"] - assert client.objects[ref["object_key"]]["Body"] == ( - state / primary["local_archive"]).read_bytes() + assert ( + client.objects[ref["object_key"]]["Body"] + == (state / primary["local_archive"]).read_bytes() + ) assert (state / "last-success.json").read_bytes() == before assert len(client.objects) == 2 assert sum(op == "complete" for op, _ in client.calls) == 1 assert sum(op == "put" for op, _ in client.calls) == 1 assert all(a["Bucket"] == worker.health.MIRROR_BUCKET for _, a in client.calls) - assert all(a["Key"].startswith("archives/" + primary["object_key"]) - for _, a in client.calls) + assert all( + a["Key"].startswith("archives/" + primary["object_key"]) for _, a in client.calls + ) assert result["receipt_readback_verified"] - scenarios.append({ - "lost_receipt_put_reply": lost_reply, "ciphertext_multipart_completions": 1, - "metadata_conditional_puts": 1, "retained_objects": 2, - "primary_unchanged": True, "exact_version_hash_readbacks": True, - "last_mirror_only_after_full_readback": True, - "only_coulomb_archive_prefix_used": True, - }) + scenarios.append( + { + "lost_receipt_put_reply": lost_reply, + "ciphertext_multipart_completions": 1, + "metadata_conditional_puts": 1, + "retained_objects": 2, + "primary_unchanged": True, + "exact_version_hash_readbacks": True, + "last_mirror_only_after_full_readback": True, + "only_coulomb_archive_prefix_used": True, + } + ) proof = { "schema": "coulomb.mirror-worker-rehearsal.v1", "status": "pinned_engine_synthetic_integration_passed", "worker_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "capacity_sha256": hashlib.sha256( + (ROOT / "scripts/backup_capacity.py").read_bytes() + ).hexdigest(), "checker_sha256": hashlib.sha256( - (ROOT / "scripts/check-paired-backup-health.py").read_bytes()).hexdigest(), + (ROOT / "scripts/check-paired-backup-health.py").read_bytes() + ).hexdigest(), "driver_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), - "fixture_sha256": {name: hashlib.sha256((ROOT / "tests" / name).read_bytes()).hexdigest() - for name in ["test_backup_health.py", "test_backup_mirror_worker.py"]}, + "fixture_sha256": { + name: hashlib.sha256((ROOT / "tests" / name).read_bytes()).hexdigest() + for name in ["test_backup_health.py", "test_backup_mirror_worker.py"] + }, "runtime_manifest": json.loads(args.runtime_manifest.read_bytes()), - "scenarios": scenarios, "synthetic_store": True, "provider_contacted": False, - "credentials_read": False, "node_files_changed": False, "timers_activated": False, + "scenarios": scenarios, + "synthetic_store": True, + "provider_contacted": False, + "credentials_read": False, + "node_files_changed": False, + "timers_activated": False, } fd = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) with os.fdopen(fd, "w") as stream: json.dump(proof, stream, indent=2, sort_keys=True) stream.write("\n") - print(json.dumps({"status": proof["status"], "scenarios": len(scenarios), - "provider_contacted": False})) + print( + json.dumps( + {"status": proof["status"], "scenarios": len(scenarios), "provider_contacted": False} + ) + ) if __name__ == "__main__": diff --git a/tests/test_backup_capacity.py b/tests/test_backup_capacity.py new file mode 100644 index 0000000..c6d7f6b --- /dev/null +++ b/tests/test_backup_capacity.py @@ -0,0 +1,136 @@ +"""Capacity checks happen inside writer locks; actual file limits run in children.""" + +import importlib.util +import resource +import subprocess +import sys +from pathlib import Path + +import pytest + +from tests.test_paired_backup import NOW, setup +from tests.test_paired_backup import backup as backup + + +@pytest.fixture +def capacity(monkeypatch): + source = Path(__file__).resolve().parents[1] / "scripts/backup_capacity.py" + spec = importlib.util.spec_from_file_location("capacity_cases", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "active_file_limit", lambda: module.FILE_LIMIT) + monkeypatch.setattr(module, "disk_free", lambda path: 128 * 1024**3) + monkeypatch.setattr(module, "free_inodes", lambda path: 1000000) + return module + + +@pytest.mark.parametrize("phase,slots", [("capture", 7), ("resume", 3), ("mirror", 1)]) +def test_reserve_includes_complete_phase_scratch_and_preserves_state( + capacity, tmp_path, phase, slots +): + tmp_path.chmod(0o700) + original = tmp_path / "retained.age" + original.write_bytes(b"retained synthetic ciphertext") + original.chmod(0o600) + before = original.read_bytes() + plan = capacity.require_capacity(tmp_path, phase=phase) + assert plan["status"] == "ready" + assert plan["state_reserve_bytes"] == slots * capacity.FILE_LIMIT + capacity.CONTROL_MARGIN + assert plan["disk_reserve_bytes"] == plan["state_reserve_bytes"] + capacity.FILE_LIMIT + assert not plan["filesystem_blocks_reserved"] and not plan["limits_owner_admitted"] + assert list(tmp_path.iterdir()) == [original] and original.read_bytes() == before + + +@pytest.mark.parametrize("condition", ["disk", "budget", "inodes", "unbounded"]) +def test_limits_refuse_before_capture_or_pending_creation(backup, monkeypatch, tmp_path, condition): + state, calls, publications = setup(backup, monkeypatch, tmp_path) + if condition == "disk": + monkeypatch.setattr(backup.capacity, "disk_free", lambda path: 1) + elif condition == "budget": + monkeypatch.setattr(backup.capacity, "state_usage", lambda path: 8 * 1024**3) + elif condition == "inodes": + monkeypatch.setattr(backup.capacity, "free_inodes", lambda path: 0) + else: + monkeypatch.setattr(backup.capacity, "active_file_limit", lambda: resource.RLIM_INFINITY) + with pytest.raises(ValueError, match="capacity_review_required"): + backup.run_backup(state, tmp_path / "primary.py", "age1-synthetic", NOW) + assert not calls and not publications and not list(state.glob("pending-*")) + + +def test_read_only_preflight_distinguishes_unbounded_executor(capacity, tmp_path, monkeypatch): + tmp_path.chmod(0o700) + monkeypatch.setattr(capacity, "active_file_limit", lambda: resource.RLIM_INFINITY) + report = capacity.reserve_plan(tmp_path, require_file_limit=False) + assert report["status"] == "reviewed_unbounded_executor" + assert not report["bounded_file_executor"] and not list(tmp_path.iterdir()) + with pytest.raises(ValueError, match="capacity_review_required"): + capacity.require_capacity(tmp_path) + + +def test_state_scan_counts_allocated_blocks_and_refuses_symlinks(capacity, tmp_path): + tmp_path.chmod(0o700) + path = tmp_path / "metadata.json" + path.write_bytes(b"{}") + path.chmod(0o600) + assert capacity.state_usage(tmp_path) >= max(path.stat().st_size, path.stat().st_blocks * 512) + (tmp_path / "pending-link").symlink_to(path) + with pytest.raises(ValueError, match="unsafe_state_entry"): + capacity.state_usage(tmp_path) + + +def test_resume_capacity_refusal_keeps_exact_pending_evidence(backup, monkeypatch, tmp_path): + state, _, _ = setup(backup, monkeypatch, tmp_path, "upload") + with pytest.raises(backup.Refused): + backup.run_backup(state, tmp_path / "primary.py", "age1-public-test-key", NOW) + before = {str(p): p.read_bytes() for p in state.rglob("*") if p.is_file()} + _, calls, publications = setup(backup, monkeypatch, tmp_path) + monkeypatch.setattr(backup.capacity, "disk_free", lambda path: 1) + with pytest.raises(ValueError, match="capacity_review_required"): + backup.run_backup(state, tmp_path / "primary.py", "age1-public-test-key", NOW, resume=True) + assert {str(p): p.read_bytes() for p in state.rglob("*") if p.is_file()} == before + assert not calls and not publications + + +def test_real_inherited_file_limit_raises_write_error_and_runs_cleanup(tmp_path): + root = Path(__file__).resolve().parents[1] + code = ''' +import errno,json,subprocess,sys +from pathlib import Path +sys.path.insert(0,sys.argv[1]) +import backup_capacity as capacity +assert capacity.enforce_file_limit(65536)==65536 +assert capacity.enforce_file_limit(1073741824)==65536 +child = r""" +import errno,json,resource,sys +from pathlib import Path +root=Path(sys.argv[1]) +try: + (root/'bounded.bin').write_bytes(b'x'*131072) +except OSError as error: + assert error.errno==errno.EFBIG + print(json.dumps({'bounded':resource.getrlimit(resource.RLIMIT_FSIZE)[0]==65536})) +else: + raise AssertionError('file write escaped inherited cap') +finally: + (root/'cleanup-ran').write_text('cleanup') +""" +result=subprocess.run([sys.executable,'-c',child,sys.argv[2]],capture_output=True,text=True) +assert result.returncode==0 +assert json.loads(result.stdout)['bounded'] +assert (Path(sys.argv[2])/'cleanup-ran').read_text()=='cleanup' +assert (Path(sys.argv[2])/'bounded.bin').stat().st_size<=65536 +print('inherited limit and finally cleanup verified') +''' + result = subprocess.run( + [sys.executable, "-c", code, str(root / "scripts"), str(tmp_path)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert "cleanup verified" in result.stdout + + +def test_service_applies_file_limit_to_capture_and_mirror_commands(): + root = Path(__file__).resolve().parents[1] + service = (root / "deploy/backup/coulomb-social-paired-backup.service").read_text() + assert "\nLimitFSIZE=1G\n" in service diff --git a/tests/test_backup_mirror_worker.py b/tests/test_backup_mirror_worker.py index 71d80f9..ad34a38 100644 --- a/tests/test_backup_mirror_worker.py +++ b/tests/test_backup_mirror_worker.py @@ -95,11 +95,15 @@ class MemoryS3: @pytest.fixture -def worker(): +def worker(monkeypatch): source = Path(__file__).resolve().parents[1] / "scripts/mirror-paired-backup.py" + monkeypatch.syspath_prepend(str(source.parent)) spec = importlib.util.spec_from_file_location("mirror_worker", source) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + monkeypatch.setattr(module.capacity, "active_file_limit", lambda: module.capacity.FILE_LIMIT) + monkeypatch.setattr(module.capacity, "disk_free", lambda path: 128 * 1024**3) + monkeypatch.setattr(module.capacity, "free_inodes", lambda path: 1000000) return module @@ -224,6 +228,16 @@ def test_tampered_ciphertext_refuses_before_publication(worker, health_state, en assert not client.calls and not engine.calls +def test_capacity_failure_refuses_mirror_before_provider_calls( + worker, health_state, engine, monkeypatch, +): + monkeypatch.setattr(worker.capacity, "disk_free", lambda path: 1) + client = MemoryS3() + with pytest.raises(ValueError, match="capacity_review_required"): + run(worker, health_state, engine, client) + assert not client.calls and not engine.calls + + def test_preview_needs_no_runtime_credentials_or_state(worker, tmp_path): environment = {k: v for k, v in os.environ.items() if not k.startswith("AWS_")} result = subprocess.run( diff --git a/tests/test_paired_backup.py b/tests/test_paired_backup.py index 82c8bc3..47133d8 100644 --- a/tests/test_paired_backup.py +++ b/tests/test_paired_backup.py @@ -14,11 +14,18 @@ import pytest @pytest.fixture -def backup(): +def backup(monkeypatch): path = Path(__file__).resolve().parents[1] / "scripts/paired-backup.py" + monkeypatch.syspath_prepend(str(path.parent)) spec = importlib.util.spec_from_file_location("csoc_paired_backup", path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + # Other cases test publication/capture under an already bounded executor. + # Actual RLIMIT behavior is exercised in separate disposable subprocesses. + monkeypatch.setattr(module.capacity, "active_file_limit", lambda: module.capacity.FILE_LIMIT) + monkeypatch.setattr(module.capacity, "disk_free", lambda path: 128 * 1024**3) + monkeypatch.setattr(module.capacity, "free_inodes", lambda path: 1000000) + monkeypatch.setattr(module.capacity, "enforce_file_limit", lambda: module.capacity.FILE_LIMIT) return module diff --git a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md index 3787b55..4be22d9 100644 --- a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md +++ b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md @@ -217,6 +217,20 @@ docs/release/mirror-worker-operating-review-2026-10-30.md and mirror-worker-rehearsal-2026-10-30.json. Native publication/independent retrieval, received alerts, retention and pre-capture capacity remain required. +Capacity follow-through: prepared a shared capacity guard inside each writer's +backup lock, before capture/provider work. Candidate profile reserves the complete +pipeline's scratch, keeps an 8 GiB free-space floor and 8 GiB local budget, checks +inodes/private state and bounds new files to 1 GiB through inherited RLIMIT_FSIZE. +Sealed retry reserves less new scratch and preserves all previous pending bytes. +Actual subprocess tests prove oversized-write refusal and finally cleanup. +Read-only bounded native preflight fits current 2,134,016 state bytes against +118,842,212,352 free bytes; exact candidate units verify. Installed old helpers +remain unchanged and storage proposal3389 remains sealed; new operating sources +must not replace its expected helper before release execution or explicit rebase. +Review/proof: docs/release/backup-capacity-review-2026-10-30.{md,json}. Limits are +still proposed; provider retention, native operating publication/retrieval, +received alerts and historical-copy disposition remain open. No timer activated. + ## Completion Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records.