tegwick
08906da7d0
Some checks failed
Use-case regression gate / usecase-regressions (push) Has been cancelled
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a18e2d-f513-7742-b8e7-12af0b744ff3
385 lines
14 KiB
Python
385 lines
14 KiB
Python
"""Seal only eight completed historical evidence entries; preserve bytes and paths."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import stat
|
|
import subprocess
|
|
import tempfile
|
|
import types
|
|
from contextlib import contextmanager
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
STATE = Path("/var/lib/coulomb-social-backup")
|
|
JOURNALS = Path("/var/lib/coulomb-social-backup-custody")
|
|
DATE = "47829c7ac2e6fd2c81fde1c914aec7c6e433098fa66985db6cfb9b34069942dc"
|
|
IMAGE = "9f1daf6c5fb56b2b4fc9817f28f5ad090c884b0295f7d6f4469e42d48fa8d59f"
|
|
STORAGE = "5fd7ccd8f68057346367d81efb82b48280a16dd31d87683a5bd83e60e3d1837e"
|
|
PRIVATE = "date-operations/" + DATE + "/private"
|
|
PATHS = (
|
|
"releases",
|
|
"releases/" + IMAGE + "/capsule/charts",
|
|
"releases/" + IMAGE + "/capsule/release",
|
|
"releases/" + STORAGE + "/capsule/charts",
|
|
PRIVATE,
|
|
PRIVATE + "/result.json",
|
|
PRIVATE + "/journal.json",
|
|
PRIVATE + "/execute.lock",
|
|
)
|
|
CLUSTER = "a553c742-0115-43d4-99a4-a5ca56fe0786"
|
|
|
|
|
|
class Refused(ValueError):
|
|
pass
|
|
|
|
|
|
def require(ok, reason):
|
|
if not ok:
|
|
raise Refused(reason)
|
|
|
|
|
|
def canonical(value):
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
|
|
|
|
|
|
def sha(raw):
|
|
return hashlib.sha256(raw).hexdigest()
|
|
|
|
|
|
@contextmanager
|
|
def opened(relative):
|
|
parts = Path(relative).parts
|
|
require(
|
|
not Path(relative).is_absolute() and ".." not in parts and parts, "invalid_relative_path"
|
|
)
|
|
handles = []
|
|
try:
|
|
fd = os.open(STATE, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
handles.append(fd)
|
|
info = os.fstat(fd)
|
|
require(info.st_uid == os.geteuid() and not info.st_mode & 0o077, "private_state_required")
|
|
for index, part in enumerate(parts):
|
|
if index:
|
|
info = os.fstat(fd)
|
|
require(
|
|
info.st_uid in (os.geteuid(), 10001) and not info.st_mode & 0o022,
|
|
"unsafe_ancestor",
|
|
)
|
|
flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK
|
|
if index < len(parts) - 1:
|
|
flags |= os.O_DIRECTORY
|
|
fd = os.open(part, flags, dir_fd=fd)
|
|
handles.append(fd)
|
|
yield fd
|
|
finally:
|
|
for fd in reversed(handles):
|
|
os.close(fd)
|
|
|
|
|
|
def fingerprint(relative):
|
|
with opened(relative) as fd:
|
|
info = os.fstat(fd)
|
|
directory = stat.S_ISDIR(info.st_mode)
|
|
require(directory or stat.S_ISREG(info.st_mode) and info.st_nlink == 1, "unsupported_entry")
|
|
if directory:
|
|
raw = json.dumps(sorted(os.listdir(fd)), separators=(",", ":")).encode()
|
|
else:
|
|
require(info.st_size <= 1024 * 1024, "oversized_evidence")
|
|
raw = b""
|
|
while chunk := os.read(fd, 65536):
|
|
raw += chunk
|
|
require(len(raw) <= 1024 * 1024, "growing_evidence")
|
|
after = os.fstat(fd)
|
|
require(
|
|
(info.st_mtime_ns, info.st_size, info.st_uid, info.st_gid, info.st_mode)
|
|
== (after.st_mtime_ns, after.st_size, after.st_uid, after.st_gid, after.st_mode),
|
|
"evidence_changed_during_read",
|
|
)
|
|
return dict(
|
|
relative_path=relative,
|
|
kind="directory" if directory else "regular",
|
|
mode=oct(stat.S_IMODE(info.st_mode)),
|
|
uid=info.st_uid,
|
|
gid=info.st_gid,
|
|
nlink=info.st_nlink,
|
|
bytes=info.st_size,
|
|
inode=info.st_ino,
|
|
device=info.st_dev,
|
|
mtime_ns=info.st_mtime_ns,
|
|
sha256=sha(raw),
|
|
hash_kind="directory-entry-names" if directory else "file-bytes",
|
|
)
|
|
|
|
|
|
@contextmanager
|
|
def locked():
|
|
from contextlib import ExitStack
|
|
|
|
with ExitStack() as stack:
|
|
for path in (
|
|
"date-operations/supervisor.lock",
|
|
"backup.lock",
|
|
"snapshot.lock",
|
|
PRIVATE + "/execute.lock",
|
|
):
|
|
fd = stack.enter_context(opened(path))
|
|
info = os.fstat(fd)
|
|
require(
|
|
stat.S_ISREG(info.st_mode)
|
|
and info.st_nlink == 1
|
|
and info.st_uid in (os.geteuid(), 10001)
|
|
and not info.st_mode & 0o077,
|
|
"unsafe_operation_lock",
|
|
)
|
|
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
yield
|
|
|
|
|
|
def native_fence(plan):
|
|
for timer in ("coulomb-social-paired-backup.timer", "coulomb-social-backup-freshness.timer"):
|
|
result = subprocess.run(
|
|
["systemctl", "show", timer, "--property=ActiveState,UnitFileState"],
|
|
capture_output=True,
|
|
timeout=15,
|
|
)
|
|
require(result.returncode == 0, "timer_observation_failed")
|
|
values = dict(
|
|
line.split("=", 1) for line in result.stdout.decode().splitlines() if "=" in line
|
|
)
|
|
require(
|
|
values.get("ActiveState") == "inactive"
|
|
and values.get("UnitFileState", "") in ("", "disabled", "static"),
|
|
"timers_must_remain_inactive",
|
|
)
|
|
|
|
def kube(*args):
|
|
result = subprocess.run(
|
|
["k3s", "kubectl", *args, "-o", "json"], capture_output=True, timeout=30
|
|
)
|
|
require(
|
|
result.returncode == 0 and len(result.stdout) < 16 * 1024 * 1024, "native_read_failed"
|
|
)
|
|
return json.loads(result.stdout)
|
|
|
|
require(kube("get", "namespace", "kube-system")["metadata"]["uid"] == CLUSTER, "wrong_cluster")
|
|
for entry in plan["receipts"]:
|
|
with opened(entry["relative_path"]) as fd:
|
|
info = os.fstat(fd)
|
|
require(
|
|
stat.S_ISREG(info.st_mode)
|
|
and info.st_nlink == 1
|
|
and info.st_uid == 0
|
|
and not info.st_mode & 0o077
|
|
and info.st_size < 65536,
|
|
"unsafe_receipt",
|
|
)
|
|
raw = os.read(fd, 65537)
|
|
require(sha(raw) == entry["sha256"], "historical_receipt_changed")
|
|
receipt = json.loads(raw)
|
|
require(
|
|
receipt["pid"] == entry["pid"] and not Path("/proc", str(entry["pid"])).exists(),
|
|
"historical_process_present",
|
|
)
|
|
status = (
|
|
receipt.get("phase")
|
|
if entry["relative_path"].startswith("date-")
|
|
else receipt.get("status")
|
|
)
|
|
require(status == entry["terminal_status"], "historical_operation_not_terminal")
|
|
pods = kube("get", "pods", "-A")["items"]
|
|
for pod in pods:
|
|
require(
|
|
pod["metadata"]["name"] != "csoc-date-" + DATE[:20]
|
|
and pod["metadata"]["uid"] != "60c3b803-69d3-45da-9b09-19a6c0b274da",
|
|
"historical_helper_present",
|
|
)
|
|
for volume in pod["spec"].get("volumes", []):
|
|
path = volume.get("hostPath", {}).get("path")
|
|
if path:
|
|
require(
|
|
not (
|
|
path.startswith(str(STATE)) or str(STATE).startswith(path.rstrip("/") + "/")
|
|
),
|
|
"state_hostpath_mount_present",
|
|
)
|
|
|
|
|
|
def desired(before):
|
|
return dict(before, uid=0, gid=0, mode="0o700" if before["kind"] == "directory" else "0o600")
|
|
|
|
|
|
def save(directory, value):
|
|
fd, name = tempfile.mkstemp(prefix=".receipt-", dir=directory)
|
|
with os.fdopen(fd, "wb") as stream:
|
|
stream.write(canonical(value) + b"\n")
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(name, directory / "receipt.json")
|
|
fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY)
|
|
try:
|
|
os.fsync(fd)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def change(row, before, after):
|
|
# Descriptor operations cannot follow a replacement final symlink.
|
|
require(fingerprint(row["relative_path"]) == before, "entry_changed_before_metadata_write")
|
|
with opened(row["relative_path"]) as fd:
|
|
info = os.fstat(fd)
|
|
require((info.st_ino, info.st_dev) == (before["inode"], before["device"]), "entry_replaced")
|
|
if (before["uid"], before["gid"]) != (after["uid"], after["gid"]):
|
|
os.fchown(fd, after["uid"], after["gid"])
|
|
if before["mode"] != after["mode"]:
|
|
os.fchmod(fd, int(after["mode"], 8))
|
|
os.fsync(fd)
|
|
require(fingerprint(row["relative_path"]) == after, "metadata_postflight_differs")
|
|
|
|
|
|
def transact(plan, directory, *, rollback=False, postflight=lambda: None):
|
|
rows = plan["targets"]
|
|
current = {row["relative_path"]: fingerprint(row["relative_path"]) for row in rows}
|
|
for row in rows:
|
|
allowed = (row, desired(row)) if rollback else (row,)
|
|
require(current[row["relative_path"]] in allowed, "evidence_drift_requires_review")
|
|
record = dict(
|
|
schema="coulomb.completed-evidence-custody.v1",
|
|
proposal_digest=plan["proposal_digest"],
|
|
status="rollback_started" if rollback else "seal_started",
|
|
started_at=datetime.now(UTC).isoformat(),
|
|
changes=[],
|
|
bytes_modified=False,
|
|
paths_moved=False,
|
|
data_deleted=False,
|
|
capture_executed=False,
|
|
timers_activated=False,
|
|
)
|
|
if rollback:
|
|
fd = os.open(directory / "receipt.json", os.O_RDONLY | os.O_NOFOLLOW)
|
|
with os.fdopen(fd, "rb") as stream:
|
|
info = os.fstat(stream.fileno())
|
|
require(
|
|
stat.S_ISREG(info.st_mode)
|
|
and info.st_nlink == 1
|
|
and info.st_uid == os.geteuid()
|
|
and not info.st_mode & 0o077
|
|
and info.st_size < 1024 * 1024,
|
|
"private_prior_receipt_required",
|
|
)
|
|
prior = json.load(stream)
|
|
require(prior["proposal_digest"] == plan["proposal_digest"], "rollback_proposal_differs")
|
|
# Retain the earlier evidence rather than replace its outcome history.
|
|
record["prior_receipt"] = prior
|
|
else:
|
|
directory.mkdir(mode=0o700, parents=True, exist_ok=False)
|
|
save(directory, record)
|
|
try:
|
|
for row in reversed(rows) if rollback else rows:
|
|
before = current[row["relative_path"]]
|
|
after = row if rollback else desired(row)
|
|
if before == after:
|
|
continue
|
|
record["changes"].append(dict(path=row["relative_path"], before=before, after=after))
|
|
save(directory, record)
|
|
change(row, before, after)
|
|
for row in rows:
|
|
require(
|
|
fingerprint(row["relative_path"]) == (row if rollback else desired(row)),
|
|
"whole_scope_postflight_differs",
|
|
)
|
|
record["postflight"] = postflight()
|
|
record.update(
|
|
status="original_metadata_restored_verified"
|
|
if rollback
|
|
else "completed_evidence_private_root_custody_verified",
|
|
verified_entries=len(rows),
|
|
)
|
|
except Exception as error:
|
|
record.update(status="requires_inspection", failure_type=type(error).__name__)
|
|
raise
|
|
finally:
|
|
record["updated_at"] = datetime.now(UTC).isoformat()
|
|
save(directory, record)
|
|
return record
|
|
|
|
|
|
def run(plan, action, capacity_source=None):
|
|
require(__debug__ and os.geteuid() == 0, "attended_root_required")
|
|
require(action in ("preflight", "seal", "rollback"), "unknown_action")
|
|
scope = dict(plan)
|
|
value = scope.pop("proposal_digest")
|
|
require(value == "sha256:" + sha(canonical(scope)), "proposal_digest_differs")
|
|
require(
|
|
tuple(row["relative_path"] for row in plan["targets"]) == PATHS,
|
|
"exact_eight_paths_required",
|
|
)
|
|
require(len(plan["receipts"]) == 3, "three_completed_receipts_required")
|
|
require(
|
|
tuple(entry["relative_path"] for entry in plan["receipts"])
|
|
== (
|
|
"releases/" + STORAGE + "/receipt.json",
|
|
"releases/" + IMAGE + "/receipt.json",
|
|
"date-operations/" + DATE + "/receipt.json",
|
|
),
|
|
"exact_historical_receipts_required",
|
|
)
|
|
for row in plan["targets"]:
|
|
require(
|
|
row["uid"] == row["gid"] == (10001 if row["relative_path"].startswith(PRIVATE) else 0),
|
|
"original_ownership_differs",
|
|
)
|
|
require(
|
|
isinstance(capacity_source, str)
|
|
and sha(capacity_source.encode()) == plan["capacity_source_sha256"],
|
|
"capacity_source_differs",
|
|
)
|
|
capacity = types.ModuleType("reviewed_backup_capacity")
|
|
exec(compile(capacity_source, "<reviewed-backup-capacity>", "exec"), capacity.__dict__)
|
|
directory = JOURNALS / plan["proposal_digest"].removeprefix("sha256:")
|
|
require(directory.name.isalnum() and len(directory.name) == 64, "invalid_proposal_digest")
|
|
for parent in (JOURNALS, JOURNALS.parent):
|
|
if parent.exists() or parent.is_symlink():
|
|
info = parent.lstat()
|
|
require(
|
|
stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and not info.st_mode & 0o022,
|
|
"unsafe_journal_parent",
|
|
)
|
|
if parent == JOURNALS:
|
|
require(not info.st_mode & 0o077, "private_journal_root_required")
|
|
with locked():
|
|
native_fence(plan)
|
|
if action == "preflight":
|
|
require(
|
|
all(fingerprint(row["relative_path"]) == row for row in plan["targets"]),
|
|
"evidence_drift_requires_review",
|
|
)
|
|
require(not directory.exists(), "existing_attempt_requires_inspection")
|
|
return dict(
|
|
status="completed_evidence_seal_preflight_verified",
|
|
proposal_digest=plan["proposal_digest"],
|
|
entries=8,
|
|
modified=False,
|
|
historical_operations_terminal=True,
|
|
)
|
|
if action == "rollback":
|
|
info = directory.lstat()
|
|
require(
|
|
stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and not info.st_mode & 0o077,
|
|
"private_attempt_required",
|
|
)
|
|
else:
|
|
JOURNALS.mkdir(mode=0o700, exist_ok=True)
|
|
|
|
def postflight():
|
|
native_fence(plan)
|
|
result = {"historical_operations_still_terminal": True}
|
|
if action == "seal":
|
|
result["state_bytes"] = capacity.state_usage(STATE)
|
|
result["whole_state_custody_guard_passed"] = True
|
|
return result
|
|
|
|
return transact(plan, directory, rollback=action == "rollback", postflight=postflight)
|