663 lines
35 KiB
Python
663 lines
35 KiB
Python
|
"""Node supervisor for an exact founder-reviewed date-correction capsule.
|
||
|
|
|
||
|
|
Default is read-only. This tool is not an approval service: only the founder may
|
||
|
|
run execute/recover in their terminal under the deployment owner's supervision.
|
||
|
|
Uncertain attempts retain their helper, journals and stopped application.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import base64
|
||
|
|
import copy
|
||
|
|
import fcntl
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import stat
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
import tempfile
|
||
|
|
from contextlib import ExitStack, contextmanager
|
||
|
|
from datetime import UTC, datetime
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
NS = "coulomb-social"
|
||
|
|
CLAIM = "coulomb-social-content"
|
||
|
|
STATE_ROOT = Path("/var/lib/coulomb-social-backup/date-operations")
|
||
|
|
MEMBERS = {"supervisor.py", "wire.py", "date_backfill.py", "date_backfill_journal.py",
|
||
|
|
"date_backfill_operation.py", "plan.json"}
|
||
|
|
HELPERS = {"/opt/coulomb-social-backup/paired-backup.py",
|
||
|
|
"/opt/coulomb-social-backup/snapshot-content-and-db.py",
|
||
|
|
"/opt/railiance-platform/scripts/primary_application_archive.py"}
|
||
|
|
|
||
|
|
|
||
|
|
def canonical(value):
|
||
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode()
|
||
|
|
|
||
|
|
|
||
|
|
def sha(data):
|
||
|
|
return hashlib.sha256(data).hexdigest()
|
||
|
|
|
||
|
|
|
||
|
|
class Refused(RuntimeError):
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
def private_file(path):
|
||
|
|
info = path.lstat()
|
||
|
|
if (not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o077
|
||
|
|
or info.st_nlink != 1 or any(p.is_symlink() for p in path.parents)):
|
||
|
|
raise Refused("private_root_file_required")
|
||
|
|
return path.read_bytes()
|
||
|
|
|
||
|
|
|
||
|
|
def sync_directory(path):
|
||
|
|
descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||
|
|
try:
|
||
|
|
os.fsync(descriptor)
|
||
|
|
finally:
|
||
|
|
os.close(descriptor)
|
||
|
|
|
||
|
|
|
||
|
|
def save(path, value):
|
||
|
|
fd, temporary = tempfile.mkstemp(prefix=".state-", dir=path.parent)
|
||
|
|
try:
|
||
|
|
with os.fdopen(fd, "wb") as stream:
|
||
|
|
stream.write(canonical(value))
|
||
|
|
stream.flush()
|
||
|
|
os.fsync(stream.fileno())
|
||
|
|
os.replace(temporary, path)
|
||
|
|
directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
|
||
|
|
try:
|
||
|
|
os.fsync(directory)
|
||
|
|
finally:
|
||
|
|
os.close(directory)
|
||
|
|
finally:
|
||
|
|
if os.path.exists(temporary):
|
||
|
|
os.unlink(temporary)
|
||
|
|
|
||
|
|
|
||
|
|
class Capsule:
|
||
|
|
def __init__(self, path, expected):
|
||
|
|
self.manifest = json.loads((path / "manifest.json").read_bytes())
|
||
|
|
if "sha256:" + sha(canonical(self.manifest)) != expected:
|
||
|
|
raise Refused("proposal_digest_changed")
|
||
|
|
self.digest = expected
|
||
|
|
if (self.manifest.get("schema") != "coulomb.supervised-date-capsule.v1"
|
||
|
|
or set(self.manifest["files"]) != MEMBERS
|
||
|
|
or self.manifest.get("requires_founder_attended_execution") is not True
|
||
|
|
or not re.fullmatch(r"[a-f0-9]{32}", self.manifest.get("attempt_id", ""))):
|
||
|
|
raise Refused("exact_capsule_members_required")
|
||
|
|
self.content = {}
|
||
|
|
for name, digest in self.manifest["files"].items():
|
||
|
|
member = path / name
|
||
|
|
if not member.is_file() or any(p.is_symlink() for p in [member, *member.parents]):
|
||
|
|
raise Refused("capsule_links_refused")
|
||
|
|
data = member.read_bytes()
|
||
|
|
if sha(data) != digest:
|
||
|
|
raise Refused("capsule_member_changed")
|
||
|
|
self.content[name] = data
|
||
|
|
self.target = self.manifest["target"]
|
||
|
|
fields = {"cluster_uid", "deployment_uid", "pvc_uid", "node_name", "node_uid", "image",
|
||
|
|
"deployment_spec_sha256", "helm_revision", "helm_manifest_sha256",
|
||
|
|
"public_recipient", "installed_helpers"}
|
||
|
|
if (set(self.target) != fields or set(self.target["installed_helpers"]) != HELPERS
|
||
|
|
or not re.fullmatch(r"[a-z0-9.-]+", self.target["node_name"])
|
||
|
|
or not re.fullmatch(r"forgejo\.coulomb\.social/coulomb/"
|
||
|
|
r"coulomb-social@sha256:[a-f0-9]{64}", self.target["image"])):
|
||
|
|
raise Refused("exact_application_target_required")
|
||
|
|
hashes = [self.target[n] for n in ("deployment_spec_sha256", "helm_manifest_sha256")]
|
||
|
|
hashes += list(self.target["installed_helpers"].values())
|
||
|
|
if (any(not re.fullmatch(r"[a-f0-9]{64}", h) for h in hashes)
|
||
|
|
or type(self.target["helm_revision"]) is not int or self.target["helm_revision"] < 1
|
||
|
|
or not self.target["public_recipient"].startswith("age1")):
|
||
|
|
raise Refused("exact_target_pins_required")
|
||
|
|
self.plan = json.loads(self.content["plan.json"])
|
||
|
|
self.baseline = self.manifest["baseline"]
|
||
|
|
if (self.plan.get("schema") != "coulomb.source-date-backfill-plan.v1"
|
||
|
|
or self.plan.get("blockers") or not self.plan.get("prepared_change_set_valid")
|
||
|
|
or set(self.baseline) != {"tables_sha256", "files_sha256",
|
||
|
|
"tables", "rows", "files"}
|
||
|
|
or any(not re.fullmatch(r"[a-f0-9]{64}", self.baseline[n])
|
||
|
|
for n in ("tables_sha256", "files_sha256"))
|
||
|
|
or any(type(self.baseline[n]) is not int or self.baseline[n] < 1
|
||
|
|
for n in ("tables", "rows", "files"))):
|
||
|
|
raise Refused("exact_reviewed_date_plan_and_baseline_required")
|
||
|
|
|
||
|
|
def payload(self, mode, backup=None):
|
||
|
|
names = ("date_backfill.py", "date_backfill_journal.py", "date_backfill_operation.py")
|
||
|
|
return {"mode": mode, "plan": self.plan, "baseline": self.baseline,
|
||
|
|
"database_name": "coulomb_social_db",
|
||
|
|
"image": self.target["image"], "recipient": self.target["public_recipient"],
|
||
|
|
"backup": backup,
|
||
|
|
"modules": {n: base64.b64encode(self.content[n]).decode() for n in names},
|
||
|
|
"module_hashes": {n: self.manifest["files"][n] for n in names}}
|
||
|
|
|
||
|
|
|
||
|
|
class Node:
|
||
|
|
def __init__(self):
|
||
|
|
self.env = {**os.environ, "KUBECONFIG": "/etc/rancher/k3s/k3s.yaml"}
|
||
|
|
|
||
|
|
def command(self, arguments, payload=None, *, env=None, timeout=210):
|
||
|
|
with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr:
|
||
|
|
process = subprocess.Popen(arguments,
|
||
|
|
stdin=subprocess.PIPE if payload else subprocess.DEVNULL,
|
||
|
|
stdout=stdout, stderr=stderr, env=env or self.env)
|
||
|
|
try:
|
||
|
|
process.communicate(payload, timeout=timeout)
|
||
|
|
except BaseException:
|
||
|
|
process.terminate()
|
||
|
|
try:
|
||
|
|
process.wait(timeout=400) # permit admitted backup's resume cleanup
|
||
|
|
except subprocess.TimeoutExpired:
|
||
|
|
process.kill()
|
||
|
|
process.wait()
|
||
|
|
raise Refused("child_outcome_requires_review") from None
|
||
|
|
stdout.seek(0)
|
||
|
|
data = stdout.read(8 * 1024 * 1024 + 1)
|
||
|
|
if process.returncode or len(data) > 8 * 1024 * 1024:
|
||
|
|
raise Refused("private_child_refused")
|
||
|
|
return data
|
||
|
|
|
||
|
|
def doc(self, *arguments):
|
||
|
|
return json.loads(self.command(["kubectl", *arguments, "-o", "json"]))
|
||
|
|
|
||
|
|
def verify_local_node(self, node):
|
||
|
|
machine = Path("/etc/machine-id").read_text().strip()
|
||
|
|
if not machine or machine != node["status"]["nodeInfo"]["machineID"]:
|
||
|
|
raise Refused("run_on_the_pinned_content_node")
|
||
|
|
|
||
|
|
def installation_pins(self):
|
||
|
|
helpers = {}
|
||
|
|
for name in HELPERS:
|
||
|
|
path = Path(name)
|
||
|
|
info = path.lstat()
|
||
|
|
if (not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022
|
||
|
|
or info.st_nlink != 1 or any(p.is_symlink() for p in path.parents)):
|
||
|
|
raise Refused("installed_backup_helper_changed")
|
||
|
|
helpers[name] = sha(path.read_bytes())
|
||
|
|
fields = dict(line.split("=", 1) for line in private_file(
|
||
|
|
Path("/etc/coulomb-social-backup/recipient.env")).decode().splitlines()
|
||
|
|
if line and not line.startswith("#"))
|
||
|
|
if set(fields) != {"COULOMB_SOCIAL_BACKUP_AGE_RECIPIENT"}:
|
||
|
|
raise Refused("public_recipient_environment_required")
|
||
|
|
return {"installed_helpers": helpers,
|
||
|
|
"public_recipient": fields["COULOMB_SOCIAL_BACKUP_AGE_RECIPIENT"]}
|
||
|
|
|
||
|
|
def verify_backup_installation(self, target):
|
||
|
|
expected = {name: target[name] for name in ("installed_helpers", "public_recipient")}
|
||
|
|
if self.installation_pins() != expected:
|
||
|
|
raise Refused("installed_backup_configuration_changed")
|
||
|
|
for name in ("paired-backup", "backup-freshness"):
|
||
|
|
result = subprocess.run(["systemctl", "is-active", f"coulomb-social-{name}.timer"],
|
||
|
|
capture_output=True, timeout=15)
|
||
|
|
if result.returncode not in {3, 4}:
|
||
|
|
raise Refused("timer_state_requires_owner_review")
|
||
|
|
|
||
|
|
def capture(self, target):
|
||
|
|
env = dict(self.env)
|
||
|
|
allowed = {"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY",
|
||
|
|
"COULOMB_SOCIAL_BACKUP_AGE_RECIPIENT"}
|
||
|
|
for name in ("primary.env", "recipient.env"):
|
||
|
|
raw = private_file(Path("/etc/coulomb-social-backup") / name)
|
||
|
|
for line in raw.decode().splitlines():
|
||
|
|
if line and not line.startswith("#"):
|
||
|
|
key, value = line.split("=", 1)
|
||
|
|
if key not in allowed:
|
||
|
|
raise Refused("unexpected_backup_environment_field")
|
||
|
|
env[key] = value
|
||
|
|
if env.get("COULOMB_SOCIAL_BACKUP_AGE_RECIPIENT") != target["public_recipient"]:
|
||
|
|
raise Refused("recipient_binding_changed")
|
||
|
|
point = json.loads(self.command(
|
||
|
|
["python3", "/opt/coulomb-social-backup/paired-backup.py", "--execute"],
|
||
|
|
env=env, timeout=1200))
|
||
|
|
# Validate before scaling or executing any date code. The installed
|
||
|
|
# admitted helper owns the actual immutable publication/readback.
|
||
|
|
if (point.get("status") != "offsite_fetched_verified"
|
||
|
|
or point.get("remote_retrieval_verified") is not True
|
||
|
|
or point.get("immutable_create") is not True
|
||
|
|
or point.get("image") != target["image"]
|
||
|
|
or point.get("age_recipient") != target["public_recipient"]
|
||
|
|
or not point.get("version_id") or point["version_id"] == "null"):
|
||
|
|
raise Refused("fresh_native_capture_required")
|
||
|
|
name = point.get("local_archive", "")
|
||
|
|
if not re.fullmatch(r"\d{8}T\d{6}Z-[a-f0-9]{24}\.tar\.age", name):
|
||
|
|
raise Refused("own_local_ciphertext_required")
|
||
|
|
ciphertext = Path("/var/lib/coulomb-social-backup") / name
|
||
|
|
info = ciphertext.lstat()
|
||
|
|
if (not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o077
|
||
|
|
or info.st_nlink != 1 or info.st_size != point["ciphertext_bytes"]):
|
||
|
|
raise Refused("private_complete_ciphertext_required")
|
||
|
|
with ciphertext.open("rb") as stream:
|
||
|
|
digest = hashlib.file_digest(stream, "sha256").hexdigest()
|
||
|
|
captured = datetime.fromisoformat(point["captured_at"].replace("Z", "+00:00"))
|
||
|
|
completed = datetime.fromisoformat(point["completed_at"].replace("Z", "+00:00"))
|
||
|
|
now = datetime.now(UTC)
|
||
|
|
prefix = "platform-pg/application-archives/coulomb-social/"
|
||
|
|
if (point.get("application") != NS or digest != point["ciphertext_sha256"]
|
||
|
|
or digest != point["encrypted_sha256"] or captured.tzinfo is None
|
||
|
|
or completed.tzinfo is None or not captured <= completed <= now
|
||
|
|
or (now - captured).total_seconds() > 600
|
||
|
|
or not point["object_key"].startswith(prefix)
|
||
|
|
or not point["object_key"].endswith("/" + name)
|
||
|
|
or point["destination"] != "s3://railiance-platform-pg-backup/"
|
||
|
|
+ point["object_key"]):
|
||
|
|
raise Refused("fresh_native_ciphertext_binding_changed")
|
||
|
|
return point
|
||
|
|
|
||
|
|
def prepare_root(self, root):
|
||
|
|
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||
|
|
info = root.lstat()
|
||
|
|
if (not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o077
|
||
|
|
or any(p.is_symlink() for p in root.parents)):
|
||
|
|
raise Refused("private_supervisor_directory_required")
|
||
|
|
sync_directory(root)
|
||
|
|
sync_directory(root.parent)
|
||
|
|
|
||
|
|
@contextmanager
|
||
|
|
def operation_lock(self, root):
|
||
|
|
self.prepare_root(root)
|
||
|
|
fd = os.open(root / "supervisor.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
|
||
|
|
with os.fdopen(fd, "w"):
|
||
|
|
info = os.fstat(fd)
|
||
|
|
if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid()
|
||
|
|
or info.st_mode & 0o077 or info.st_nlink != 1):
|
||
|
|
raise Refused("private_supervisor_lock_required")
|
||
|
|
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
|
|
yield
|
||
|
|
|
||
|
|
def lock_backups(self, stack):
|
||
|
|
for name in ("backup.lock", "snapshot.lock"):
|
||
|
|
path = Path("/var/lib/coulomb-social-backup") / name
|
||
|
|
private_file(path)
|
||
|
|
stream = stack.enter_context(path.open("r+"))
|
||
|
|
fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
|
|
|
||
|
|
def helper_directory(self, path):
|
||
|
|
if not path.exists():
|
||
|
|
path.mkdir(mode=0o700)
|
||
|
|
os.chown(path, 10001, 10001)
|
||
|
|
info = path.lstat()
|
||
|
|
if not stat.S_ISDIR(info.st_mode) or info.st_uid != 10001 or info.st_mode & 0o077:
|
||
|
|
raise Refused("private_helper_journal_directory_required")
|
||
|
|
sync_directory(path)
|
||
|
|
sync_directory(path.parent)
|
||
|
|
|
||
|
|
def read_state(self, path):
|
||
|
|
return json.loads(private_file(path))
|
||
|
|
|
||
|
|
|
||
|
|
class Supervisor:
|
||
|
|
def __init__(self, capsule, node, *, state_root=STATE_ROOT):
|
||
|
|
self.capsule, self.node = capsule, node
|
||
|
|
self.target, self.state_root = capsule.target, state_root
|
||
|
|
self.directory = state_root / capsule.digest.removeprefix("sha256:")
|
||
|
|
self.helper = "csoc-date-" + capsule.digest.removeprefix("sha256:")[:20]
|
||
|
|
self.state = None
|
||
|
|
|
||
|
|
def record(self, phase, **changes):
|
||
|
|
self.state.update(changes, phase=phase, updated_at=datetime.now(UTC).isoformat())
|
||
|
|
save(self.directory / "receipt.json", self.state)
|
||
|
|
|
||
|
|
def deployment(self, replicas):
|
||
|
|
dep = self.node.doc("get", "deployment", NS, "-n", NS)
|
||
|
|
spec = copy.deepcopy(dep["spec"])
|
||
|
|
if dep["metadata"]["uid"] != self.target["deployment_uid"] or spec["replicas"] != replicas:
|
||
|
|
raise Refused("deployment_identity_or_scale_changed")
|
||
|
|
spec["replicas"] = 1
|
||
|
|
if sha(canonical(spec)) != self.target["deployment_spec_sha256"]:
|
||
|
|
raise Refused("deployment_spec_changed")
|
||
|
|
if (len(spec["template"]["spec"]["containers"]) != 1
|
||
|
|
or spec["template"]["spec"]["containers"][0]["image"] != self.target["image"]
|
||
|
|
or spec["strategy"]["type"] != "Recreate"):
|
||
|
|
raise Refused("single_pinned_recreate_writer_required")
|
||
|
|
claims = [(v["name"], v["persistentVolumeClaim"]["claimName"])
|
||
|
|
for v in spec["template"]["spec"].get("volumes", [])
|
||
|
|
if "persistentVolumeClaim" in v]
|
||
|
|
if claims != [("content", CLAIM)]:
|
||
|
|
raise Refused("exact_content_claim_required")
|
||
|
|
return dep
|
||
|
|
|
||
|
|
def writers(self):
|
||
|
|
return [pod for pod in self.node.doc("get", "pods", "-n", NS)["items"]
|
||
|
|
if pod.get("status", {}).get("phase") not in {"Succeeded", "Failed"}
|
||
|
|
and any(v.get("persistentVolumeClaim", {}).get("claimName") == CLAIM
|
||
|
|
for v in pod["spec"].get("volumes", []))]
|
||
|
|
|
||
|
|
def identity(self):
|
||
|
|
if self.node.doc("get", "namespace", "kube-system")["metadata"]["uid"] != self.target[
|
||
|
|
"cluster_uid"]:
|
||
|
|
raise Refused("cluster_changed")
|
||
|
|
node = self.node.doc("get", "node", self.target["node_name"])
|
||
|
|
self.node.verify_local_node(node)
|
||
|
|
if (node["metadata"]["uid"] != self.target["node_uid"] or node["spec"].get("unschedulable")
|
||
|
|
or not any(c["type"] == "Ready" and c["status"] == "True"
|
||
|
|
for c in node["status"]["conditions"])):
|
||
|
|
raise Refused("node_changed_or_not_ready")
|
||
|
|
pvc = self.node.doc("get", "pvc", CLAIM, "-n", NS)
|
||
|
|
if pvc["metadata"]["uid"] != self.target["pvc_uid"] or pvc["status"]["phase"] != "Bound":
|
||
|
|
raise Refused("pvc_changed")
|
||
|
|
if self.node.doc("get", "hpa", "-n", NS)["items"]:
|
||
|
|
raise Refused("autoscaling_requires_owner_path")
|
||
|
|
if any(a.get("spec", {}).get("destination", {}).get("namespace") == NS for a in
|
||
|
|
self.node.doc("get", "applications.argoproj.io", "-A")["items"]):
|
||
|
|
raise Refused("gitops_requires_owner_path")
|
||
|
|
history = json.loads(self.node.command(["helm", "history", NS, "-n", NS, "-o", "json"]))
|
||
|
|
if (not history or history[-1]["status"] != "deployed"
|
||
|
|
or history[-1]["revision"] != self.target["helm_revision"]
|
||
|
|
or sha(self.node.command(["helm", "get", "manifest", NS, "-n", NS]))
|
||
|
|
!= self.target["helm_manifest_sha256"]):
|
||
|
|
raise Refused("helm_release_changed")
|
||
|
|
|
||
|
|
def wire(self, pod, mode, backup=None):
|
||
|
|
return json.loads(self.node.command(
|
||
|
|
["kubectl", "exec", "-i", "-n", NS, pod, "--", "env",
|
||
|
|
"PYTHONDONTWRITEBYTECODE=1", "python", "-c", self.capsule.content["wire.py"].decode()],
|
||
|
|
canonical(self.capsule.payload(mode, backup))))
|
||
|
|
|
||
|
|
def preflight(self):
|
||
|
|
self.identity()
|
||
|
|
dep = self.deployment(1)
|
||
|
|
pods = self.writers()
|
||
|
|
if (len(pods) != 1 or pods[0]["spec"].get("nodeName") != self.target["node_name"]
|
||
|
|
or pods[0].get("metadata", {}).get("deletionTimestamp")
|
||
|
|
or not any(c["type"] == "Ready" and c["status"] == "True"
|
||
|
|
for c in pods[0]["status"].get("conditions", []))):
|
||
|
|
raise Refused("single_ready_application_writer_required")
|
||
|
|
containers = pods[0]["spec"].get("containers", [])
|
||
|
|
statuses = pods[0]["status"].get("containerStatuses", [])
|
||
|
|
if (len(containers) != 1 or containers[0]["image"] != self.target["image"]
|
||
|
|
or len(statuses) != 1 or not statuses[0].get("imageID", "").endswith(
|
||
|
|
self.target["image"].split("@", 1)[1])):
|
||
|
|
raise Refused("actual_running_image_changed")
|
||
|
|
owner = pods[0]["metadata"]["ownerReferences"]
|
||
|
|
if len(owner) != 1 or owner[0]["kind"] != "ReplicaSet":
|
||
|
|
raise Refused("unexpected_application_writer")
|
||
|
|
rs = self.node.doc("get", "replicaset", owner[0]["name"], "-n", NS)
|
||
|
|
if not any(r["uid"] == self.target["deployment_uid"]
|
||
|
|
for r in rs["metadata"]["ownerReferences"]):
|
||
|
|
raise Refused("unexpected_application_controller")
|
||
|
|
self.node.verify_backup_installation(self.target)
|
||
|
|
result = self.wire(pods[0]["metadata"]["name"], "prepare")
|
||
|
|
if (result["fingerprint"] != self.capsule.baseline
|
||
|
|
or not result["storage_constraint_verified"]):
|
||
|
|
raise Refused("reviewed_dataset_or_storage_prerequisite_changed")
|
||
|
|
requested = self.helper_manifest(dep)
|
||
|
|
dry = json.loads(self.node.command(["kubectl", "create", "--dry-run=server",
|
||
|
|
"-f", "-", "-o", "json"], canonical(requested)))
|
||
|
|
self.validate_helper(dry["spec"], requested["spec"])
|
||
|
|
self.preview_helper_spec = sha(canonical(dry["spec"]))
|
||
|
|
return dep, result
|
||
|
|
|
||
|
|
def validate_helper(self, actual, requested):
|
||
|
|
if (len(actual.get("containers", [])) != 1 or actual.get("initContainers")
|
||
|
|
or actual.get("hostNetwork") or actual.get("hostPID") or actual.get("hostIPC")
|
||
|
|
or actual.get("automountServiceAccountToken") is not False
|
||
|
|
or actual.get("nodeName") != self.target["node_name"]
|
||
|
|
or actual.get("securityContext") != requested["securityContext"]
|
||
|
|
or actual.get("volumes") != requested["volumes"]):
|
||
|
|
raise Refused("unreviewed_helper_pod_admission")
|
||
|
|
container = actual["containers"][0]
|
||
|
|
for key in ("image", "command", "env", "envFrom", "securityContext", "resources",
|
||
|
|
"volumeMounts"):
|
||
|
|
if container.get(key) != requested["containers"][0].get(key):
|
||
|
|
raise Refused("unreviewed_helper_container_admission")
|
||
|
|
if container.get("args") or container.get("lifecycle"):
|
||
|
|
raise Refused("unreviewed_helper_process_admission")
|
||
|
|
|
||
|
|
def create_helper(self, dep):
|
||
|
|
self.node.helper_directory(self.directory / "private")
|
||
|
|
helper = self.helper_manifest(dep)
|
||
|
|
dry = json.loads(self.node.command(["kubectl", "create", "--dry-run=server",
|
||
|
|
"-f", "-", "-o", "json"], canonical(helper)))
|
||
|
|
self.validate_helper(dry["spec"], helper["spec"])
|
||
|
|
digest = sha(canonical(dry["spec"]))
|
||
|
|
if self.state.get("helper_spec_sha256", digest) != digest:
|
||
|
|
raise Refused("helper_admission_changed")
|
||
|
|
self.record("helper_create_started", helper_spec_sha256=digest, helper_uid=None)
|
||
|
|
created = json.loads(self.node.command(
|
||
|
|
["kubectl", "create", "-f", "-", "-o", "json"], canonical(helper)))
|
||
|
|
self.record("helper_created", helper_uid=created["metadata"]["uid"])
|
||
|
|
self.node.command(["kubectl", "wait", "--for=condition=Ready", f"pod/{self.helper}",
|
||
|
|
"-n", NS, "--timeout=120s"])
|
||
|
|
self.assert_fenced(helper=True)
|
||
|
|
|
||
|
|
def helper_manifest(self, dep):
|
||
|
|
app = dep["spec"]["template"]["spec"]
|
||
|
|
container = app["containers"][0]
|
||
|
|
mounts = [v for v in container.get("volumeMounts", [])
|
||
|
|
if v["mountPath"] == "/app/var/content"]
|
||
|
|
if len(mounts) != 1 or mounts[0]["name"] != "content":
|
||
|
|
raise Refused("expected_content_mount_required")
|
||
|
|
return {"apiVersion": "v1", "kind": "Pod", "metadata": {
|
||
|
|
"name": self.helper, "namespace": NS,
|
||
|
|
"annotations": {"csoc-date-proposal": self.capsule.digest}}, "spec": {
|
||
|
|
"nodeName": self.target["node_name"], "restartPolicy": "Never",
|
||
|
|
"automountServiceAccountToken": False,
|
||
|
|
"securityContext": {"runAsNonRoot": True, "runAsUser": 10001,
|
||
|
|
"runAsGroup": 10001, "fsGroup": 10001,
|
||
|
|
"seccompProfile": {"type": "RuntimeDefault"}},
|
||
|
|
"imagePullSecrets": copy.deepcopy(app.get("imagePullSecrets", [])),
|
||
|
|
"containers": [{"name": "operation", "image": self.target["image"],
|
||
|
|
"imagePullPolicy": "IfNotPresent",
|
||
|
|
"command": ["python", "-c", "import time; time.sleep(1800)"],
|
||
|
|
"envFrom": copy.deepcopy(container.get("envFrom", [])),
|
||
|
|
"env": copy.deepcopy(container.get("env", [])),
|
||
|
|
"securityContext": {"allowPrivilegeEscalation": False,
|
||
|
|
"readOnlyRootFilesystem": True,
|
||
|
|
"capabilities": {"drop": ["ALL"]}},
|
||
|
|
"resources": {"requests": {"cpu": "10m", "memory": "64Mi"},
|
||
|
|
"limits": {"cpu": "100m", "memory": "256Mi"}},
|
||
|
|
"volumeMounts": [{"name": "content", "mountPath": "/app/var/content"},
|
||
|
|
{"name": "operation", "mountPath": "/operation"},
|
||
|
|
{"name": "temporary", "mountPath": "/tmp"}]}],
|
||
|
|
"volumes": [{"name": "content", "persistentVolumeClaim": {"claimName": CLAIM}},
|
||
|
|
{"name": "operation", "hostPath": {
|
||
|
|
"path": str(self.directory / "private"), "type": "Directory"}},
|
||
|
|
{"name": "temporary", "emptyDir": {
|
||
|
|
"medium": "Memory", "sizeLimit": "32Mi"}}]}}
|
||
|
|
|
||
|
|
def assert_fenced(self, helper=False):
|
||
|
|
self.identity()
|
||
|
|
self.deployment(0)
|
||
|
|
writers = self.writers()
|
||
|
|
if helper:
|
||
|
|
if (len(writers) != 1 or writers[0]["metadata"]["name"] != self.helper
|
||
|
|
or writers[0]["metadata"]["uid"] != self.state["helper_uid"]
|
||
|
|
or sha(canonical(writers[0]["spec"])) != self.state["helper_spec_sha256"]):
|
||
|
|
raise Refused("helper_identity_or_writer_fence_changed")
|
||
|
|
elif writers:
|
||
|
|
raise Refused("application_writers_not_gone")
|
||
|
|
|
||
|
|
def finish(self, expected):
|
||
|
|
self.assert_fenced(helper=True)
|
||
|
|
if self.wire(self.helper, "probe")["fingerprint"] != expected:
|
||
|
|
raise Refused("full_dataset_postflight_changed")
|
||
|
|
self.record("helper_delete_started", expected_final=expected)
|
||
|
|
options = {"apiVersion": "v1", "kind": "DeleteOptions", "gracePeriodSeconds": 30,
|
||
|
|
"preconditions": {"uid": self.state["helper_uid"]}}
|
||
|
|
self.node.command(["kubectl", "delete", "--raw",
|
||
|
|
f"/api/v1/namespaces/{NS}/pods/{self.helper}", "-f", "-"],
|
||
|
|
canonical(options))
|
||
|
|
self.node.command(["kubectl", "wait", "--for=delete", f"pod/{self.helper}", "-n", NS,
|
||
|
|
"--timeout=120s"])
|
||
|
|
self.assert_fenced()
|
||
|
|
dep = self.deployment(0)
|
||
|
|
self.record("resume_started")
|
||
|
|
self.node.command(["kubectl", "scale", f"deployment/{NS}", "-n", NS,
|
||
|
|
"--replicas=1", "--current-replicas=0",
|
||
|
|
"--resource-version=" + dep["metadata"]["resourceVersion"]])
|
||
|
|
self.node.command(["kubectl", "rollout", "status", f"deployment/{NS}", "-n", NS,
|
||
|
|
"--timeout=180s"])
|
||
|
|
self.deployment(1)
|
||
|
|
self.record("completed", application_resumed=True)
|
||
|
|
return self.state
|
||
|
|
|
||
|
|
def execute(self):
|
||
|
|
with self.node.operation_lock(self.state_root):
|
||
|
|
return self._execute()
|
||
|
|
|
||
|
|
def _execute(self):
|
||
|
|
if self.directory.exists():
|
||
|
|
raise Refused("prior_attempt_requires_recovery_not_replay")
|
||
|
|
dep, before = self.preflight()
|
||
|
|
self.directory.mkdir(mode=0o700, parents=True)
|
||
|
|
sync_directory(self.state_root)
|
||
|
|
self.state = {"schema": "coulomb.supervised-date-attempt.v1",
|
||
|
|
"proposal_digest": self.capsule.digest, "application_resumed": False,
|
||
|
|
"pid": os.getpid(), "expected_after": before["expected_after"],
|
||
|
|
"helper_spec_sha256": self.preview_helper_spec}
|
||
|
|
frozen = self.directory / "capsule"
|
||
|
|
frozen.mkdir(mode=0o700)
|
||
|
|
for name, data in {**self.capsule.content,
|
||
|
|
"manifest.json": canonical(self.capsule.manifest)}.items():
|
||
|
|
fd = os.open(frozen / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||
|
|
with os.fdopen(fd, "wb") as stream:
|
||
|
|
stream.write(data)
|
||
|
|
stream.flush()
|
||
|
|
os.fsync(stream.fileno())
|
||
|
|
sync_directory(frozen)
|
||
|
|
self.record("capture_started")
|
||
|
|
try:
|
||
|
|
point = self.node.capture(self.target)
|
||
|
|
self.record("captured", backup=point)
|
||
|
|
_, after = self.preflight()
|
||
|
|
if after != before or self.preview_helper_spec != self.state["helper_spec_sha256"]:
|
||
|
|
raise Refused("dataset_changed_across_capture")
|
||
|
|
with ExitStack() as locks:
|
||
|
|
self.node.lock_backups(locks)
|
||
|
|
dep = self.deployment(1)
|
||
|
|
self.record("fencing")
|
||
|
|
self.node.command(["kubectl", "scale", f"deployment/{NS}", "-n", NS,
|
||
|
|
"--replicas=0", "--current-replicas=1",
|
||
|
|
"--resource-version=" + dep["metadata"]["resourceVersion"]])
|
||
|
|
self.node.command(["kubectl", "wait", "--for=delete", "pod", "-n", NS,
|
||
|
|
"-l", "app.kubernetes.io/instance=" + NS, "--timeout=120s"])
|
||
|
|
self.assert_fenced()
|
||
|
|
self.create_helper(dep)
|
||
|
|
if self.wire(self.helper, "prepare") != before:
|
||
|
|
raise Refused("dataset_changed_before_correction")
|
||
|
|
self.record("apply_started")
|
||
|
|
result = self.wire(self.helper, "apply", point)
|
||
|
|
if (result.get("status") != "completed" or not result.get("whole_store_postflight")
|
||
|
|
or result.get("fingerprint") != before["expected_after"]):
|
||
|
|
raise Refused("date_result_unconfirmed")
|
||
|
|
self.record("applied", operation=result, correction_status="completed")
|
||
|
|
return self.finish(before["expected_after"])
|
||
|
|
except BaseException as exc:
|
||
|
|
self.record("requires_review", failed_phase=self.state["phase"],
|
||
|
|
failure_reason=str(exc) if isinstance(exc, Refused) else type(exc).__name__)
|
||
|
|
raise
|
||
|
|
|
||
|
|
def recover(self):
|
||
|
|
"""Explicit attended reconciliation; never captures or replays apply."""
|
||
|
|
with self.node.operation_lock(self.state_root), ExitStack() as locks:
|
||
|
|
self.node.lock_backups(locks)
|
||
|
|
self.state = self.node.read_state(self.directory / "receipt.json")
|
||
|
|
if (self.state.get("schema") != "coulomb.supervised-date-attempt.v1"
|
||
|
|
or self.state["proposal_digest"] != self.capsule.digest):
|
||
|
|
raise Refused("original_attempt_required")
|
||
|
|
self.identity()
|
||
|
|
dep = self.node.doc("get", "deployment", NS, "-n", NS)
|
||
|
|
if dep["spec"]["replicas"] == 1:
|
||
|
|
# A lost response after scale-up must never undo later writes.
|
||
|
|
if (self.state.get("failed_phase", self.state["phase"]) != "resume_started"
|
||
|
|
or "expected_final" not in self.state):
|
||
|
|
raise Refused("running_application_requires_owner_review")
|
||
|
|
self.deployment(1)
|
||
|
|
self.node.command(["kubectl", "rollout", "status", f"deployment/{NS}",
|
||
|
|
"-n", NS, "--timeout=180s"])
|
||
|
|
self.record("completed", application_resumed=True,
|
||
|
|
resumed_reply_reconciled=True, post_resume_writes_preserved=True)
|
||
|
|
return self.state
|
||
|
|
self.deployment(0)
|
||
|
|
writers = self.writers()
|
||
|
|
if not writers:
|
||
|
|
# Lost deletion reply or interruption before helper creation:
|
||
|
|
# recreate only the same pinned helper against retained state.
|
||
|
|
terminal = [p for p in self.node.doc("get", "pods", "-n", NS)["items"]
|
||
|
|
if p["metadata"]["name"] == self.helper]
|
||
|
|
if terminal:
|
||
|
|
old = terminal[0]
|
||
|
|
if (len(terminal) != 1 or old.get("status", {}).get("phase") not in {
|
||
|
|
"Succeeded", "Failed"}
|
||
|
|
or sha(canonical(old["spec"])) != self.state.get("helper_spec_sha256")
|
||
|
|
or old["metadata"].get("annotations", {}).get("csoc-date-proposal")
|
||
|
|
!= self.capsule.digest
|
||
|
|
or (self.state.get("helper_uid") and self.state["helper_uid"]
|
||
|
|
!= old["metadata"]["uid"])):
|
||
|
|
raise Refused("terminal_helper_identity_requires_review")
|
||
|
|
self.record("terminal_helper_delete_started", helper_uid=old["metadata"]["uid"])
|
||
|
|
self.node.command(["kubectl", "delete", "--raw",
|
||
|
|
f"/api/v1/namespaces/{NS}/pods/{self.helper}", "-f", "-"],
|
||
|
|
canonical({"apiVersion": "v1", "kind": "DeleteOptions",
|
||
|
|
"gracePeriodSeconds": 30,
|
||
|
|
"preconditions": {
|
||
|
|
"uid": self.state["helper_uid"]}}))
|
||
|
|
self.node.command(["kubectl", "wait", "--for=delete", f"pod/{self.helper}",
|
||
|
|
"-n", NS, "--timeout=120s"])
|
||
|
|
self.create_helper(dep)
|
||
|
|
writers = self.writers()
|
||
|
|
if (len(writers) != 1 or writers[0]["metadata"]["name"] != self.helper
|
||
|
|
or sha(canonical(writers[0]["spec"])) != self.state.get("helper_spec_sha256")):
|
||
|
|
raise Refused("original_helper_requires_owner_review")
|
||
|
|
if (self.state.get("helper_uid")
|
||
|
|
and self.state["helper_uid"] != writers[0]["metadata"]["uid"]):
|
||
|
|
raise Refused("helper_replaced_requires_owner_review")
|
||
|
|
self.state["helper_uid"] = writers[0]["metadata"]["uid"]
|
||
|
|
self.assert_fenced(helper=True)
|
||
|
|
observed = self.wire(self.helper, "inspect", self.state.get("backup"))
|
||
|
|
status = observed.get("status")
|
||
|
|
if status == "operation_busy":
|
||
|
|
raise Refused("original_operation_still_running")
|
||
|
|
if status == "pending":
|
||
|
|
self.record("reconcile_started")
|
||
|
|
observed = self.wire(self.helper, "reconcile", self.state["backup"])
|
||
|
|
status = observed.get("status")
|
||
|
|
expected = (self.state["expected_after"] if status == "completed"
|
||
|
|
else self.capsule.baseline)
|
||
|
|
if status not in {"completed", "rolled_back", "absent"} or observed.get(
|
||
|
|
"fingerprint") != expected:
|
||
|
|
raise Refused("reconciled_dataset_requires_owner_review")
|
||
|
|
self.record("reconciled", correction_status=status)
|
||
|
|
return self.finish(expected)
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
||
|
|
parser.add_argument("--capsule", type=Path, required=True)
|
||
|
|
parser.add_argument("--expect-digest", required=True)
|
||
|
|
modes = parser.add_mutually_exclusive_group()
|
||
|
|
modes.add_argument("--execute", action="store_true")
|
||
|
|
modes.add_argument("--recover", action="store_true")
|
||
|
|
args = parser.parse_args()
|
||
|
|
capsule = Capsule(args.capsule, args.expect_digest)
|
||
|
|
if sha(Path(__file__).read_bytes()) != capsule.manifest["files"]["supervisor.py"]:
|
||
|
|
raise Refused("run_the_reviewed_supervisor_source")
|
||
|
|
if os.geteuid() != 0:
|
||
|
|
raise Refused("node_root_context_required")
|
||
|
|
supervisor = Supervisor(capsule, Node())
|
||
|
|
if args.execute or args.recover:
|
||
|
|
if not sys.stdin.isatty():
|
||
|
|
raise Refused("founder_attended_terminal_required")
|
||
|
|
action = "RECOVER-DATE" if args.recover else "APPLY-DATE"
|
||
|
|
if input("Type " + action + " " + capsule.digest + " to continue: ").strip() != (
|
||
|
|
action + " " + capsule.digest):
|
||
|
|
raise Refused("not_confirmed")
|
||
|
|
print(json.dumps(supervisor.recover() if args.recover else supervisor.execute()))
|
||
|
|
else:
|
||
|
|
_, proof = supervisor.preflight()
|
||
|
|
print(json.dumps({"status": "read_only_preflight", "proposal_digest": capsule.digest,
|
||
|
|
"before": proof, "live_changes": False}))
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
try:
|
||
|
|
main()
|
||
|
|
except Exception as exc:
|
||
|
|
print(json.dumps({"status": "refused", "failure_type": type(exc).__name__,
|
||
|
|
"reason": str(exc) if isinstance(exc, Refused) else type(exc).__name__}))
|
||
|
|
raise SystemExit(1)
|