tegwick
1a35ee8113
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a1abf1-7e95-7141-87c9-ee78332bf790
566 lines
33 KiB
Python
566 lines
33 KiB
Python
#!/usr/bin/env python3
|
|
"""Founder node runner: restore the provider-retrieved pilot into fresh PVCs.
|
|
|
|
One isolated namespace/Pod, loopback PostgreSQL, no service or outgoing traffic.
|
|
Production resources are read only. All namespace/PV cleanup is identity guarded.
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pwd
|
|
import re
|
|
import signal
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import vergabe_pilot_export as export
|
|
from vergabe_pilot_recovery_store import Refused, digest, private_directory, require
|
|
|
|
CLUSTER_UID = "a553c742-0115-43d4-99a4-a5ca56fe0786"
|
|
PG_IMAGE = "postgres@sha256:ff23cdce56cac62ada6f66013e1a50864c0abbe79d132d40b6e05bd80f378a70"
|
|
LABEL = "railiance.apps/restore-run"
|
|
CLAIMS = {"database": "restore-database", "media": "restore-media", "app-state": "restore-issues"}
|
|
FILES = ("database.dump", "stores.tar.gz", "manifest.private.json", "receipt.json", "provider.json", "origin.json")
|
|
CIPHER = "4915288384e3ba8a87f1427b8e10106ee7241c04184adb97f439e57846bef76f"
|
|
PLAIN = "791a2cf0af25d9e54ea37203c584e6aa5cb1b8a0d4d467a281947160a7bb32e0"
|
|
|
|
STORE_LOAD = r'''
|
|
import os,sys,tarfile
|
|
from pathlib import Path
|
|
roots={'media':Path('/app/media'),'app-state':Path('/app/.issue-facade')}
|
|
seen=set();total=0
|
|
with tarfile.open(fileobj=sys.stdin.buffer,mode='r|gz') as archive:
|
|
for member in archive:
|
|
parts=member.name.rstrip('/').split('/')
|
|
assert parts[0] in roots and all(p not in ('','..','.') for p in parts)
|
|
assert '\\' not in member.name and (member.isdir() or member.isfile())
|
|
assert member.name not in seen
|
|
seen.add(member.name);total+=member.size
|
|
assert len(seen)<=10000 and total<=134217728
|
|
target=roots[parts[0]].joinpath(*parts[1:])
|
|
if member.isdir():
|
|
target.mkdir(mode=0o750,parents=True,exist_ok=True)
|
|
else:
|
|
target.parent.mkdir(mode=0o750,parents=True,exist_ok=True)
|
|
fd=os.open(target,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o640)
|
|
with os.fdopen(fd,'wb') as out,archive.extractfile(member) as source:
|
|
remaining=member.size
|
|
while remaining:
|
|
chunk=source.read(min(1048576,remaining))
|
|
assert chunk
|
|
out.write(chunk);remaining-=len(chunk)
|
|
'''
|
|
|
|
APP_CHECK = r'''
|
|
import hashlib,json,sys,sqlite3,stat
|
|
from pathlib import Path
|
|
from contextlib import closing
|
|
from urllib.parse import quote
|
|
import django;django.setup()
|
|
from django.conf import settings
|
|
from django.db import connection,transaction
|
|
from django.forms.models import model_to_dict
|
|
from django.test import Client
|
|
from vergabe_teilnahme.apps.accounts.models import Mitarbeiter,OIDCIdentity
|
|
from vergabe_teilnahme.apps.ausschreibungen.models import Ausschreibung
|
|
from vergabe_teilnahme.apps.aufgaben.models import Aufgabe,ExternalIssue
|
|
from vergabe_teilnahme.apps.aufgaben.forms import AufgabeForm
|
|
from vergabe_teilnahme.apps.dokumente.models import Dokument
|
|
expected=json.load(sys.stdin)
|
|
assert settings.NETKINGDOM_ENABLED is False and settings.ISSUE_FACADE_GITEA is None
|
|
with connection.cursor() as cursor:
|
|
cursor.execute('SELECT current_database(),current_user')
|
|
assert cursor.fetchone()==('vergabe_demo_company','vergabe_demo_company')
|
|
members=list(Mitarbeiter.objects.filter(is_active=True,is_staff=False,is_superuser=False).order_by('pk')[:2])
|
|
assert len(members)==2 and OIDCIdentity.objects.filter(user__in=members).count()==2
|
|
tender=Ausschreibung.objects.get(pk=1)
|
|
task=Aufgabe.objects.get(pk=1,ausschreibung_id=1)
|
|
document=Dokument.objects.get(pk=1,ausschreibung_id=1)
|
|
issue=ExternalIssue.objects.get(aufgabe_id=1,issue_facade_backend='local')
|
|
assert issue.issue_facade_id==expected['issue_id'] and document.datei.name==expected['document_path']
|
|
def data_checks():
|
|
files=[]
|
|
for path in sorted(Path(settings.MEDIA_ROOT).rglob('*')):
|
|
info=path.lstat()
|
|
assert stat.S_ISDIR(info.st_mode) or stat.S_ISREG(info.st_mode)
|
|
if stat.S_ISREG(info.st_mode):
|
|
assert info.st_uid==info.st_gid==999
|
|
files.append({'path':path.relative_to(settings.MEDIA_ROOT).as_posix(),'bytes':info.st_size,
|
|
'sha256':hashlib.sha256(path.read_bytes()).hexdigest()})
|
|
fp=lambda value:hashlib.sha256(json.dumps(value,sort_keys=True).encode()).hexdigest()
|
|
assert fp(files)==expected['stores']['media_logical_sha256']
|
|
state=Path(settings.ISSUE_FACADE_LOCAL_DB)
|
|
assert state.is_file() and not state.is_symlink() and state.stat().st_uid==state.stat().st_gid==999
|
|
with closing(sqlite3.connect('file:'+quote(str(state),safe='/')+'?mode=ro',uri=True)) as db:
|
|
assert db.execute('PRAGMA integrity_check').fetchall()==[('ok',)]
|
|
assert db.execute('SELECT count(*) FROM issues WHERE id=?',(expected['issue_id'],)).fetchone()[0]==1
|
|
tables=[]
|
|
for name, in db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"):
|
|
ident='"'+name.replace('"','""')+'"'
|
|
rows=[hashlib.sha256(json.dumps(list(row),sort_keys=True,separators=(',',':'),allow_nan=False,
|
|
default=lambda b:{'bytes':b.hex()}).encode()).hexdigest() for row in db.execute('SELECT * FROM '+ident)]
|
|
tables.append({'table':name,'rows':len(rows),'digest':hashlib.sha256(''.join(sorted(rows)).encode()).hexdigest()})
|
|
assert fp(tables)==expected['stores']['sqlite']['logical_sha256']
|
|
return {'all_media_hashes_match':True,'sqlite_integrity_and_rows_match':True,'local_issue_link_verified':True,
|
|
'restored_files_uid_gid_verified':True,'media_files':len(files),'sqlite_tables':len(tables),
|
|
'sqlite_rows':sum(t['rows'] for t in tables)}
|
|
result=data_checks()
|
|
paths=['/ausschreibungen/1/','/ausschreibungen/1/aufgaben/1/','/ausschreibungen/1/dokumente/1/']
|
|
media='/media/'+document.datei.name
|
|
def request(client,path):return client.get(path,secure=True,HTTP_HOST='restore.invalid')
|
|
with transaction.atomic():
|
|
anonymous=Client()
|
|
assert all(request(anonymous,p).status_code==302 for p in [*paths,media])
|
|
csrf=Client(enforce_csrf_checks=True)
|
|
csrf.force_login(members[0],backend='django.contrib.auth.backends.ModelBackend')
|
|
assert csrf.post('/ausschreibungen/1/aufgaben/1/bearbeiten/',{'titel':'blocked'},secure=True,HTTP_HOST='restore.invalid').status_code==403
|
|
first,second=Client(enforce_csrf_checks=True),Client(enforce_csrf_checks=True)
|
|
first.force_login(members[0],backend='django.contrib.auth.backends.ModelBackend')
|
|
second.force_login(members[1],backend='django.contrib.auth.backends.ModelBackend')
|
|
for client in (first,second):
|
|
assert all(request(client,p).status_code==200 for p in paths)
|
|
response=request(client,media)
|
|
assert response.status_code==200
|
|
checksum=hashlib.sha256(b''.join(response.streaming_content)).hexdigest()
|
|
assert checksum==expected['document_sha256']
|
|
data=model_to_dict(task,fields=AufgabeForm._meta.fields)
|
|
data={key:('' if value is None else value.isoformat() if hasattr(value,'isoformat') else value) for key,value in data.items()}
|
|
marker='ISOLATED-RESTORE-WORKFLOW-CHECK'
|
|
data['beschreibung']=marker
|
|
edit='/ausschreibungen/1/aufgaben/1/bearbeiten/'
|
|
assert request(first,edit).status_code==200
|
|
token=first.cookies[settings.CSRF_COOKIE_NAME].value
|
|
response=first.post(edit,data,secure=True,HTTP_HOST='restore.invalid',HTTP_ORIGIN='https://restore.invalid',HTTP_X_CSRFTOKEN=token)
|
|
assert response.status_code==302
|
|
assert marker.encode() in request(second,paths[1]).content
|
|
Mitarbeiter.objects.filter(pk=members[0].pk).update(is_active=False)
|
|
assert request(first,paths[0]).status_code==302 and request(first,media).status_code==302
|
|
transaction.set_rollback(True)
|
|
assert data_checks()==result
|
|
result.update({'two_restored_members_verified':True,'restored_identity_mappings':2,
|
|
'member_document_checksums_match':True,'cross_member_task_edit_verified':True,
|
|
'anonymous_pages_and_binary_denied':True,'inactive_existing_session_denied':True,
|
|
'csrf_write_denied':True,'workflow_test_transaction_rolled_back':True,
|
|
'captured_tender_status':tender.status,'captured_tender_archived':tender.archiviert,
|
|
'captured_status_in_dashboard_range':1<=tender.status<=9,
|
|
'native_sso_login_verified':False,'outgoing_integrations_enabled':False})
|
|
print(json.dumps(result))
|
|
'''
|
|
|
|
APP_COMMIT = r'''
|
|
import json,sys
|
|
import django;django.setup()
|
|
from vergabe_teilnahme.apps.aufgaben.models import Aufgabe
|
|
marker=json.load(sys.stdin)['marker']
|
|
assert marker.startswith('RESTORE-ROLLBACK-') and len(marker)<100
|
|
assert Aufgabe.objects.get(pk=1).beschreibung!=marker
|
|
assert Aufgabe.objects.filter(pk=1,ausschreibung_id=1).update(beschreibung=marker)==1
|
|
print(json.dumps({'clone_marker_committed':True}))
|
|
'''
|
|
|
|
APP_BAD_CONFIG = r'''
|
|
import json
|
|
import django;django.setup()
|
|
from django.db import connection,OperationalError
|
|
try:
|
|
connection.ensure_connection()
|
|
except OperationalError:
|
|
print(json.dumps({'expected_bad_configuration_refused':True}))
|
|
else:
|
|
raise AssertionError('bad clone database configuration unexpectedly worked')
|
|
'''
|
|
|
|
|
|
def command(args, payload=None, timeout=60):
|
|
try:
|
|
result = subprocess.run(args, input=payload, capture_output=True, timeout=timeout, check=False)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
raise Refused("restore_command_unavailable_or_timeout") from None
|
|
require(result.returncode == 0, "restore_command_failed")
|
|
return result.stdout
|
|
|
|
|
|
def k(*args, payload=None, timeout=60):
|
|
request_timeout = "0" if args[0] == "wait" else "30s"
|
|
return command(["kubectl", "--request-timeout=" + request_timeout, *args], payload, timeout)
|
|
|
|
|
|
def get(kind, name, namespace=None):
|
|
return json.loads(k("get", kind, name, *(["-n", namespace] if namespace else []), "-o", "json"))
|
|
|
|
|
|
def apply(body):
|
|
return json.loads(k("create", "-f", "-", "-o", "json", payload=json.dumps(body).encode()))
|
|
|
|
|
|
def fingerprint(value):
|
|
return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()
|
|
|
|
|
|
def production():
|
|
require(get("namespace", "kube-system")["metadata"]["uid"] == CLUSTER_UID, "wrong_cluster")
|
|
nodes = json.loads(k("get", "nodes", "-o", "json"))["items"]
|
|
require(len(nodes) == 1 and any(a["address"] == "92.205.62.239" for a in nodes[0]["status"]["addresses"]),
|
|
"unexpected_node")
|
|
conditions = {c["type"]: c["status"] for c in nodes[0]["status"]["conditions"]}
|
|
require(conditions.get("Ready") == "True" and all(conditions.get(c) == "False"
|
|
for c in ("MemoryPressure", "DiskPressure", "PIDPressure")), "node_pressure_or_not_ready")
|
|
deployment = get("deployment", "vergabe-teilnahme", "vergabe-demo-company")
|
|
app = next(c for c in deployment["spec"]["template"]["spec"]["containers"] if c["name"] == "app")
|
|
require(deployment["spec"]["replicas"] == deployment["status"].get("readyReplicas") == 1
|
|
and app["image"] == export.IMAGE, "production_release_not_ready")
|
|
require(os.statvfs("/var/lib").f_bavail * os.statvfs("/var/lib").f_frsize >= 1024**3,
|
|
"one_gib_free_required")
|
|
return {"deployment_uid": deployment["metadata"]["uid"], "spec_sha256": fingerprint(deployment["spec"])}
|
|
|
|
|
|
def resources(namespace, nonce):
|
|
metadata = {"namespace": namespace, "labels": {LABEL: nonce}}
|
|
policy = {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy",
|
|
"metadata": {**metadata, "name": "default-deny"},
|
|
"spec": {"podSelector": {}, "policyTypes": ["Ingress", "Egress"]}}
|
|
claims = [{"apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": {**metadata, "name": name},
|
|
"spec": {"accessModes": ["ReadWriteOnce"], "storageClassName": "local-path",
|
|
"resources": {"requests": {"storage": "1Gi"}}}} for name in CLAIMS.values()]
|
|
security = {"allowPrivilegeEscalation": False, "capabilities": {"drop": ["ALL"]}}
|
|
app_env = {"DATABASE_URL": "postgres://vergabe_demo_company@127.0.0.1/vergabe_demo_company",
|
|
"SECRET_KEY": "isolated-restore-only-not-a-production-key", "ALLOWED_HOSTS": "restore.invalid",
|
|
"APP_BASE_PATH": "/demo-company", "NETKINGDOM_ENABLED": "false",
|
|
"PYTHONDONTWRITEBYTECODE": "1"}
|
|
pg_env = {"POSTGRES_HOST_AUTH_METHOD": "trust", "POSTGRES_DB": "vergabe_demo_company",
|
|
"PGDATA": "/var/lib/postgresql/data/pgdata"}
|
|
pod = {"apiVersion": "v1", "kind": "Pod", "metadata": {**metadata, "name": "restore"}, "spec": {
|
|
"restartPolicy": "Never", "activeDeadlineSeconds": 900, "automountServiceAccountToken": False,
|
|
"enableServiceLinks": False,
|
|
"securityContext": {"runAsNonRoot": True, "runAsUser": 999, "runAsGroup": 999, "fsGroup": 999,
|
|
"seccompProfile": {"type": "RuntimeDefault"}},
|
|
"containers": [
|
|
{"name": "database", "image": PG_IMAGE, "imagePullPolicy": "IfNotPresent", "securityContext": security,
|
|
"args": ["postgres", "-c", "listen_addresses=127.0.0.1"],
|
|
"env": [{"name": n, "value": v} for n, v in pg_env.items()],
|
|
"resources": {"requests": {"cpu": "10m", "memory": "128Mi"},
|
|
"limits": {"cpu": "250m", "memory": "256Mi"}},
|
|
"readinessProbe": {"exec": {"command": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"]},
|
|
"periodSeconds": 2},
|
|
"volumeMounts": [{"name": "database", "mountPath": "/var/lib/postgresql/data"}]},
|
|
{"name": "app", "image": export.IMAGE, "imagePullPolicy": "Never", "securityContext": security,
|
|
"command": ["python", "-c", "import time;time.sleep(900)"],
|
|
"env": [{"name": n, "value": v} for n, v in app_env.items()],
|
|
"resources": {"requests": {"cpu": "10m", "memory": "64Mi"},
|
|
"limits": {"cpu": "500m", "memory": "512Mi"}},
|
|
"volumeMounts": [{"name": "media", "mountPath": "/app/media"},
|
|
{"name": "app-state", "mountPath": "/app/.issue-facade"}]}],
|
|
"volumes": [{"name": n, "persistentVolumeClaim": {"claimName": claim}} for n, claim in CLAIMS.items()]}}
|
|
return policy, claims, pod
|
|
|
|
|
|
def validate_pod(actual, expected):
|
|
require(actual["metadata"]["namespace"] == expected["metadata"]["namespace"]
|
|
and actual["metadata"].get("labels") == expected["metadata"]["labels"], "restore_pod_identity_changed")
|
|
spec = actual["spec"]
|
|
require(not any(spec.get(k) for k in ("hostNetwork", "hostPID", "hostIPC", "initContainers", "ephemeralContainers"))
|
|
and spec.get("automountServiceAccountToken") is False and spec.get("enableServiceLinks") is False
|
|
and len(spec["containers"]) == 2, "restore_pod_scope_changed")
|
|
require(spec["volumes"] == expected["spec"]["volumes"], "restore_mounts_changed")
|
|
require(spec.get("activeDeadlineSeconds") == 900 and spec.get("restartPolicy") == "Never",
|
|
"restore_lifetime_changed")
|
|
for key, value in expected["spec"]["securityContext"].items():
|
|
require(spec["securityContext"].get(key) == value, "restore_security_context_changed")
|
|
for current, wanted in zip(spec["containers"], expected["spec"]["containers"], strict=True):
|
|
require(not current.get("envFrom"), "restore_credential_injection")
|
|
for key in ("name", "image", "imagePullPolicy", "env", "resources", "volumeMounts", "command", "args"):
|
|
require(current.get(key) == wanted.get(key), "restore_container_scope_changed")
|
|
require(not current.get("securityContext", {}).get("privileged")
|
|
and current["securityContext"].get("allowPrivilegeEscalation") is False
|
|
and current["securityContext"].get("capabilities") == {"drop": ["ALL"]},
|
|
"restore_security_context_changed")
|
|
|
|
|
|
def pg(namespace, sql):
|
|
return k("exec", "-i", "-n", namespace, "restore", "-c", "database", "--", "psql",
|
|
"-XqAt", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", "vergabe_demo_company", payload=sql.encode())
|
|
|
|
|
|
def replace_clone_pod(journal, wanted):
|
|
namespace = journal["namespace"]
|
|
ns = get("namespace", namespace)
|
|
require(ns["metadata"]["uid"] == journal["namespace_uid"]
|
|
and ns["metadata"]["labels"].get(LABEL) == journal["nonce"], "restore_namespace_ownership_changed")
|
|
current = get("pod", "restore", namespace)
|
|
require(current["metadata"]["uid"] == journal["pod_uid"]
|
|
and current["metadata"]["labels"].get(LABEL) == journal["nonce"], "restore_pod_identity_changed")
|
|
options = {"apiVersion": "v1", "kind": "DeleteOptions", "preconditions": {"uid": journal["pod_uid"]}}
|
|
k("delete", "--raw", "/api/v1/namespaces/" + namespace + "/pods/restore", "-f", "-",
|
|
payload=json.dumps(options).encode())
|
|
k("wait", "--for=delete", "pod/restore", "-n", namespace, "--timeout=120s", timeout=140)
|
|
admitted = apply(wanted)
|
|
validate_pod(admitted, wanted)
|
|
journal["pod_uid"] = admitted["metadata"]["uid"]
|
|
k("wait", "--for=condition=Ready", "pod/restore", "-n", namespace, "--timeout=240s", timeout=260)
|
|
|
|
|
|
def database_manifest(psql):
|
|
require(psql("SELECT count(*) FROM pg_tables WHERE schemaname NOT IN "
|
|
"('public','information_schema') AND schemaname !~ '^pg_';").strip() == b"0", "unsupported_schema")
|
|
tables = psql("SELECT tablename FROM pg_tables WHERE schemaname='public' ORDER BY tablename;").decode().splitlines()
|
|
require(tables and all(re.fullmatch(r"[a-z_][a-z0-9_]*", t) for t in tables), "unsupported_schema")
|
|
sql = ("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; SET LOCAL statement_timeout='30s';"
|
|
"SET LOCAL timezone='UTC'; SET LOCAL datestyle='ISO,YMD'; SET LOCAL extra_float_digits=3;\n")
|
|
row_hash = "encode(sha256(convert_to(row_to_json(t)::text,'UTF8')),'hex')"
|
|
for table in tables:
|
|
sql += ("SELECT json_build_object('table','" + table + "','rows',count(*),'digest',"
|
|
"encode(sha256(convert_to(coalesce(string_agg(" + row_hash + ",'' ORDER BY "
|
|
+ row_hash + " COLLATE \"C\"),''),'UTF8')),'hex')) FROM public.\"" + table + "\" t;\n")
|
|
rows = [json.loads(line) for line in psql(sql + "COMMIT;\n").decode().splitlines() if line]
|
|
require(len(rows) == len(tables) and sum(r["rows"] for r in rows) <= 100000, "database_manifest_bound")
|
|
return rows
|
|
|
|
|
|
def isolation_policy(namespace):
|
|
policies = json.loads(k("get", "networkpolicy", "-n", namespace, "-o", "json"))["items"]
|
|
require(len(policies) == 1 and policies[0]["metadata"]["name"] == "default-deny", "restore_network_policy_changed")
|
|
spec = policies[0]["spec"]
|
|
require(spec["podSelector"] == {} and set(spec["policyTypes"]) == {"Ingress", "Egress"}
|
|
and not spec.get("ingress") and not spec.get("egress"), "restore_network_policy_changed")
|
|
|
|
|
|
def runtime_status(namespace):
|
|
try:
|
|
status = get("pod", "restore", namespace)["status"]
|
|
allowed = {"ContainerCreating", "PodInitializing", "ImagePullBackOff", "ErrImagePull", "ErrImageNeverPull",
|
|
"CreateContainerConfigError", "CreateContainerError", "CrashLoopBackOff", "RunContainerError"}
|
|
result = {"pod_phase": status.get("phase") if status.get("phase") in
|
|
{"Pending", "Running", "Succeeded", "Failed", "Unknown"} else "other"}
|
|
for container in status.get("containerStatuses", []):
|
|
if container["name"] in {"app", "database"}:
|
|
state = container.get("state", {})
|
|
reason = state.get("waiting", {}).get("reason")
|
|
result[container["name"] + "_waiting"] = reason if reason in allowed else "other" if reason else None
|
|
result[container["name"] + "_oom_killed"] = state.get("terminated", {}).get("reason") == "OOMKilled"
|
|
return result
|
|
except Exception: # noqa: BLE001 — best-effort sanitized diagnostics
|
|
return {"status_read_unavailable": True}
|
|
|
|
|
|
def input_point(stage, owner):
|
|
private = stage.lstat()
|
|
require(private.st_uid == owner and private.st_mode & 0o077 == 0 and stage.is_dir() and not stage.is_symlink(),
|
|
"private_founder_stage_required")
|
|
require({p.name for p in stage.iterdir()} == set(FILES), "staged_file_set_changed")
|
|
for name in FILES:
|
|
info = export.regular(stage / name, owner)
|
|
require(0 < info.st_size <= 1024**2, "staged_file_bound")
|
|
origin = json.loads((stage / "origin.json").read_text())
|
|
require(origin["run_id"] == export.RUN_ID and set(origin["files"]) == set(FILES[:4]), "wrong_staged_origin")
|
|
for name in FILES[:4]:
|
|
require(origin["files"][name] == {"bytes": (stage / name).stat().st_size,
|
|
"sha256": digest(stage / name)}, "staged_file_mismatch")
|
|
for name, (size, checksum) in export.EXPECTED.items():
|
|
require((stage / name).stat().st_size == size and digest(stage / name) == checksum, "capture_hash_mismatch")
|
|
provider = json.loads((stage / "provider.json").read_text())
|
|
require(provider["run_id"] == export.RUN_ID and provider["status"] == "published_read_back_and_decrypted"
|
|
and provider["warden_exit_code"] == 0 and provider["ciphertext_sha256"] == CIPHER
|
|
and provider["plaintext_packet_sha256"] == PLAIN
|
|
and all(provider.get(k) is True for k in ("provider_publication_verified", "independent_readback_verified",
|
|
"decryption_verified", "session_cleanup_confirmed")), "provider_provenance_required")
|
|
manifest = json.loads((stage / "manifest.private.json").read_text())
|
|
require(manifest["image"] == export.IMAGE and manifest["database"] == "vergabe_demo_company", "wrong_capture")
|
|
return manifest
|
|
|
|
|
|
def save(directory, journal):
|
|
path = directory / "journal.json"
|
|
temporary = directory / ".journal.new"
|
|
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(fd, "w") as output:
|
|
json.dump(journal, output, indent=2)
|
|
output.flush()
|
|
os.fsync(output.fileno())
|
|
os.replace(temporary, path)
|
|
|
|
|
|
def cleanup(journal):
|
|
require(get("namespace", "kube-system")["metadata"]["uid"] == CLUSTER_UID, "wrong_cluster")
|
|
namespace = journal["namespace"]
|
|
objects = json.loads(k("get", "namespace", "-o", "json"))["items"]
|
|
found = [n for n in objects if n["metadata"]["name"] == namespace]
|
|
if found:
|
|
obj = found[0]
|
|
require(obj["metadata"].get("labels", {}).get(LABEL) == journal["nonce"]
|
|
and obj["metadata"]["uid"] == journal.get("namespace_uid"), "restore_namespace_ownership_changed")
|
|
body = {"apiVersion": "v1", "kind": "DeleteOptions", "preconditions": {"uid": obj["metadata"]["uid"]}}
|
|
k("delete", "--raw", "/api/v1/namespaces/" + namespace, "-f", "-", payload=json.dumps(body).encode())
|
|
k("wait", "--for=delete", "namespace/" + namespace, "--timeout=180s", timeout=200)
|
|
for name, uid in journal.get("volumes", {}).items():
|
|
current = k("get", "pv", name, "--ignore-not-found", "-o", "json").strip()
|
|
if current:
|
|
require(json.loads(current)["metadata"]["uid"] == uid, "restore_pv_identity_changed")
|
|
k("wait", "--for=delete", "pv/" + name, "--timeout=120s", timeout=140)
|
|
return True
|
|
|
|
|
|
def execute(nonce, stage, directory, baseline):
|
|
manifest = input_point(stage, int(os.environ["SUDO_UID"]))
|
|
namespace = "vergabe-restore-" + nonce
|
|
journal = {"nonce": nonce, "namespace": namespace, "volumes": {}, "baseline": baseline, "status": "prepared"}
|
|
receipt = {"schema": "railiance.vergabe-pilot-isolated-restore.v1", "run_id": export.RUN_ID,
|
|
"nonce": nonce, "namespace": namespace, "status": "failed", "restore_verified": False,
|
|
"application_changed": False, "production_application_changed": False,
|
|
"provider_retrieved_source": True, "ciphertext_sha256": CIPHER,
|
|
"plaintext_packet_sha256": PLAIN, "application_image": export.IMAGE,
|
|
"postgres_image": PG_IMAGE, "native_sso_login_verified": False, "helm_rollback_verified": False}
|
|
started = time.monotonic()
|
|
save(directory, journal)
|
|
print(json.dumps({"status": "isolated_restore_started", "nonce": nonce, "namespace": namespace}), flush=True)
|
|
try:
|
|
receipt["stage"] = "namespace_creation"
|
|
obj = apply({"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": namespace,
|
|
"labels": {LABEL: nonce, "railiance.io/workload-class": "application",
|
|
"pod-security.kubernetes.io/enforce": "restricted"}}})
|
|
journal["namespace_uid"] = obj["metadata"]["uid"]
|
|
save(directory, journal)
|
|
policy, claims, pod = resources(namespace, nonce)
|
|
receipt["stage"] = "policy_and_claims"
|
|
apply(policy)
|
|
for claim in claims:
|
|
apply(claim)
|
|
receipt["stage"] = "pod_admission"
|
|
admitted = apply(pod)
|
|
validate_pod(admitted, pod)
|
|
journal["pod_uid"] = admitted["metadata"]["uid"]
|
|
save(directory, journal)
|
|
receipt["stage"] = "pod_readiness"
|
|
k("wait", "--for=condition=Ready", "pod/restore", "-n", namespace, "--timeout=240s", timeout=260)
|
|
for name in CLAIMS.values():
|
|
claim = get("pvc", name, namespace)
|
|
volume = get("pv", claim["spec"]["volumeName"])
|
|
require(claim["status"]["phase"] == "Bound" and volume["spec"]["persistentVolumeReclaimPolicy"] == "Delete"
|
|
and volume["spec"]["claimRef"]["uid"] == claim["metadata"]["uid"], "restore_claim_binding_changed")
|
|
journal["volumes"][volume["metadata"]["name"]] = volume["metadata"]["uid"]
|
|
save(directory, journal)
|
|
isolation_policy(namespace)
|
|
receipt["stage"] = "database_restore"
|
|
pg(namespace, "CREATE ROLE vergabe_demo_company LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE; "
|
|
"ALTER DATABASE vergabe_demo_company OWNER TO vergabe_demo_company;")
|
|
k("exec", "-i", "-n", namespace, "restore", "-c", "database", "--", "pg_restore", "-U", "postgres",
|
|
"-d", "vergabe_demo_company", "--role", "vergabe_demo_company", "--no-owner", "--no-acl", "--exit-on-error",
|
|
payload=(stage / "database.dump").read_bytes(), timeout=120)
|
|
receipt["stage"] = "database_fingerprints"
|
|
require(database_manifest(lambda sql: pg(namespace, sql)) == manifest["postgres"], "restored_database_mismatch")
|
|
receipt.update(database_restored=True, all_postgres_table_digests_match=True,
|
|
database_tables=len(manifest["postgres"]), database_rows=sum(t["rows"] for t in manifest["postgres"]),
|
|
database_logical_sha256=fingerprint(manifest["postgres"]))
|
|
receipt["stage"] = "file_store_restore"
|
|
k("exec", "-i", "-n", namespace, "restore", "-c", "app", "--", "python", "-c", STORE_LOAD,
|
|
payload=(stage / "stores.tar.gz").read_bytes())
|
|
receipt["stage"] = "migration_check"
|
|
k("exec", "-n", namespace, "restore", "-c", "app", "--", "python", "manage.py", "migrate", "--check", "--noinput")
|
|
expected = {**manifest["fixture"], "document_sha256": manifest["stores"]["fixture_document_sha256"],
|
|
"stores": manifest["stores"]}
|
|
receipt["stage"] = "application_checks"
|
|
checks = json.loads(k("exec", "-i", "-n", namespace, "restore", "-c", "app", "--", "python", "-c", APP_CHECK,
|
|
payload=json.dumps(expected).encode(), timeout=120))
|
|
receipt.update(checks)
|
|
receipt["stage"] = "workflow_rollback_verification"
|
|
require(database_manifest(lambda sql: pg(namespace, sql)) == manifest["postgres"], "workflow_writes_not_rolled_back")
|
|
isolation_policy(namespace)
|
|
receipt["restore_elapsed_seconds"] = round(time.monotonic() - started, 2)
|
|
receipt["stage"] = "configuration_rollback_marker"
|
|
committed = json.loads(k("exec", "-i", "-n", namespace, "restore", "-c", "app", "--", "python", "-c", APP_COMMIT,
|
|
payload=json.dumps({"marker": "RESTORE-ROLLBACK-" + nonce}).encode()))
|
|
require(committed.get("clone_marker_committed") is True, "clone_marker_not_committed")
|
|
current_rows = database_manifest(lambda sql: pg(namespace, sql))
|
|
require(current_rows != manifest["postgres"], "clone_marker_not_observed")
|
|
receipt["stage"] = "isolated_bad_configuration"
|
|
broken = json.loads(json.dumps(pod))
|
|
for item in broken["spec"]["containers"][1]["env"]:
|
|
if item["name"] == "DATABASE_URL":
|
|
item["value"] = "postgres://vergabe_demo_company@127.0.0.1/restore_unavailable"
|
|
replace_clone_pod(journal, broken)
|
|
save(directory, journal)
|
|
bad = json.loads(k("exec", "-n", namespace, "restore", "-c", "app", "--", "python", "-c", APP_BAD_CONFIG))
|
|
require(bad.get("expected_bad_configuration_refused") is True, "bad_configuration_not_demonstrated")
|
|
require(database_manifest(lambda sql: pg(namespace, sql)) == current_rows, "clone_rows_changed_before_inverse")
|
|
rollback_started = time.monotonic()
|
|
receipt["stage"] = "configuration_inverse"
|
|
replace_clone_pod(journal, pod)
|
|
save(directory, journal)
|
|
k("exec", "-n", namespace, "restore", "-c", "app", "--", "python", "manage.py", "migrate", "--check", "--noinput")
|
|
repeated = json.loads(k("exec", "-i", "-n", namespace, "restore", "-c", "app", "--", "python", "-c", APP_CHECK,
|
|
payload=json.dumps(expected).encode(), timeout=120))
|
|
require(repeated == checks and database_manifest(lambda sql: pg(namespace, sql)) == current_rows,
|
|
"configuration_inverse_lost_committed_writes")
|
|
isolation_policy(namespace)
|
|
receipt.update(configuration_rollback_verified=True, committed_clone_write_preserved=True,
|
|
pod_replacements=2, rollback_same_image_and_schema=True,
|
|
older_image_or_schema_downgrade_verified=False,
|
|
configuration_inverse_elapsed_seconds=round(time.monotonic() - rollback_started, 2))
|
|
receipt.update(restore_verified=True, separate_fresh_claims=True, network_policy_default_deny=True,
|
|
application_image_cached_no_pull=True, status="restored_and_verified")
|
|
journal["status"] = "verified"
|
|
except BaseException as error: # noqa: BLE001 — record failures and clean up on interruption
|
|
receipt["failure"] = str(error) if isinstance(error, Refused) else "isolated_restore_failed"
|
|
receipt["failure_stage"] = receipt.get("stage")
|
|
if journal.get("namespace_uid"):
|
|
receipt["clone_runtime"] = runtime_status(namespace)
|
|
finally:
|
|
try:
|
|
receipt["namespace_and_volumes_removed"] = cleanup(journal)
|
|
except Exception: # noqa: BLE001 — preserve independent cleanup/readback results
|
|
receipt["namespace_and_volumes_removed"] = False
|
|
try:
|
|
receipt["production_spec_and_readiness_unchanged"] = production() == baseline
|
|
except Exception: # noqa: BLE001 — preserve independent cleanup/readback results
|
|
receipt["production_spec_and_readiness_unchanged"] = False
|
|
receipt["elapsed_seconds"] = round(time.monotonic() - started, 2)
|
|
if not receipt.get("namespace_and_volumes_removed") or not receipt.get("production_spec_and_readiness_unchanged"):
|
|
receipt["status"] = "incomplete"
|
|
if receipt["status"] == "restored_and_verified":
|
|
receipt["stage"] = "complete"
|
|
journal["receipt"] = receipt
|
|
save(directory, journal)
|
|
print(json.dumps(receipt, indent=2))
|
|
return 0 if receipt["status"] == "restored_and_verified" else 1
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--nonce", required=True)
|
|
parser.add_argument("--cleanup", action="store_true")
|
|
args = parser.parse_args()
|
|
require(re.fullmatch(r"[0-9a-f]{12}", args.nonce), "invalid_restore_nonce")
|
|
require(os.geteuid() == 0 and os.environ.get("SUDO_USER") == "tegwick", "founder_node_sudo_required")
|
|
owner = pwd.getpwnam("tegwick")
|
|
require(os.environ.get("SUDO_UID") == str(owner.pw_uid), "founder_account_required")
|
|
stage = Path(owner.pw_dir) / ("vergabe-pilot-restore-" + args.nonce)
|
|
base = export.BASE / export.RUN_ID
|
|
private_directory(base)
|
|
directory = base / ("native-restore-" + args.nonce)
|
|
if args.cleanup:
|
|
private_directory(directory)
|
|
journal = json.loads((directory / "journal.json").read_text())
|
|
require(journal["nonce"] == args.nonce and journal["namespace"] == "vergabe-restore-" + args.nonce,
|
|
"restore_journal_mismatch")
|
|
print(json.dumps({"status": "cleanup_complete", "namespace_and_volumes_removed": cleanup(journal),
|
|
"restore_acceptance": False}))
|
|
return 0
|
|
baseline = production()
|
|
directory.mkdir(mode=0o700)
|
|
return execute(args.nonce, stage, directory, baseline)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
os.umask(0o077)
|
|
def interrupted(signum, frame):
|
|
raise Refused("restore_interrupted")
|
|
signal.signal(signal.SIGHUP, interrupted)
|
|
signal.signal(signal.SIGTERM, interrupted)
|
|
try:
|
|
raise SystemExit(main())
|
|
except Exception as error: # noqa: BLE001 — suppress private command/parser details
|
|
print(json.dumps({"status": "restore_failed", "failure": str(error) if isinstance(error, Refused)
|
|
else "restore_runner_failed", "details_suppressed": True}))
|
|
raise SystemExit(1)
|