Implement selected mail return operations and prepare SMTP admission
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a198ed-70cc-7362-a569-d5dbef39b86a
This commit is contained in:
parent
97cfc2bb44
commit
1997923bee
20 changed files with 3867 additions and 2 deletions
465
scripts/mail_return_native.py
Normal file
465
scripts/mail_return_native.py
Normal file
|
|
@ -0,0 +1,465 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Fixed native transport for the selected-return operator; observe by default.
|
||||
|
||||
No mailbox read, SMTP send, credential fetch or Kubernetes resource mutation.
|
||||
Explicit apply/reconcile touches only the reviewed private audit and suppression.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import mail_return_operation as operation
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
IMAGE = (
|
||||
"forgejo.coulomb.social/coulomb/email-connect@sha256:"
|
||||
"5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d"
|
||||
)
|
||||
CLUSTER = "a553c742-0115-43d4-99a4-a5ca56fe0786"
|
||||
NAMESPACE = "5678c5d7-e4f5-4a62-8f24-1c6e1d906319"
|
||||
DEPLOYMENT = "2e98e52a-a2d1-4c52-9ac8-6efbc0a18e72"
|
||||
TRANSACTIONAL = "36fdfd8cb58f10148cd491a7e34d7aea798e4289fce760af70a48776bede204d"
|
||||
CALLERS = "06fe961b9030dba2c0b26b92f8097e8fbd24e205d5fa79642aa9af60382886dc"
|
||||
SCHEMA = "coulomb.native-mail-return.v1"
|
||||
FILES = ("mail_return_native.py", "mail_return_operation.py")
|
||||
require = operation.require
|
||||
|
||||
BOOTSTRAP = r"""
|
||||
import hashlib,json,os,sys,tempfile
|
||||
from pathlib import Path
|
||||
sys.dont_write_bytecode=True
|
||||
raw=sys.stdin.buffer.read(4*1024**2+1)
|
||||
assert __debug__ and len(raw)<=4*1024**2
|
||||
r=json.loads(raw)
|
||||
assert set(r['sources'])=={'mail_return_native.py','mail_return_operation.py'}
|
||||
assert sum(len(s.encode()) for s in r['sources'].values())<=128*1024
|
||||
hashes={n:hashlib.sha256(v.encode()).hexdigest() for n,v in r['sources'].items()}
|
||||
assert hashes==r['source_sha256']
|
||||
if r.get('reviewed_source_sha256') is not None:assert hashes==r['reviewed_source_sha256']
|
||||
with tempfile.TemporaryDirectory(prefix='mail-return-source-') as directory:
|
||||
for name,source in r['sources'].items():
|
||||
fd=os.open(Path(directory)/name,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
|
||||
with os.fdopen(fd,'w') as stream:stream.write(source)
|
||||
sys.path.insert(0,directory)
|
||||
import mail_return_native as native
|
||||
try:print(json.dumps(native.run_node(r)))
|
||||
except Exception as error:
|
||||
print(json.dumps({'status':'native_return_requires_inspection','failure_type':type(error).__name__,
|
||||
'retry_automatically':False}))
|
||||
sys.exit(1)
|
||||
"""
|
||||
|
||||
POD_PROGRAM = r"""
|
||||
import base64,hashlib,json,os,sys,types
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
sys.dont_write_bytecode=True
|
||||
from email_connect import transactional,callers
|
||||
r=json.load(sys.stdin)
|
||||
assert __debug__ and os.getuid()==10001
|
||||
actual=hashlib.sha256(Path(transactional.__file__).read_bytes()).hexdigest()
|
||||
assert actual==r['transactional_sha256']
|
||||
assert hashlib.sha256(Path(callers.__file__).read_bytes()).hexdigest()==r['callers_sha256']
|
||||
assert os.environ.get('EMAIL_CONNECT_CALLERS_FILE')=='/etc/email-connect-callers/callers.json'
|
||||
database=os.environ.get('EMAIL_CONNECT_DATABASE_PATH','/data/email-connect.db')
|
||||
assert database=='/data/email-connect.db'
|
||||
p=Path('/etc/email-connect-callers/callers.json')
|
||||
assert p.stat().st_size<=64*1024
|
||||
policy=json.loads(p.read_text())
|
||||
m=types.ModuleType('selected_mail_return')
|
||||
exec(compile(r['core_source'],'<reviewed-mail-return>','exec'),m.__dict__)
|
||||
context={'caller_policy_sha256':m.digest(policy)}
|
||||
if r.get('policy_sha256') is not None:assert context['caller_policy_sha256']==r['policy_sha256']
|
||||
ledger=Path('/data/email-connect.db')
|
||||
audit=Path('/data/mail-returns')
|
||||
action=r['action']
|
||||
if action=='observe':
|
||||
with m.closing(m.connect(ledger)) as db:
|
||||
db.execute('BEGIN')
|
||||
tables={row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||
assert tables=={'deliveries','verifications','suppressions','send_attempts',
|
||||
'verification_attempts','caller_buckets'}
|
||||
size=db.execute('PRAGMA page_count').fetchone()[0]*db.execute('PRAGMA page_size').fetchone()[0]
|
||||
assert 0<size<=128*1024**2
|
||||
state={}
|
||||
for name in sorted(tables):
|
||||
state[name]=sorted(m.canonical(row).hex() for row in db.execute('SELECT * FROM "'+name+'"'))
|
||||
result={'status':'native_return_observed','table_counts':{n:len(v) for n,v in state.items()},
|
||||
'ledger_sha256':m.digest(state),'audit_root_present':audit.exists(),'ledger_modified':False}
|
||||
elif action=='prepare':
|
||||
raw=base64.b64decode(r['raw_base64'],validate=True)
|
||||
result=m.prepare(ledger,raw,r['recipient'],policy,r['review'],invitation_payload=r.get('invitation_payload'))
|
||||
elif action=='release':
|
||||
result=m.prepare_release(ledger,audit,r['source_operation'],r['review'])
|
||||
elif action=='apply':
|
||||
if not audit.exists():
|
||||
audit.mkdir(mode=0o700)
|
||||
fd=os.open(audit.parent,os.O_RDONLY|os.O_DIRECTORY)
|
||||
try:os.fsync(fd)
|
||||
finally:os.close(fd)
|
||||
raw=base64.b64decode(r['raw_base64'],validate=True) if r.get('raw_base64') is not None else None
|
||||
result=m.apply(ledger,r['plan'],raw,policy,m.digest(r['plan']),audit)
|
||||
elif action=='inspect':
|
||||
result=m.inspect(ledger,audit/r['operation_id'])
|
||||
elif action=='reconcile':
|
||||
result=m.reconcile(ledger,audit/r['operation_id'],r['operation_id'])
|
||||
elif action=='prepare-expiry':
|
||||
result={'action':'expire-raw','as_of':r['as_of'],'preview':m.expire_raw(audit,now=datetime.fromisoformat(r['as_of']),execute=False)}
|
||||
elif action=='apply-expiry':
|
||||
plan=r['plan']; now=datetime.fromisoformat(plan['as_of'])
|
||||
assert now.tzinfo is not None and now<=datetime.now(now.tzinfo)
|
||||
assert m.expire_raw(audit,now=now,execute=False)==plan['preview']
|
||||
result=m.expire_raw(audit,now=now)
|
||||
elif action in {'inspect-expiry','reconcile-expiry'}:
|
||||
plan=r['plan']; expected=plan['preview']['expired_raw_copies']
|
||||
states={}
|
||||
for key in expected:
|
||||
assert len(key)==64 and all(c in '0123456789abcdef' for c in key)
|
||||
directory=audit/key
|
||||
m.private_directory(directory)
|
||||
original=m.read_json(directory/'plan.json')
|
||||
assert m.digest(original)==key
|
||||
present=(directory/'message.eml').exists()
|
||||
if not present:
|
||||
intent=m.read_json(directory/'expiry.json')
|
||||
assert intent['raw_sha256']==original['evidence_sha256'] and intent['retention_days']==30
|
||||
states[key]='raw_present' if present else 'raw_expiry_verified'
|
||||
if action=='reconcile-expiry':
|
||||
now=datetime.fromisoformat(plan['as_of'])
|
||||
assert now.tzinfo is not None and now<=datetime.now(now.tzinfo)
|
||||
pending=m.expire_raw(audit,now=now,execute=False)['expired_raw_copies']
|
||||
assert set(pending)=={key for key,state in states.items() if state=='raw_present'}
|
||||
result=m.expire_raw(audit,now=now)
|
||||
else:result={'selected_raw_states':states,'ledger_modified':False}
|
||||
else:raise ValueError('unknown_action')
|
||||
print(json.dumps({'context':context,'result':result}))
|
||||
"""
|
||||
|
||||
|
||||
def native_command(args, payload=None):
|
||||
result = subprocess.run(
|
||||
["k3s", "kubectl", "--kubeconfig=/etc/rancher/k3s/k3s.yaml", *args],
|
||||
input=payload,
|
||||
capture_output=True,
|
||||
timeout=90,
|
||||
check=False,
|
||||
)
|
||||
require(result.returncode == 0, "native_command_unconfirmed")
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def identity(command=native_command):
|
||||
def get(kind, name, namespace="email-connect"):
|
||||
args = ["get", kind, name, "-o", "json"]
|
||||
return command(args + (["-n", namespace] if namespace else []))
|
||||
|
||||
cluster = get("namespace", "kube-system", None)
|
||||
namespace = get("namespace", "email-connect", None)
|
||||
require(
|
||||
cluster["metadata"]["uid"] == CLUSTER and namespace["metadata"]["uid"] == NAMESPACE,
|
||||
"native_cluster_changed",
|
||||
)
|
||||
deployment = get("deployment", "email-connect")
|
||||
require(
|
||||
deployment["metadata"]["uid"] == DEPLOYMENT
|
||||
and not deployment["metadata"].get("deletionTimestamp")
|
||||
and deployment["spec"]["replicas"] == 1
|
||||
and deployment["status"].get("observedGeneration") == deployment["metadata"]["generation"]
|
||||
and deployment["status"].get("readyReplicas") == 1,
|
||||
"sole_ready_deployment_required",
|
||||
)
|
||||
pods = command(
|
||||
[
|
||||
"get",
|
||||
"pods",
|
||||
"-n",
|
||||
"email-connect",
|
||||
"-l",
|
||||
"app.kubernetes.io/name=email-connect",
|
||||
"-o",
|
||||
"json",
|
||||
]
|
||||
)["items"]
|
||||
require(len(pods) == 1, "sole_receiver_required")
|
||||
pod = pods[0]
|
||||
require(
|
||||
not pod["metadata"].get("deletionTimestamp")
|
||||
and pod["status"]["phase"] == "Running"
|
||||
and len(pod["spec"]["containers"]) == 1
|
||||
and pod["spec"]["containers"][0]["name"] == "email-connect"
|
||||
and pod["spec"]["containers"][0]["image"] == IMAGE
|
||||
and any(
|
||||
c.get("name") == "email-connect"
|
||||
and c.get("ready") is True
|
||||
and c.get("imageID", "").endswith(IMAGE.split("@")[1])
|
||||
for c in pod["status"].get("containerStatuses", [])
|
||||
),
|
||||
"pinned_ready_receiver_required",
|
||||
)
|
||||
owners = [
|
||||
o
|
||||
for o in pod["metadata"].get("ownerReferences", [])
|
||||
if o.get("controller") is True and o.get("kind") == "ReplicaSet"
|
||||
]
|
||||
require(len(owners) == 1, "receiver_owner_required")
|
||||
replica = get("replicaset", owners[0]["name"])
|
||||
require(
|
||||
replica["metadata"]["uid"] == owners[0]["uid"]
|
||||
and any(
|
||||
o.get("controller") is True and o.get("uid") == DEPLOYMENT
|
||||
for o in replica["metadata"].get("ownerReferences", [])
|
||||
),
|
||||
"receiver_owner_changed",
|
||||
)
|
||||
volumes = {
|
||||
v["name"]: v.get("persistentVolumeClaim", {}).get("claimName")
|
||||
for v in pod["spec"].get("volumes", [])
|
||||
}
|
||||
mounts = [
|
||||
v
|
||||
for v in pod["spec"]["containers"][0].get("volumeMounts", [])
|
||||
if v.get("mountPath") == "/data"
|
||||
]
|
||||
require(
|
||||
len(mounts) == 1
|
||||
and volumes.get(mounts[0]["name"]) == "email-connect-data"
|
||||
and not mounts[0].get("subPath")
|
||||
and not mounts[0].get("subPathExpr")
|
||||
and not mounts[0].get("readOnly"),
|
||||
"ledger_pvc_mount_required",
|
||||
)
|
||||
pvc = get("pvc", "email-connect-data")
|
||||
require(
|
||||
pvc["status"]["phase"] == "Bound" and not pvc["metadata"].get("deletionTimestamp"),
|
||||
"bound_ledger_pvc_required",
|
||||
)
|
||||
return {
|
||||
"cluster_uid": CLUSTER,
|
||||
"namespace_uid": NAMESPACE,
|
||||
"deployment_uid": DEPLOYMENT,
|
||||
"generation": deployment["metadata"]["generation"],
|
||||
"pod_uid": pod["metadata"]["uid"],
|
||||
"pod_name": pod["metadata"]["name"],
|
||||
"pvc_uid": pvc["metadata"]["uid"],
|
||||
"image": IMAGE,
|
||||
}
|
||||
|
||||
|
||||
def run_node(request, command=native_command):
|
||||
if request["action"] in {"inspect", "reconcile"}:
|
||||
require(
|
||||
isinstance(request.get("operation_id"), str)
|
||||
and re.fullmatch(r"[a-f0-9]{64}", request["operation_id"]),
|
||||
"operation_id_invalid",
|
||||
)
|
||||
current = identity(command)
|
||||
before = request.get("expected_context")
|
||||
if before:
|
||||
fields = (
|
||||
set(current)
|
||||
if request["action"] in {"apply", "apply-expiry"}
|
||||
else {"cluster_uid", "namespace_uid", "deployment_uid", "pvc_uid", "image"}
|
||||
)
|
||||
require(all(current[k] == before[k] for k in fields), "reviewed_native_identity_changed")
|
||||
child = {k: v for k, v in request.items() if k not in {"sources", "source_sha256"}}
|
||||
child.update(
|
||||
core_source=request["sources"]["mail_return_operation.py"],
|
||||
transactional_sha256=TRANSACTIONAL,
|
||||
callers_sha256=CALLERS,
|
||||
)
|
||||
if before and request["action"] in {"apply", "apply-expiry"}:
|
||||
child["policy_sha256"] = before["caller_policy_sha256"]
|
||||
response = command(
|
||||
[
|
||||
"exec",
|
||||
"-i",
|
||||
"-n",
|
||||
"email-connect",
|
||||
current["pod_name"],
|
||||
"--",
|
||||
"python3",
|
||||
"-I",
|
||||
"-c",
|
||||
POD_PROGRAM,
|
||||
],
|
||||
operation.canonical(child),
|
||||
)
|
||||
require(identity(command) == current, "receiver_changed_during_operation_inspect_outcome")
|
||||
return {"context": {**current, **response["context"]}, "result": response["result"]}
|
||||
|
||||
|
||||
def sources():
|
||||
return {name: (ROOT / name).read_text() for name in FILES}
|
||||
|
||||
|
||||
def remote(request):
|
||||
code = sources()
|
||||
hashes = {n: hashlib.sha256(v.encode()).hexdigest() for n, v in code.items()}
|
||||
if request["action"] in {"apply", "apply-expiry", "reconcile", "reconcile-expiry"}:
|
||||
require(request.get("reviewed_source_sha256") == hashes, "reviewed_source_changed")
|
||||
if request.get("reviewed_source_sha256") is not None:
|
||||
require(request["reviewed_source_sha256"] == hashes, "reviewed_source_changed")
|
||||
request = {
|
||||
**request,
|
||||
"sources": code,
|
||||
"source_sha256": hashes,
|
||||
}
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"ssh",
|
||||
"-o",
|
||||
"BatchMode=yes",
|
||||
"-o",
|
||||
"ConnectTimeout=10",
|
||||
"railiance01",
|
||||
"sudo -n python3 -I -c " + shlex.quote(BOOTSTRAP),
|
||||
],
|
||||
input=operation.canonical(request),
|
||||
capture_output=True,
|
||||
timeout=180,
|
||||
check=False,
|
||||
)
|
||||
response = json.loads(result.stdout)
|
||||
require(result.returncode == 0, "native_return_outcome_unconfirmed_inspect_before_retry")
|
||||
return {**response, "source_sha256": hashes}
|
||||
except (subprocess.TimeoutExpired, ValueError):
|
||||
raise operation.Refused("native_return_outcome_unconfirmed_inspect_before_retry") from None
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"action",
|
||||
nargs="?",
|
||||
default="observe",
|
||||
choices=(
|
||||
"observe",
|
||||
"prepare",
|
||||
"release",
|
||||
"apply",
|
||||
"inspect",
|
||||
"reconcile",
|
||||
"prepare-expiry",
|
||||
),
|
||||
)
|
||||
parser.add_argument("--packet", type=Path)
|
||||
parser.add_argument("--message", type=Path)
|
||||
parser.add_argument("--recipient")
|
||||
parser.add_argument("--review")
|
||||
parser.add_argument("--source-operation")
|
||||
parser.add_argument("--invitation-payload", type=Path)
|
||||
parser.add_argument("--expect-digest")
|
||||
args = parser.parse_args(argv)
|
||||
require(__debug__, "optimized_python_refused")
|
||||
request = {"action": args.action}
|
||||
if args.action == "observe":
|
||||
print(json.dumps(remote(request)))
|
||||
return 0
|
||||
require(args.packet is not None and args.packet.is_absolute(), "private_packet_required")
|
||||
operation.private_directory(args.packet.parent)
|
||||
if args.action in {"prepare", "release", "prepare-expiry"}:
|
||||
require(not args.packet.exists(), "new_packet_required")
|
||||
request.update(
|
||||
recipient=args.recipient, review=args.review, source_operation=args.source_operation
|
||||
)
|
||||
if args.action == "prepare":
|
||||
require(
|
||||
args.message is not None
|
||||
and args.message.is_file()
|
||||
and not args.message.is_symlink()
|
||||
and args.message.stat().st_size <= operation.MAX_MESSAGE,
|
||||
"selected_message_required",
|
||||
)
|
||||
request["raw_base64"] = base64.b64encode(args.message.read_bytes()).decode()
|
||||
if args.invitation_payload:
|
||||
request["invitation_payload"] = operation.read_json(args.invitation_payload)
|
||||
if args.action == "prepare-expiry":
|
||||
request["as_of"] = datetime.now(UTC).isoformat()
|
||||
response = remote(request)
|
||||
packet = {
|
||||
"schema": SCHEMA,
|
||||
"context": response["context"],
|
||||
"plan": response["result"],
|
||||
"source_sha256": response["source_sha256"],
|
||||
}
|
||||
value = operation.digest(packet)
|
||||
operation.durable(args.packet, operation.canonical({**packet, "packet_sha256": value}))
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "native_return_plan_prepared",
|
||||
"packet_sha256": value,
|
||||
"ledger_modified": False,
|
||||
"plan_sha256": operation.digest(packet["plan"]),
|
||||
}
|
||||
)
|
||||
)
|
||||
return 0
|
||||
packet = operation.read_json(args.packet)
|
||||
value = packet.pop("packet_sha256")
|
||||
require(
|
||||
packet.get("schema") == SCHEMA and operation.digest(packet) == value,
|
||||
"retained_packet_changed",
|
||||
)
|
||||
require(
|
||||
packet["source_sha256"]
|
||||
== {n: hashlib.sha256(v.encode()).hexdigest() for n, v in sources().items()},
|
||||
"reviewed_source_changed",
|
||||
)
|
||||
if args.action in {"apply", "reconcile"}:
|
||||
require(args.expect_digest == value, "reviewed_packet_required")
|
||||
request.update(
|
||||
expected_context=packet["context"],
|
||||
plan=packet["plan"],
|
||||
operation_id=operation.digest(packet["plan"]),
|
||||
reviewed_source_sha256=packet["source_sha256"],
|
||||
)
|
||||
if packet["plan"].get("action") == "expire-raw" and args.action in {"inspect", "reconcile"}:
|
||||
request["action"] = args.action + "-expiry"
|
||||
if args.action == "apply":
|
||||
if packet["plan"].get("action") == "expire-raw":
|
||||
request["action"] = "apply-expiry"
|
||||
elif packet["plan"]["action"] == "suppress":
|
||||
require(
|
||||
args.message is not None
|
||||
and args.message.is_file()
|
||||
and not args.message.is_symlink()
|
||||
and args.message.stat().st_size <= operation.MAX_MESSAGE,
|
||||
"selected_message_required",
|
||||
)
|
||||
request["raw_base64"] = base64.b64encode(args.message.read_bytes()).decode()
|
||||
response = remote(request)
|
||||
print(
|
||||
json.dumps(
|
||||
{"status": "native_return_result", "result": response["result"], "packet_sha256": value}
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except Exception as error:
|
||||
reason = str(error) if isinstance(error, operation.Refused) else type(error).__name__
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "native_return_requires_inspection",
|
||||
"reason": reason,
|
||||
"retry_automatically": False,
|
||||
}
|
||||
)
|
||||
)
|
||||
sys.exit(1)
|
||||
Loading…
Add table
Add a link
Reference in a new issue