coulomb-social/scripts/source_date_wire.py

"""Pinned helper protocol. Stdin/private journals hold data; stdout is metadata only."""

import base64
import copy
import fcntl
import hashlib
import json
import os
import signal
import stat
import sys
import tempfile
import types
from pathlib import Path


def canonical(value):
    return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode()


def run(payload):
    import django

    django.setup()
    for filename in ("date_backfill.py", "date_backfill_journal.py", "date_backfill_operation.py"):
        source = base64.b64decode(payload["modules"][filename], validate=True)
        if hashlib.sha256(source).hexdigest() != payload["module_hashes"][filename]:
            raise ValueError("Reviewed module changed")
        name = "coulomb_social.apps.spaces." + filename[:-3]
        module = types.ModuleType(name)
        module.__package__ = "coulomb_social.apps.spaces"
        module.__file__ = "<reviewed-" + filename + ">"
        sys.modules[name] = module
        exec(compile(source, module.__file__, "exec"), module.__dict__)

    from django.conf import settings
    from django.db import connection

    from coulomb_social.apps.spaces.date_backfill import prepare_index_date_patch
    from coulomb_social.apps.spaces.date_backfill_operation import (
        _set_date,
        _snapshot,
        execute_date_operation,
        fingerprint,
    )

    if (connection.vendor != "postgresql" or settings.CONTENT_ROOT != "/app/var/content"
            or connection.settings_dict["NAME"] != payload["database_name"]):
        raise ValueError("Pinned application database/content context required")
    with connection.cursor() as cursor:
        cursor.execute("SET lock_timeout='5s'")
        cursor.execute("SET statement_timeout='60s'")
        cursor.execute("SELECT conname FROM pg_constraint "
                       "WHERE conrelid='public.spaces_space'::regclass")
        constraints = {r[0] for r in cursor.fetchall()}
        cursor.execute("SELECT count(*) FROM django_migrations WHERE app='spaces' "
                       "AND name='0003_space_slug_global_uniq'")
        migrated = cursor.fetchone()[0] == 1
    if "spaces_space_slug_global_uniq" not in constraints or not migrated:
        raise ValueError("Storage release prerequisite missing")
    root = Path(settings.CONTENT_ROOT)
    mode = payload["mode"]
    if mode in {"probe", "prepare"}:
        snapshot = _snapshot(root)
        result = {"fingerprint": fingerprint(snapshot), "storage_constraint_verified": True}
        if mode == "prepare":
            if result["fingerprint"] != payload["baseline"]:
                raise ValueError("Reviewed dataset differs")
            selected = payload["plan"]["selection"]
            name = f"spaces/{selected['space_slug']}/pages/index.md"
            patched = prepare_index_date_patch(payload["plan"], (root / name).read_bytes())
            expected = copy.deepcopy(snapshot)
            _set_date(expected, payload["plan"], payload["plan"]["source_created_at"])
            expected["files"][name] = hashlib.sha256(patched).hexdigest()
            result["expected_after"] = fingerprint(expected)
        return result
    if mode not in {"apply", "inspect", "reconcile"}:
        raise ValueError("Unsupported operation")
    journal = Path("/operation/journal.json")
    lock_fd = os.open("/operation/execute.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
    with os.fdopen(lock_fd, "w"):
        try:
            fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
        except BlockingIOError:
            return {"status": "operation_busy", "writers_must_remain_fenced": True}
        if mode == "inspect":
            status = "absent"
            if journal.exists() or journal.is_symlink():
                fd = os.open(journal, os.O_RDONLY | os.O_NOFOLLOW)
                with os.fdopen(fd, "rb") as stream:
                    info = os.fstat(stream.fileno())
                    raw = stream.read(4 * 1024 * 1024 + 1)
                if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid()
                        or info.st_mode & 0o077 or info.st_nlink != 1
                        or len(raw) > 4 * 1024 * 1024):
                    raise ValueError("Private bound journal required")
                record = json.loads(raw)
                original = base64.b64decode(record["original"], validate=True)
                patched = prepare_index_date_patch(payload["plan"], original)
                if (record["schema"] != "coulomb.source-date-journal.v1"
                        or record["plan"] != payload["plan"]
                        or record["backup_receipt_sha256"] != hashlib.sha256(
                            canonical(payload["backup"])).hexdigest()
                        or record["patched_sha256"] != hashlib.sha256(patched).hexdigest()
                        or record["state"] not in {"pending", "completed", "rolled_back"}):
                    raise ValueError("Original operation journal binding changed")
                status = record["state"]
            return {"status": status, "fingerprint": fingerprint(_snapshot(root))}
        proof_path = Path("/operation/result.json")
        result = execute_date_operation(
            root, payload["plan"], journal, payload["baseline"], payload["backup"],
            image=payload["image"], recipient=payload["recipient"], reconcile=mode == "reconcile")
        result["fingerprint"] = fingerprint(_snapshot(root))
        fd, temporary = tempfile.mkstemp(dir="/operation", prefix=".result-")
        with os.fdopen(fd, "wb") as stream:
            stream.write(canonical(result))
            stream.flush()
            os.fsync(stream.fileno())
        os.replace(temporary, proof_path)
        directory = os.open("/operation", os.O_RDONLY | os.O_DIRECTORY)
        try:
            os.fsync(directory)
        finally:
            os.close(directory)
        return result


if __name__ == "__main__":
    # A lost kubectl client must not leave an unbounded helper operation behind.
    # SIGALRM kills this process/SQL connection; the durable journal reconciles it.
    signal.alarm(150)
    try:
        raw = sys.stdin.buffer.read(4 * 1024 * 1024 + 1)
        if len(raw) > 4 * 1024 * 1024:
            raise ValueError("Bounded input required")
        print(json.dumps(run(json.loads(raw))))
    except Exception as exc:
        print(json.dumps({"status": "refused", "failure_type": type(exc).__name__}))
        raise SystemExit(1)