net-kingdom/tools/bootstrap-recovery/prune-storage-share-dumps.py

#!/usr/bin/env python3
"""List-first pruning of one CNPG dump folder in the Storage Share Backup account.

Founder-run emergency relief for a full Backup account (NK-WP-0055-T07 finding, 2026-10-29): the
CNPG Option A dumps (`railiance-platform/tools/cmd/cnpg-option-a-backup`) prune only their local copy,
so `railiance-backups/r01-*` folders grow without bound and the identity backup refuses on quota.
Permanent remote retention belongs to railiance-platform; this tool only frees space once.

  plan   --folder r01-net-kingdom-pg --plan ~/prune-plan.json [--keep-daily 7] [--keep-weekly 4]
         read-only: lists the folder, keeps per database the newest N dumps, the newest dump of each of
         the last W ISO weeks and always the newest one, writes the plan, prints its sha256.
  delete --plan ~/prune-plan.json --confirm <plan sha256> --receipt ~/prune-receipt.json
         re-lists, refuses unless every planned file is still there with its size and every kept file
         still exists, deletes only the planned files and checks each is gone.

The Backup app password is read at a hidden prompt. Only `r01-*` dump folders are accepted; the
identity bundle and OpenBao folders are refused. No credential, URL or provider response is printed.
"""

from __future__ import annotations

import argparse
import base64
import getpass
import hashlib
import json
import os
import re
import sys
import urllib.error
import urllib.parse
import urllib.request
import warnings
import xml.etree.ElementTree as ET
from datetime import UTC, datetime
from pathlib import Path

HOST = "https://nx4069.your-storageshare.de"
BASE = "/remote.php/dav/files/Backup/railiance-backups/"
FOLDER = re.compile(r"r01-[a-z0-9-]{1,60}\Z")
DUMP = re.compile(r"(?P<prefix>[a-z0-9_-]{1,120})-(?P<stamp>\d{8}T\d{6}Z)\.dump\.age\Z")
LIMIT = 16 * 1024 * 1024


class Refused(Exception):
    pass


class NoRedirect(urllib.request.HTTPRedirectHandler):
    def redirect_request(self, *args, **kwargs):
        raise Refused("redirect refused")


def opener():
    return urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect())


def prompt():
    if not sys.stdin.isatty() or not sys.stderr.isatty():
        raise Refused("run in your own terminal")
    with warnings.catch_warnings():
        warnings.simplefilter("error", getpass.GetPassWarning)
        secret = getpass.getpass("Storage Share Backup app password: ")
    if not secret or any(c in secret for c in "\r\n\x00"):
        raise Refused("empty or invalid password")
    return secret


def auth(secret):
    return {
        "Authorization": "Basic "
        + base64.b64encode(("Backup:" + secret).encode()).decode(),
        "Accept-Encoding": "identity",
    }


def folder_url(folder):
    if not FOLDER.fullmatch(folder):
        raise Refused("only r01-* dump folders are accepted")
    return HOST + BASE + folder + "/"


def listing(client, headers, folder):
    url = folder_url(folder)
    request = urllib.request.Request(
        url,
        method="PROPFIND",
        headers=dict(headers, Depth="1", **{"Content-Type": "application/xml"}),
        data=b'<d:propfind xmlns:d="DAV:"><d:prop><d:resourcetype/><d:getcontentlength/></d:prop></d:propfind>',
    )
    with client.open(request, timeout=60) as response:
        if response.status != 207 or response.url != url:
            raise Refused("listing refused")
        raw = response.read(LIMIT + 1)
    if len(raw) > LIMIT:
        raise Refused("listing too large")
    files = {}
    for item in ET.fromstring(raw).findall("{DAV:}response"):
        path = urllib.parse.unquote(
            urllib.parse.urlsplit(item.findtext("{DAV:}href") or "").path
        )
        if not path.startswith(BASE + folder + "/"):
            raise Refused("unexpected path in listing")
        name = path[len(BASE + folder + "/") :].rstrip("/")
        if not name:
            continue
        size = None
        for row in item.findall("{DAV:}propstat"):
            if " 200 " in (row.findtext("{DAV:}status") or ""):
                if (
                    row.find("{DAV:}prop/{DAV:}resourcetype/{DAV:}collection")
                    is not None
                ):
                    raise Refused("unexpected subfolder")
                size = row.findtext("{DAV:}prop/{DAV:}getcontentlength")
        if not DUMP.fullmatch(name) or size is None or not size.isdigit():
            raise Refused("unexpected entry in folder; nothing planned")
        files[name] = int(size)
    return files


def select(files, keep_daily, keep_weekly):
    keep, groups = set(), {}
    for name in files:
        match = DUMP.fullmatch(name)
        groups.setdefault(match["prefix"], []).append((match["stamp"], name))
    for entries in groups.values():
        entries.sort(reverse=True)
        keep.update(name for _, name in entries[: max(1, keep_daily)])
        weeks = []
        for stamp, name in entries:
            week = datetime.strptime(stamp, "%Y%m%dT%H%M%SZ").isocalendar()[:2]
            if week not in weeks:
                weeks.append(week)
                if len(weeks) <= keep_weekly:
                    keep.add(name)
    return sorted(keep), sorted(set(files) - keep)


def summary(files, keep, delete):
    groups = {}
    for name in files:
        prefix = DUMP.fullmatch(name)["prefix"]
        g = groups.setdefault(
            prefix,
            {
                "files": 0,
                "bytes": 0,
                "keep": 0,
                "delete": 0,
                "delete_bytes": 0,
                "newest": None,
                "newest_bytes": 0,
                "oldest": None,
            },
        )
        g["files"] += 1
        g["bytes"] += files[name]
        g["keep" if name in keep else "delete"] += 1
        if name in delete:
            g["delete_bytes"] += files[name]
        stamp = DUMP.fullmatch(name)["stamp"]
        if g["newest"] is None or stamp > g["newest"]:
            g["newest"], g["newest_bytes"] = stamp, files[name]
        if g["oldest"] is None or stamp < g["oldest"]:
            g["oldest"] = stamp
    return groups


def gib(n):
    return f"{n / 1024**3:.1f} GiB"


def plan(args, client, secret):
    files = listing(client, auth(secret), args.folder)
    keep, delete = select(files, args.keep_daily, args.keep_weekly)
    record = {
        "schema": "netkingdom.storage-share-dump-prune-plan.v1",
        "folder": args.folder,
        "created_at": datetime.now(UTC).isoformat(),
        "keep_daily": args.keep_daily,
        "keep_weekly": args.keep_weekly,
        "keep": {n: files[n] for n in keep},
        "delete": {n: files[n] for n in delete},
    }
    raw = (json.dumps(record, indent=2, sort_keys=True) + "\n").encode()
    fd = os.open(
        Path(args.plan).expanduser(),
        os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
        0o600,
    )
    with os.fdopen(fd, "wb") as out:
        out.write(raw)
    for prefix, g in sorted(summary(files, set(keep), set(delete)).items()):
        print(
            f"{prefix}: {g['files']} files, {gib(g['bytes'])}, {g['oldest']} .. {g['newest']} "
            f"(newest {g['newest_bytes'] / 1024**2:.0f} MiB); keep {g['keep']}, delete {g['delete']} "
            f"freeing {gib(g['delete_bytes'])}"
        )
    print(
        f"total: keep {len(keep)}, delete {len(delete)} freeing {gib(sum(record['delete'].values()))}"
    )
    print(f"plan sha256: {hashlib.sha256(raw).hexdigest()}")
    return 0


def delete(args, client, secret):
    raw = Path(args.plan).expanduser().read_bytes()
    if hashlib.sha256(raw).hexdigest() != args.confirm:
        raise Refused("plan sha256 does not match --confirm")
    record = json.loads(raw)
    if (
        record.get("schema") != "netkingdom.storage-share-dump-prune-plan.v1"
        or not record["keep"]
    ):
        raise Refused("not a prune plan, or it keeps nothing")
    headers = auth(secret)
    files = listing(client, headers, record["folder"])
    if any(files.get(n) != size for n, size in record["delete"].items()):
        raise Refused("a planned file changed or vanished; make a new plan")
    if any(n not in files for n in record["keep"]):
        raise Refused("a kept file is missing; make a new plan")
    base = folder_url(record["folder"])
    deleted, freed = [], 0
    for name in sorted(record["delete"]):
        url = base + urllib.parse.quote(name)
        with client.open(
            urllib.request.Request(url, method="DELETE", headers=headers), timeout=60
        ) as response:
            if response.status not in (200, 204):
                raise Refused("delete refused")
        try:
            client.open(
                urllib.request.Request(url, method="HEAD", headers=headers), timeout=30
            )
            raise Refused("file still present after delete")
        except urllib.error.HTTPError as error:
            if error.code != 404:
                raise Refused("unexpected status after delete") from None
        deleted.append(name)
        freed += record["delete"][name]
    remaining = listing(client, headers, record["folder"])
    receipt = {
        "schema": "netkingdom.storage-share-dump-prune.v1",
        "folder": record["folder"],
        "plan_sha256": args.confirm,
        "finished_at": datetime.now(UTC).isoformat(),
        "deleted": len(deleted),
        "bytes_freed": freed,
        "kept_all_present": all(n in remaining for n in record["keep"]),
        "remaining_files": len(remaining),
        "credential_recorded": False,
    }
    fd = os.open(
        Path(args.receipt).expanduser(),
        os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
        0o600,
    )
    with os.fdopen(fd, "w") as out:
        out.write(json.dumps(receipt, indent=2, sort_keys=True) + "\n")
    print(
        f"deleted {len(deleted)} files, freed {gib(freed)}; kept files present: {receipt['kept_all_present']}"
    )
    return 0 if receipt["kept_all_present"] else 1


def main(argv=None, client=None, secret=None):
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    sub = parser.add_subparsers(dest="command", required=True)
    p = sub.add_parser("plan")
    p.add_argument("--folder", required=True)
    p.add_argument("--plan", required=True)
    p.add_argument("--keep-daily", type=int, default=7)
    p.add_argument("--keep-weekly", type=int, default=4)
    d = sub.add_parser("delete")
    d.add_argument("--plan", required=True)
    d.add_argument("--confirm", required=True)
    d.add_argument("--receipt", required=True)
    args = parser.parse_args(argv)
    try:
        if args.command == "plan":
            folder_url(args.folder)
            if args.keep_daily < 1 or args.keep_weekly < 0:
                raise Refused("keep at least one daily dump")
        return (plan if args.command == "plan" else delete)(
            args, client or opener(), secret or prompt()
        )
    except Refused as error:
        print(f"refused: {error}", file=sys.stderr)
    except urllib.error.HTTPError as error:
        print(f"refused: provider answered HTTP {error.code}", file=sys.stderr)
    except (urllib.error.URLError, OSError, ET.ParseError):
        print("refused: provider or local I/O failure", file=sys.stderr)
    return 1


if __name__ == "__main__":
    sys.exit(main())