293 lines
13 KiB
Python
293 lines
13 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Value-safe service-identity metadata export for OpenBao and PostgreSQL.
|
||
|
|
|
||
|
|
NK-WP-0054-T04 compares NetKingdom's service-identity declarations with
|
||
|
|
observed state. This exporter produces the observed side as non-secret JSON.
|
||
|
|
|
||
|
|
OpenBao (read-only): ``bao auth list``, ``bao list auth/<mount>/role``,
|
||
|
|
``bao read auth/<mount>/role/<name>`` and ``bao policy read <name>``. It never
|
||
|
|
reads a role-id, secret-id, KV data, token or Secret. Only allowlisted role
|
||
|
|
fields are emitted; every other field returned by OpenBao is dropped.
|
||
|
|
|
||
|
|
PostgreSQL (read-only): ``postgresql-sql`` prints one SELECT over ``pg_roles``,
|
||
|
|
``pg_auth_members`` and ``pg_database`` (never ``pg_authid`` or
|
||
|
|
``rolpassword``); ``postgresql`` turns its single JSON output row into the
|
||
|
|
export. The operator runs the SELECT with their own access; this script holds
|
||
|
|
no database connection.
|
||
|
|
|
||
|
|
Every output string is screened for secret shapes before printing. On a hit
|
||
|
|
the export is refused and only a count is reported, never the value.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
import re
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
from datetime import datetime, timezone
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Any, Callable
|
||
|
|
|
||
|
|
SCHEMA = "railiance-platform.service-identity-metadata.v1"
|
||
|
|
ROLE_MOUNT_TYPES = {"jwt", "oidc", "approle", "kubernetes"}
|
||
|
|
ROLE_FIELDS = (
|
||
|
|
"role_type", "bound_audiences", "bound_subject", "bound_claims", "bound_claims_type",
|
||
|
|
"user_claim", "groups_claim", "claim_mappings", "oidc_scopes", "allowed_redirect_uris",
|
||
|
|
"bound_service_account_names", "bound_service_account_namespaces", "audience",
|
||
|
|
"bind_secret_id", "secret_id_num_uses", "secret_id_bound_cidrs", "token_bound_cidrs",
|
||
|
|
"token_policies", "policies", "token_no_default_policy", "token_type", "token_num_uses",
|
||
|
|
)
|
||
|
|
DURATION_FIELDS = ("token_ttl", "token_max_ttl", "token_explicit_max_ttl", "secret_id_ttl", "ttl", "max_ttl")
|
||
|
|
SECRET_SHAPES = (
|
||
|
|
re.compile(r"^(hvs|hvb|hvr|s|b)\.[A-Za-z0-9_-]{20,}"),
|
||
|
|
re.compile(r"^eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}"),
|
||
|
|
re.compile(r"-----BEGIN [A-Z ]*(PRIVATE KEY|CERTIFICATE)"),
|
||
|
|
re.compile(r"^(SCRAM-SHA-256\$|md5[0-9a-f]{32}$)"),
|
||
|
|
re.compile(r"^[A-Fa-f0-9]{40,}$"),
|
||
|
|
)
|
||
|
|
OPAQUE = re.compile(r"^[A-Za-z0-9+/_=-]{32,}$")
|
||
|
|
DIGEST_KEYS = {"sha256"} # our own policy-text digests, not credentials
|
||
|
|
POLICY_RULE = re.compile(r'path\s+"([^"]+)"\s*\{\s*capabilities\s*=\s*\[([^\]]*)\]', re.S)
|
||
|
|
DURATION = re.compile(r"(\d+)([smhd])")
|
||
|
|
UNIT = {"s": 1, "m": 60, "h": 3600, "d": 86400}
|
||
|
|
|
||
|
|
PG_SQL = (
|
||
|
|
"SELECT json_build_object("
|
||
|
|
"'server_version', current_setting('server_version'),"
|
||
|
|
"'roles', COALESCE((SELECT json_agg(json_build_object("
|
||
|
|
"'name', r.rolname, 'superuser', r.rolsuper, 'create_role', r.rolcreaterole,"
|
||
|
|
"'create_db', r.rolcreatedb, 'can_login', r.rolcanlogin, 'replication', r.rolreplication,"
|
||
|
|
"'bypass_rls', r.rolbypassrls, 'inherit', r.rolinherit, 'connection_limit', r.rolconnlimit,"
|
||
|
|
"'valid_until', r.rolvaliduntil,"
|
||
|
|
"'member_of', COALESCE((SELECT json_agg(json_build_object('role', g.rolname, 'admin_option', m.admin_option)"
|
||
|
|
" ORDER BY g.rolname) FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid"
|
||
|
|
" WHERE m.member = r.oid), '[]'::json)) ORDER BY r.rolname)"
|
||
|
|
" FROM pg_roles r WHERE r.rolname !~ '^pg_'), '[]'::json),"
|
||
|
|
"'databases', COALESCE((SELECT json_agg(json_build_object('name', d.datname,"
|
||
|
|
" 'owner', pg_get_userbyid(d.datdba)) ORDER BY d.datname)"
|
||
|
|
" FROM pg_database d WHERE NOT d.datistemplate), '[]'::json))"
|
||
|
|
)
|
||
|
|
PG_ROLE_FIELDS = ("name", "superuser", "create_role", "create_db", "can_login", "replication",
|
||
|
|
"bypass_rls", "inherit", "connection_limit", "valid_until", "member_of")
|
||
|
|
|
||
|
|
|
||
|
|
class Refused(Exception):
|
||
|
|
"""Raised when output would carry a secret-shaped value; never holds the value."""
|
||
|
|
|
||
|
|
|
||
|
|
def seconds(value: Any) -> int | None:
|
||
|
|
if value is None or value == "":
|
||
|
|
return None
|
||
|
|
if isinstance(value, bool):
|
||
|
|
raise ValueError("duration")
|
||
|
|
if isinstance(value, (int, float)):
|
||
|
|
return int(value)
|
||
|
|
text = str(value).strip()
|
||
|
|
if text.isdigit():
|
||
|
|
return int(text)
|
||
|
|
parts = DURATION.findall(text)
|
||
|
|
if not parts or "".join(n + u for n, u in parts) != text:
|
||
|
|
raise ValueError("duration")
|
||
|
|
return sum(int(n) * UNIT[u] for n, u in parts)
|
||
|
|
|
||
|
|
|
||
|
|
def policy_rules(text: str) -> list[dict[str, Any]]:
|
||
|
|
rules = [{"path": p, "capabilities": sorted(c.strip().strip('"') for c in caps.split(",") if c.strip())}
|
||
|
|
for p, caps in POLICY_RULE.findall(text)]
|
||
|
|
return sorted(rules, key=lambda r: r["path"])
|
||
|
|
|
||
|
|
|
||
|
|
def effective_rules(policies: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
|
|
"""Merge exact-path rules across a role's policies; deny wins (OpenBao semantics).
|
||
|
|
|
||
|
|
Glob and segment-wildcard paths are kept as written, not expanded."""
|
||
|
|
merged: dict[str, set[str]] = {}
|
||
|
|
for policy in policies:
|
||
|
|
for rule in policy["rules"]:
|
||
|
|
merged.setdefault(rule["path"], set()).update(rule["capabilities"])
|
||
|
|
out = []
|
||
|
|
for path, caps in sorted(merged.items()):
|
||
|
|
out.append({"path": path, "capabilities": ["deny"] if "deny" in caps else sorted(caps),
|
||
|
|
"wildcard": "*" in path or "+" in path})
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def role_view(mount: str, mount_type: str, name: str, data: dict[str, Any]) -> dict[str, Any]:
|
||
|
|
view: dict[str, Any] = {"mount": mount, "mount_type": mount_type, "name": name}
|
||
|
|
for key in ROLE_FIELDS:
|
||
|
|
if key in data and data[key] not in (None, "", [], {}):
|
||
|
|
view[key] = data[key]
|
||
|
|
for key in DURATION_FIELDS:
|
||
|
|
if key in data:
|
||
|
|
value = seconds(data[key])
|
||
|
|
if value is not None:
|
||
|
|
view[key + "_seconds"] = value
|
||
|
|
if "policies" in view and "token_policies" not in view:
|
||
|
|
view["token_policies"] = view["policies"]
|
||
|
|
view.pop("policies", None)
|
||
|
|
view.setdefault("role_type", mount_type)
|
||
|
|
return view
|
||
|
|
|
||
|
|
|
||
|
|
class BaoCLI:
|
||
|
|
"""Read-only calls through the operator's own ``bao`` login; no token is handled here."""
|
||
|
|
|
||
|
|
def __init__(self, bao: str = "bao") -> None:
|
||
|
|
self.bao = bao
|
||
|
|
|
||
|
|
def _json(self, *args: str) -> Any:
|
||
|
|
proc = subprocess.run([self.bao, *args, "-format=json"], capture_output=True, text=True, check=False)
|
||
|
|
if proc.returncode != 0:
|
||
|
|
raise RuntimeError(f"bao {args[0]} {args[1] if len(args) > 1 else ''} failed (exit {proc.returncode})")
|
||
|
|
return json.loads(proc.stdout or "null")
|
||
|
|
|
||
|
|
def auth_mounts(self) -> dict[str, Any]:
|
||
|
|
return self._json("auth", "list")
|
||
|
|
|
||
|
|
def roles(self, mount: str) -> list[str]:
|
||
|
|
proc = subprocess.run([self.bao, "list", "-format=json", f"auth/{mount}/role"],
|
||
|
|
capture_output=True, text=True, check=False)
|
||
|
|
if proc.returncode != 0:
|
||
|
|
return [] # no roles, or list not permitted: reported as an empty mount
|
||
|
|
return list(json.loads(proc.stdout or "[]") or [])
|
||
|
|
|
||
|
|
def role(self, mount: str, name: str) -> dict[str, Any]:
|
||
|
|
reply = self._json("read", f"auth/{mount}/role/{name}")
|
||
|
|
return (reply or {}).get("data", {}) or {}
|
||
|
|
|
||
|
|
def policy(self, name: str) -> str | None:
|
||
|
|
proc = subprocess.run([self.bao, "policy", "read", name], capture_output=True, text=True, check=False)
|
||
|
|
return proc.stdout if proc.returncode == 0 else None
|
||
|
|
|
||
|
|
|
||
|
|
class FixtureDir:
|
||
|
|
"""Offline source: auth-list.json, roles/<mount>/<role>.json, policies/<name>.hcl."""
|
||
|
|
|
||
|
|
def __init__(self, root: Path) -> None:
|
||
|
|
self.root = root
|
||
|
|
|
||
|
|
def auth_mounts(self) -> dict[str, Any]:
|
||
|
|
return json.loads((self.root / "auth-list.json").read_text())
|
||
|
|
|
||
|
|
def roles(self, mount: str) -> list[str]:
|
||
|
|
folder = self.root / "roles" / mount
|
||
|
|
return sorted(p.stem for p in folder.glob("*.json")) if folder.is_dir() else []
|
||
|
|
|
||
|
|
def role(self, mount: str, name: str) -> dict[str, Any]:
|
||
|
|
return json.loads((self.root / "roles" / mount / f"{name}.json").read_text())
|
||
|
|
|
||
|
|
def policy(self, name: str) -> str | None:
|
||
|
|
path = self.root / "policies" / f"{name}.hcl"
|
||
|
|
return path.read_text() if path.is_file() else None
|
||
|
|
|
||
|
|
|
||
|
|
def export_openbao(source: Any, mounts_filter: set[str] | None = None) -> dict[str, Any]:
|
||
|
|
mounts = []
|
||
|
|
roles = []
|
||
|
|
for path, meta in sorted((source.auth_mounts() or {}).items()):
|
||
|
|
mount = path.rstrip("/")
|
||
|
|
mtype = (meta or {}).get("type", "unknown")
|
||
|
|
mounts.append({"path": mount, "type": mtype})
|
||
|
|
if mtype not in ROLE_MOUNT_TYPES or (mounts_filter and mount not in mounts_filter):
|
||
|
|
continue
|
||
|
|
for name in source.roles(mount):
|
||
|
|
roles.append(role_view(mount, mtype, name, source.role(mount, name)))
|
||
|
|
policies: dict[str, dict[str, Any]] = {}
|
||
|
|
for name in sorted({p for r in roles for p in r.get("token_policies", [])}):
|
||
|
|
text = source.policy(name)
|
||
|
|
policies[name] = ({"name": name, "readable": False, "rules": []} if text is None else
|
||
|
|
{"name": name, "readable": True, "sha256": hashlib.sha256(text.encode()).hexdigest(),
|
||
|
|
"rules": policy_rules(text)})
|
||
|
|
for role in roles:
|
||
|
|
attached = [policies[p] for p in role.get("token_policies", [])]
|
||
|
|
role["effective_rules"] = effective_rules(attached)
|
||
|
|
role["effective_complete"] = (all(p["readable"] for p in attached)
|
||
|
|
and role.get("token_no_default_policy") is True)
|
||
|
|
return {"auth_mounts": mounts, "roles": roles, "policies": list(policies.values())}
|
||
|
|
|
||
|
|
|
||
|
|
def export_postgresql(raw: str, cluster: str, namespace: str) -> dict[str, Any]:
|
||
|
|
lines = [line for line in raw.splitlines() if line.strip()]
|
||
|
|
if len(lines) != 1:
|
||
|
|
raise ValueError("expected exactly one JSON row from the postgresql-sql query")
|
||
|
|
data = json.loads(lines[0])
|
||
|
|
roles = [{k: r.get(k) for k in PG_ROLE_FIELDS} for r in data.get("roles", [])]
|
||
|
|
databases = [{"name": d["name"], "owner": d["owner"]} for d in data.get("databases", [])]
|
||
|
|
return {"cluster": cluster, "namespace": namespace, "server_version": data.get("server_version"),
|
||
|
|
"roles": roles, "databases": databases}
|
||
|
|
|
||
|
|
|
||
|
|
def looks_secret(text: str) -> bool:
|
||
|
|
if any(p.search(text) for p in SECRET_SHAPES):
|
||
|
|
return True
|
||
|
|
# opaque high-entropy strings: mixed case plus digits, no spaces (names and paths are lower case)
|
||
|
|
return bool(OPAQUE.match(text) and re.search(r"[A-Z]", text) and re.search(r"[a-z]", text)
|
||
|
|
and re.search(r"[0-9]", text))
|
||
|
|
|
||
|
|
|
||
|
|
def secret_hits(value: Any) -> int:
|
||
|
|
if isinstance(value, dict):
|
||
|
|
return sum(0 if k in DIGEST_KEYS else secret_hits(v) for k, v in value.items())
|
||
|
|
if isinstance(value, list):
|
||
|
|
return sum(secret_hits(v) for v in value)
|
||
|
|
if isinstance(value, str):
|
||
|
|
return int(looks_secret(value))
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
def envelope(service: str, kind: str, body: dict[str, Any], now: Callable[[], datetime]) -> dict[str, Any]:
|
||
|
|
doc = {"schema": SCHEMA, "service": service,
|
||
|
|
"collected_at": now().astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||
|
|
"source_kind": kind, "value_safe": True, service: body}
|
||
|
|
hits = secret_hits(body)
|
||
|
|
if hits:
|
||
|
|
raise Refused(f"refused: {hits} secret-shaped value(s) in export; nothing printed")
|
||
|
|
return doc
|
||
|
|
|
||
|
|
|
||
|
|
def main(argv: list[str] | None = None, now: Callable[[], datetime] = lambda: datetime.now(timezone.utc)) -> int:
|
||
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
|
|
sub = parser.add_subparsers(dest="cmd", required=True)
|
||
|
|
bao = sub.add_parser("openbao", help="export auth roles and their policies")
|
||
|
|
bao.add_argument("--fixture-dir", type=Path, help="offline source instead of the bao CLI")
|
||
|
|
bao.add_argument("--bao", default="bao", help="bao CLI binary (uses the operator's own login)")
|
||
|
|
bao.add_argument("--mount", action="append", help="limit to these auth mounts (repeatable)")
|
||
|
|
sub.add_parser("postgresql-sql", help="print the read-only SELECT to run with psql -AtX")
|
||
|
|
pg = sub.add_parser("postgresql", help="export from the SELECT's single output row")
|
||
|
|
pg.add_argument("--input", type=Path, help="file with the psql output (default stdin)")
|
||
|
|
pg.add_argument("--cluster", required=True)
|
||
|
|
pg.add_argument("--namespace", default="databases")
|
||
|
|
parser.add_argument("--output", type=Path, help="write here instead of stdout")
|
||
|
|
args = parser.parse_args(argv)
|
||
|
|
try:
|
||
|
|
if args.cmd == "postgresql-sql":
|
||
|
|
print(PG_SQL)
|
||
|
|
return 0
|
||
|
|
if args.cmd == "openbao":
|
||
|
|
source = FixtureDir(args.fixture_dir) if args.fixture_dir else BaoCLI(args.bao)
|
||
|
|
doc = envelope("openbao", "fixture" if args.fixture_dir else "live",
|
||
|
|
export_openbao(source, set(args.mount) if args.mount else None), now)
|
||
|
|
else:
|
||
|
|
raw = args.input.read_text() if args.input else sys.stdin.read()
|
||
|
|
doc = envelope("postgresql", "live" if not args.input else "file",
|
||
|
|
export_postgresql(raw, args.cluster, args.namespace), now)
|
||
|
|
except Refused as exc:
|
||
|
|
print(str(exc), file=sys.stderr)
|
||
|
|
return 3
|
||
|
|
except (RuntimeError, ValueError, KeyError, json.JSONDecodeError) as exc:
|
||
|
|
print(f"export failed: {type(exc).__name__}", file=sys.stderr)
|
||
|
|
return 2
|
||
|
|
text = json.dumps(doc, indent=2, sort_keys=True) + "\n"
|
||
|
|
if args.output:
|
||
|
|
args.output.write_text(text)
|
||
|
|
else:
|
||
|
|
sys.stdout.write(text)
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
raise SystemExit(main())
|