330 lines
18 KiB
Python
330 lines
18 KiB
Python
"""Founder-run exact scanner rollout. Default: read-only inventory and server dry-run.
|
|
|
|
This acknowledgement check binds reviewed bytes; it is not an approval service or
|
|
credential boundary. Run only in the founder's attended execution environment.
|
|
Raw Kubernetes errors/objects, Git contents and scanner findings are never printed.
|
|
"""
|
|
from pathlib import Path
|
|
import argparse
|
|
import base64
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
import time
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
HOOK = 'pre-receive.d/10-custodian-secret-scan'
|
|
REPO = r'/data/git/gitea-repositories/[a-z0-9_.-]+/[a-z0-9_.-]+\.git'
|
|
RECEIPT = Path('/tmp/forge-scanner-rollout-receipt.json')
|
|
|
|
|
|
def sha(raw):
|
|
return hashlib.sha256(raw).hexdigest()
|
|
|
|
|
|
def encode(value):
|
|
return json.dumps(value, sort_keys=True, separators=(',', ':')).encode()
|
|
|
|
|
|
def ensure(condition, reason):
|
|
if not condition:
|
|
raise RuntimeError(reason)
|
|
|
|
|
|
def verify_packet():
|
|
manifest = json.loads((HERE / 'packet.json').read_bytes())
|
|
for name, digest in manifest['files'].items():
|
|
ensure(Path(name).name == name, 'unsafe-packet-path')
|
|
ensure(sha((HERE / name).read_bytes()) == digest, 'packet-file-drift:' + name)
|
|
return sha((HERE / 'packet.json').read_bytes())
|
|
|
|
|
|
def patch_for(d, cfg, reverse=False):
|
|
spec = d['spec']['template']['spec']
|
|
gi, vi = cfg['guard_index'], cfg['volume_index']
|
|
guard = cfg['new_guard_command'] if reverse else cfg['guard_command']
|
|
replacement = cfg['guard_command'] if reverse else cfg['new_guard_command']
|
|
cm = cfg['new_configmap'] if reverse else cfg['old_configmap']
|
|
dest = cfg['old_configmap'] if reverse else cfg['new_configmap']
|
|
ensure(d['metadata']['uid'] == cfg['deployment_uid'], 'deployment-identity-drift')
|
|
ensure(d['spec'].get('strategy') == cfg['strategy'], 'strategy-drift')
|
|
ensure(spec['initContainers'][gi]['name'] == 'verify-secret-scanner', 'guard-index-drift')
|
|
ensure(spec['initContainers'][gi].get('args') == cfg['guard_args'], 'guard-args-drift')
|
|
ensure(spec['initContainers'][gi]['command'] == guard, 'guard-command-drift')
|
|
vol = copy.deepcopy(cfg['volume']); vol['configMap']['name'] = cm
|
|
ensure(spec['volumes'][vi] == vol, 'volume-drift')
|
|
a = f'/spec/template/spec/initContainers/{gi}/command'
|
|
b = f'/spec/template/spec/volumes/{vi}'
|
|
return [
|
|
{'op': 'test', 'path': '/metadata/uid', 'value': cfg['deployment_uid']},
|
|
{'op': 'test', 'path': '/metadata/resourceVersion', 'value': d['metadata']['resourceVersion']},
|
|
{'op': 'test', 'path': a, 'value': guard},
|
|
{'op': 'test', 'path': b, 'value': vol},
|
|
{'op': 'replace', 'path': a, 'value': replacement},
|
|
{'op': 'replace', 'path': b + '/configMap/name', 'value': dest},
|
|
]
|
|
|
|
|
|
def inventory_script():
|
|
return r'''set +vx
|
|
set -eu
|
|
export PATH=/usr/bin:/bin
|
|
test "$(id -u)" = 1000
|
|
for parent in /data /data/git /data/git/gitea-repositories; do test -d "$parent" && test ! -L "$parent"; done
|
|
count=0
|
|
for repo in /data/git/gitea-repositories/*/*.git; do
|
|
test -d "$repo" || continue
|
|
count=$((count+1)); test "$count" -le 500
|
|
for parent in "${repo%/*}" "$repo" "$repo/hooks" "$repo/hooks/pre-receive.d"; do test -d "$parent" && test ! -L "$parent"; done
|
|
test -z "$(find "$repo/hooks" -type l -print -quit)"
|
|
test -f "$repo/hooks/pre-receive" && test -x "$repo/hooks/pre-receive"
|
|
test -f "$repo/hooks/pre-receive.d/10-custodian-secret-scan"
|
|
find "$repo/hooks" -type f ! -name '*.sample' -exec sh -c '
|
|
for file do printf "%s %s %s " "$(stat -c %a "$file")" "$(stat -c %u "$file")" "$(stat -c %g "$file")"; sha256sum "$file"; done
|
|
' sh {} +
|
|
done
|
|
test "$count" -gt 0
|
|
'''
|
|
|
|
|
|
def parse_inventory(raw, cfg, allowed):
|
|
result = {}
|
|
for line in raw.decode().splitlines():
|
|
parts = line.split(' ', 4)
|
|
ensure(len(parts) == 5, 'malformed-inventory')
|
|
mode, uid, gid, digest, path = parts
|
|
path = path.lstrip(' ')
|
|
match = re.fullmatch('(' + REPO + r')/hooks/([a-zA-Z0-9_.-]+(?:/[a-zA-Z0-9_.-]+)*)', path)
|
|
ensure(match is not None and re.fullmatch('[0-9a-f]{64}', digest), 'unsafe-inventory-entry')
|
|
repo, name = match.groups()
|
|
entries = result.setdefault(repo, {})
|
|
ensure(name not in entries, 'duplicate-hook')
|
|
entries[name] = dict(mode=mode, uid=int(uid), gid=int(gid), sha256=digest)
|
|
ensure(0 < len(result) <= 500, 'empty-or-oversized-inventory')
|
|
for files in result.values():
|
|
ensure(HOOK in files and 'pre-receive' in files, 'uncovered-repository')
|
|
f = files[HOOK]
|
|
ensure(f['uid'] == 1000 and f['gid'] == 1000 and f['mode'] in ('755', '775'), 'scanner-hook-custody-drift')
|
|
ensure(f['sha256'] in allowed, 'unknown-scanner-hook')
|
|
return result
|
|
|
|
|
|
def protected_equal(before, after):
|
|
ensure(set(before) <= set(after), 'repository-disappeared')
|
|
for repo, files in before.items():
|
|
ensure({k: v for k, v in files.items() if k != HOOK} ==
|
|
{k: v for k, v in after[repo].items() if k != HOOK}, 'protected-hook-drift')
|
|
for k in ('mode', 'uid', 'gid'):
|
|
ensure(files[HOOK][k] == after[repo][HOOK][k], 'scanner-hook-metadata-drift')
|
|
|
|
|
|
class Rollout:
|
|
def __init__(self, cfg):
|
|
self.c = cfg
|
|
self.receipt = {'proposal_id': 'SYNC-SCANNER-JSON-DIGEST-20261101', 'revision': 2,
|
|
'events': [], 'native_mutation': False}
|
|
|
|
def k(self, *args, data=None, timeout=60):
|
|
q = subprocess.run(['kubectl', *args], input=data, capture_output=True, timeout=timeout)
|
|
ensure(q.returncode == 0, 'kubectl-command-failed; raw diagnostics withheld')
|
|
return q.stdout
|
|
|
|
def event(self, name, **details):
|
|
self.receipt['events'].append({'event': name, **details})
|
|
RECEIPT.write_text(json.dumps(self.receipt, indent=2) + '\n')
|
|
print(json.dumps({'event': name, **details}), flush=True)
|
|
|
|
def deployment(self):
|
|
d = json.loads(self.k('-n', 'forgejo', 'get', 'deployment', 'forge', '-o', 'json'))
|
|
ensure(d['metadata']['uid'] == self.c['deployment_uid'], 'deployment-identity-drift')
|
|
return d
|
|
|
|
def target(self):
|
|
nodes = json.loads(self.k('get', 'nodes', '-o', 'json'))['items']
|
|
ensure(len(nodes) == 1 and nodes[0]['metadata']['uid'] == self.c['node_uid'], 'node-identity-drift')
|
|
ensure(any(a['address'] == '92.205.62.239' for a in nodes[0]['status']['addresses']), 'node-address-drift')
|
|
|
|
def ready_pod(self, d, seconds=180):
|
|
selector = ','.join(k + '=' + v for k, v in sorted(d['spec']['selector']['matchLabels'].items()))
|
|
until = time.monotonic() + seconds
|
|
while True:
|
|
rs = json.loads(self.k('-n', 'forgejo', 'get', 'replicasets', '-l', selector, '-o', 'json'))['items']
|
|
owners = {x['metadata']['uid'] for x in rs if x['spec']['template']['spec'] == d['spec']['template']['spec'] and any(o['uid'] == self.c['deployment_uid'] for o in x['metadata'].get('ownerReferences', []))}
|
|
pods = json.loads(self.k('-n', 'forgejo', 'get', 'pods', '-l', selector, '-o', 'json'))['items']
|
|
ready = [p for p in pods if not p['metadata'].get('deletionTimestamp') and
|
|
any(o['uid'] in owners for o in p['metadata'].get('ownerReferences', [])) and
|
|
any(c['type'] == 'Ready' and c['status'] == 'True' for c in p.get('status', {}).get('conditions', []))]
|
|
if len(ready) == 1:
|
|
# Require the current Deployment rollout, not a surviving old pod.
|
|
now = self.deployment()
|
|
if now.get('status', {}).get('observedGeneration', 0) >= now['metadata']['generation'] and now.get('status', {}).get('updatedReplicas') == now['spec'].get('replicas', 1) and now.get('status', {}).get('readyReplicas') == now['spec'].get('replicas', 1):
|
|
return ready[0]['metadata']['name']
|
|
ensure(time.monotonic() < until, 'readiness-timeout')
|
|
time.sleep(2)
|
|
|
|
def shell(self, pod, script, args=(), data=None, timeout=90):
|
|
return self.k('-n', 'forgejo', 'exec', '-i', pod, '-c', self.c['container'], '--',
|
|
'sh', '-c', script, 'sh', *args, data=data, timeout=timeout)
|
|
|
|
def inventory(self, pod, allowed):
|
|
return parse_inventory(self.shell(pod, inventory_script()), self.c, allowed)
|
|
|
|
def dry_run(self, d):
|
|
cm = (HERE / 'configmap.json').read_bytes()
|
|
self.k('create', '--dry-run=server', '-f', '-', '-o', 'name', data=cm)
|
|
patch = patch_for(d, self.c)
|
|
self.k('-n', 'forgejo', 'patch', 'deployment', 'forge', '--type=json', '--dry-run=server',
|
|
'-p', json.dumps(patch), '-o', 'name')
|
|
diff = subprocess.run(['kubectl', 'diff', '-f', '-'], input=cm, capture_output=True, timeout=60)
|
|
ensure(diff.returncode in (0, 1), 'client-diff-failed; raw diagnostics withheld')
|
|
self.event('server-dry-run-passed', client_configmap_diff_exit=diff.returncode, changes=['append exact artifact guard', 'select immutable hook ConfigMap'],
|
|
old_configmap=self.c['old_configmap'], new_configmap=self.c['new_configmap'],
|
|
strategy='Recreate', resources_unchanged=True)
|
|
|
|
def preflight(self):
|
|
for name in ('scanner-source-changes.json', 'apps-source-changes.json'):
|
|
for source in json.loads((HERE / name).read_bytes()):
|
|
ensure(sha(Path(source['target']).read_bytes()) == source['original_sha256'], 'owner-source-drift')
|
|
self.target()
|
|
d = self.deployment(); patch_for(d, self.c)
|
|
pod = self.ready_pod(d)
|
|
ensure(pod == self.c['pod'], 'initial-pod-name-drift')
|
|
uid = self.k('-n', 'forgejo', 'get', 'pod', pod, '-o', 'jsonpath={.metadata.uid}').decode()
|
|
ensure(uid == self.c['pod_uid'], 'initial-pod-identity-drift')
|
|
current = self.k('-n', 'forgejo', 'get', 'configmap', self.c['old_configmap'], '-o', 'jsonpath={.data.hook}')
|
|
ensure(sha(current) == self.c['old_hook_sha256'], 'old-template-drift')
|
|
self.shell(pod, self.c['guard_command'][2], ('check',))
|
|
before = self.inventory(pod, {self.c['old_hook_sha256']})
|
|
self.receipt['before_hooks'] = before
|
|
self.event('preflight-passed', repositories=len(before), deployment_uid=self.c['deployment_uid'])
|
|
self.dry_run(d)
|
|
return d, pod, before
|
|
|
|
def migrate(self, pod, direction):
|
|
c = self.c
|
|
# Only public hooks/path names in private tmpfs. Child restores partial writes.
|
|
wrapper = '''set +vx
|
|
set -eu
|
|
umask 077
|
|
test "$(stat -f -c %t /dev/shm)" = 1021994
|
|
area=$(mktemp -d /dev/shm/forge-scanner-hooks-XXXXXXXX)
|
|
cleanup() { code=$?; trap - EXIT; rm -f -- "$area/before" "$area/after" "$area/paths"; rmdir "$area" || code=50; exit "$code"; }
|
|
trap cleanup EXIT
|
|
'''
|
|
for name in ('before', 'after'):
|
|
value = base64.b64encode((HERE / (name + '-hook')).read_bytes()).decode()
|
|
wrapper += 'printf %s ' + shlex.quote(value) + ' | base64 -d > "$area/' + name + '"\n'
|
|
wrapper += 'cat > "$area/paths"\nsh -c ' + shlex.quote((HERE / 'hooks.sh').read_text()) + ' sh "$area" "$1" "$2" "$3"\n'
|
|
before = self.inventory(pod, {c['old_hook_sha256'], c['new_hook_sha256']})
|
|
paths = ''.join(repo + '/hooks/' + HOOK + '\n' for repo, files in sorted(before.items())
|
|
if direction == 'rollback' or files[HOOK]['sha256'] == c['old_hook_sha256'])
|
|
if paths:
|
|
self.shell(pod, wrapper, (direction, c['old_hook_sha256'], c['new_hook_sha256']), paths.encode())
|
|
expected = c['old_hook_sha256'] if direction == 'rollback' else c['new_hook_sha256']
|
|
after = self.inventory(pod, {expected})
|
|
protected_equal(before, after)
|
|
self.event('hooks-' + direction, repositories=len(after))
|
|
return after
|
|
|
|
def probe(self, pod, version, inventory):
|
|
dispatcher = sorted(inventory)[0] + '/hooks/pre-receive'
|
|
result = json.loads(self.shell(pod, (HERE / 'probe.sh').read_text(), (version, dispatcher), timeout=300))
|
|
ensure(all(result[k] for k in ('json_digest_clean_push', 'intermediate_secret_refused', 'scanner_unavailable_refused', 'clean_recovery', 'temporary_probe_removed')), 'probe-incomplete')
|
|
self.event('isolated-receiver-probe-passed', version=version, checks=result)
|
|
|
|
def rollback(self, before=None):
|
|
self.target(); d = self.deployment()
|
|
spec = d['spec']['template']['spec']; cmd = spec['initContainers'][self.c['guard_index']]['command']
|
|
is_new = cmd == self.c['new_guard_command']
|
|
ensure(is_new or cmd == self.c['guard_command'], 'rollback-refused-unknown-guard')
|
|
# The exact reverse patch tests both guard and volume, and current resourceVersion.
|
|
if is_new:
|
|
patch = patch_for(d, self.c, reverse=True)
|
|
self.k('-n', 'forgejo', 'patch', 'deployment', 'forge', '--type=json', '--dry-run=server', '-p', json.dumps(patch), '-o', 'name')
|
|
self.k('-n', 'forgejo', 'patch', 'deployment', 'forge', '--type=json', '-p', json.dumps(patch), '-o', 'name')
|
|
else:
|
|
patch_for(d, self.c)
|
|
pod = self.ready_pod(self.deployment())
|
|
after = self.migrate(pod, 'rollback')
|
|
self.shell(pod, self.c['guard_command'][2], ('check',))
|
|
mounted = self.shell(pod, 'cat /opt/forge-secret-template/hooks/pre-receive.d/10-custodian-secret-scan')
|
|
ensure(sha(mounted) == self.c['old_hook_sha256'], 'rollback-mounted-template-drift')
|
|
if before:
|
|
protected_equal(before, after)
|
|
result = json.loads(self.shell(pod, (HERE / 'rollback-probe.sh').read_text(), (self.c['old_dir'], sorted(after)[0] + '/hooks/pre-receive'), timeout=300))
|
|
ensure(all(result[k] for k in ('clean_push', 'intermediate_secret_refused', 'scanner_unavailable_refused', 'clean_recovery', 'temporary_probe_removed')), 'rollback-probe-incomplete')
|
|
self.event('rollback-probe-passed', checks=result)
|
|
self.event('rollback-verified', old_artifacts_retained=True, staged_artifacts_retained=True)
|
|
|
|
def apply(self, binary):
|
|
d, pod, before = self.preflight()
|
|
raw = binary.read_bytes(); ensure(sha(raw) == self.c['new_adapter_sha256'], 'candidate-binary-drift')
|
|
self.receipt['native_mutation'] = True
|
|
self.event('staging-started')
|
|
try:
|
|
self.shell(pod, (HERE / 'stage.sh').read_text(),
|
|
(self.c['new_dir'], self.c['old_dir'], self.c['new_adapter_sha256'], self.c['scanner_sha256'],
|
|
'SYNC-SCANNER-JSON-DIGEST-20261101-R2'), data=raw)
|
|
self.shell(pod, self.c['new_guard_command'][2], ('check',))
|
|
self.probe(pod, self.c['new_dir'], before)
|
|
# Append-only immutable ConfigMap: do not overwrite an existing object.
|
|
self.k('create', '-f', '-', '-o', 'name', data=(HERE / 'configmap.json').read_bytes())
|
|
self.migrate(pod, 'apply')
|
|
# Refresh version immediately before guarded patch; unrelated fields preserved.
|
|
patch = patch_for(self.deployment(), self.c)
|
|
self.k('-n', 'forgejo', 'patch', 'deployment', 'forge', '--type=json', '-p', json.dumps(patch), '-o', 'name')
|
|
self.event('deployment-patched')
|
|
pod = self.ready_pod(self.deployment())
|
|
after = self.migrate(pod, 'apply') # Includes repos created during transition.
|
|
protected_equal(before, after)
|
|
self.shell(pod, self.c['new_guard_command'][2], ('check',))
|
|
mounted = self.shell(pod, 'cat /opt/forge-secret-template/hooks/pre-receive.d/10-custodian-secret-scan')
|
|
ensure(sha(mounted) == self.c['new_hook_sha256'], 'mounted-template-drift')
|
|
self.probe(pod, self.c['new_dir'], after)
|
|
self.event('activation-verified', repositories=len(after), hook_sha256=self.c['new_hook_sha256'])
|
|
except Exception as exc:
|
|
self.event('activation-failed', reason=str(exc) if isinstance(exc, RuntimeError) else type(exc).__name__)
|
|
try:
|
|
self.rollback(before)
|
|
except Exception as recovery:
|
|
self.event('rollback-unverified', reason=str(recovery) if isinstance(recovery, RuntimeError) else type(recovery).__name__)
|
|
raise
|
|
|
|
|
|
def main():
|
|
global RECEIPT
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('mode', choices=('preflight', 'apply', 'rollback'), nargs='?', default='preflight')
|
|
parser.add_argument('--approved-digest')
|
|
parser.add_argument('--binary', type=Path, default=Path('/tmp/sync-scanner-candidate/receive-secret-scan-candidate'))
|
|
args = parser.parse_args()
|
|
if args.mode == 'preflight':
|
|
RECEIPT = Path('/tmp/forge-scanner-rollout-preflight-receipt.json')
|
|
digest = verify_packet()
|
|
cfg = json.loads((HERE / 'inputs.json').read_bytes())
|
|
if args.mode != 'preflight':
|
|
ensure(args.approved_digest == digest, 'exact-founder-acknowledgement-required')
|
|
runner = Rollout(cfg)
|
|
runner.receipt.update(packet_sha256=digest, mode=args.mode)
|
|
if args.mode == 'apply':
|
|
ensure(not RECEIPT.exists(), 'prior-receipt-present; preserve-and-review-before-retry')
|
|
with RECEIPT.open('x') as stream:
|
|
stream.write(json.dumps(runner.receipt) + '\n')
|
|
runner.apply(args.binary)
|
|
elif args.mode == 'rollback':
|
|
prior = json.loads(RECEIPT.read_bytes()) if RECEIPT.exists() else {}
|
|
runner.receipt['prior_receipt'] = prior
|
|
runner.rollback(prior.get('before_hooks'))
|
|
else:
|
|
runner.preflight()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except Exception as error:
|
|
print(json.dumps({'result': 'refused-or-unverified', 'reason': str(error) if isinstance(error, RuntimeError) else type(error).__name__}), flush=True)
|
|
raise SystemExit(1)
|