security: close unauthenticated runner Docker exposure with native proof

This commit is contained in:
codex 2026-10-13 18:50:05 +02:00
parent c12c087b89
commit 9c18673da1
16 changed files with 614 additions and 2 deletions

View file

@ -0,0 +1,214 @@
#!/usr/bin/env python3
"""Conditional, receipt-producing reduction of the runner Docker listener scope."""
import argparse
import copy
import datetime
import hashlib
import importlib.util
import ipaddress
import json
from pathlib import Path
import re
import time
HERE = Path(__file__).parent
s = importlib.util.spec_from_file_location('native', HERE / 'verify_receive_native.py')
n = importlib.util.module_from_spec(s); s.loader.exec_module(n)
NAME = 'forgejo-runner'
SELECTOR = 'app.kubernetes.io/name=forgejo-runner'
PROBE = ['sh', '-c', 'wget -q -O- -T 3 http://127.0.0.1:2375/_ping | grep -qx OK']
def digest(value):
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
def deployment():
return json.loads(n.command(['-n', 'forgejo', 'get', 'deployment', NAME, '-o', 'json']))
def ready_pod(d):
pods = json.loads(n.command(['-n', 'forgejo', 'get', 'pods', '-l', SELECTOR, '-o', 'json']))['items']
pods = [p for p in pods if not p['metadata'].get('deletionTimestamp') and
p.get('status', {}).get('phase') == 'Running' and
any(c['type'] == 'Ready' and c['status'] == 'True' for c in p.get('status', {}).get('conditions', []))]
if len(pods) != 1 or d['status'].get('observedGeneration', 0) < d['metadata']['generation'] or d['status'].get('availableReplicas') != 1:
raise ValueError('runner-not-uniquely-ready')
return pods[0]
def execute(pod, container, *args, timeout=30):
return n.command(['-n', 'forgejo', 'exec', pod['metadata']['name'], '-c', container, '--', *args], timeout=timeout)
def job_counts():
query = "SELECT json_build_object('tasks_running',(SELECT count(*) FROM action_task WHERE status=6),'tasks_waiting',(SELECT count(*) FROM action_task WHERE status=5),'jobs_running',(SELECT count(*) FROM action_run_job WHERE status=6),'jobs_waiting',(SELECT count(*) FROM action_run_job WHERE status=5))"
return json.loads(n.command(['-n', 'databases', 'exec', 'forgejo-db-1', '--', 'psql',
'-U', 'postgres', '-d', 'forgejo', '-At', '-c', query]))
def boundary(pod):
# Emit only fixed booleans and listener addresses, never daemon inventories.
address = str(ipaddress.ip_address(pod['status']['podIP']))
local = execute(pod, 'runner', 'wget', '-qO-', '-T', '3', 'http://127.0.0.1:2375/_ping').strip() == b'OK'
peers = json.loads(n.command(['-n', 'forgejo', 'get', 'pods', '-l', 'app.kubernetes.io/instance=forgejo', '-o', 'json']))['items']
peers = [p for p in peers if not p['metadata'].get('deletionTimestamp') and
p.get('status', {}).get('phase') == 'Running' and
any(c['name'] == 'gitea' for c in p['spec']['containers'])]
if len(peers) != 1:
raise ValueError('peer-not-unique')
other = execute(peers[0], 'gitea', 'sh', '-c',
'command -v wget >/dev/null || exit 21; if output=$(wget -qO- -T 3 "$1" 2>/dev/null); then test "$output" = OK || exit 22; printf reachable; else printf refused; fi',
'sh', 'http://' + address + ':2375/_ping').decode()
listeners = execute(pod, 'dind', 'sh', '-c',
"awk '$2 ~ /:0947$/ && $4 == \"0A\" {print $2}' /proc/net/tcp /proc/net/tcp6").decode().splitlines()
if other not in ('reachable', 'refused') or any(not re.fullmatch(r'[0-9A-F]{8,32}:0947', a) for a in listeners):
raise ValueError('boundary-output-unexpected')
return {'runner_loopback_ping': local, 'other_pod_ping': other,
'docker_listen_addresses_hex': listeners, 'pod_uid': pod['metadata']['uid']}
def candidate(template, images):
result = copy.deepcopy(template)
spec = result['spec']
containers = {c['name']: c for c in spec['containers']}
if set(containers) != {'dind', 'runner'}:
raise ValueError('unexpected-containers')
dind = containers['dind']
if dind['args'] != ['dockerd', '-H', 'tcp://0.0.0.0:2375', '--tls=false']:
raise ValueError('original-listener-differs')
readiness = dind['readinessProbe']
if readiness.get('tcpSocket') != {'port': 2375} or any(k in readiness for k in ('exec', 'httpGet', 'grpc')):
raise ValueError('original-probe-differs')
hosts = [x for x in containers['runner']['env'] if x['name'] == 'DOCKER_HOST']
if hosts != [{'name': 'DOCKER_HOST', 'value': 'tcp://127.0.0.1:2375'}]:
raise ValueError('runner-docker-host-differs')
dind['args'][2] = 'tcp://127.0.0.1:2375'
del readiness['tcpSocket']
readiness['exec'] = {'command': PROBE}
# Pin the exact currently running images, including registration init.
for c in spec['containers']:
c['image'] = images[c['name']]
for c in spec.get('initContainers', []):
if c['name'] != 'register':
raise ValueError('unexpected-init-container')
c['image'] = images['runner']
return result
def patch(d, target):
return [{'op': 'test', 'path': '/metadata/uid', 'value': d['metadata']['uid']},
{'op': 'test', 'path': '/metadata/resourceVersion', 'value': d['metadata']['resourceVersion']},
{'op': 'test', 'path': '/spec/template', 'value': d['spec']['template']},
{'op': 'replace', 'path': '/spec/template', 'value': target}]
def send_patch(operations, dry_run=False):
args = ['-n', 'forgejo', 'patch', 'deployment', NAME, '--type=json', '--patch-file=/dev/stdin', '-o', 'json']
if dry_run:
args.append('--dry-run=server')
return json.loads(n.command(args, payload=json.dumps(operations).encode(), timeout=45))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--proposal', type=Path, required=True)
parser.add_argument('--receipt', type=Path, required=True)
parser.add_argument('--baseline', type=Path, required=True, help='Private exact original template for guarded rollback')
modes = parser.add_mutually_exclusive_group()
modes.add_argument('--apply', action='store_true')
modes.add_argument('--rollback', action='store_true')
args = parser.parse_args()
if args.receipt.exists():
raise SystemExit('Retain prior receipt; never replay an ambiguous change.')
p = json.loads(args.proposal.read_text())
record = {'proposal_id': p['proposal_id'], 'proposal_sha256': hashlib.sha256(args.proposal.read_bytes()).hexdigest(),
'at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'result': 'unverified',
'mode': 'rollback' if args.rollback else 'apply' if args.apply else 'dry-run'}
def save():
args.receipt.write_text(json.dumps(record, indent=2) + '\n')
try:
for name, expected in p['source_sha256'].items():
if Path(name).name != name or hashlib.sha256((HERE / name).read_bytes()).hexdigest() != expected:
raise ValueError('executor-changed')
original = json.loads(args.baseline.read_text())
if digest(original) != p['original_template_digest']:
raise ValueError('baseline-changed')
target = candidate(original, p['images'])
if digest(target) != p['candidate_template_digest']:
raise ValueError('candidate-changed')
d = deployment()
expected = p['candidate_template_digest'] if args.rollback else p['original_template_digest']
if d['metadata']['uid'] != p['deployment_uid'] or digest(d['spec']['template']) != expected:
raise ValueError('deployment-changed')
pod = ready_pod(d)
before_images = {c['name']: c['imageID'] for c in pod['status']['containerStatuses']}
if before_images != p['images'] or any(c['imageID'] != p['images']['runner'] for c in pod['status'].get('initContainerStatuses', [])):
raise ValueError('running-images-changed')
record['before'] = boundary(pod)
jobs = job_counts(); record['jobs_before'] = jobs
if any(jobs.values()):
raise ValueError('runner-not-idle')
if d['spec']['strategy'] != {'type': 'Recreate'}:
raise ValueError('rollout-strategy-changed')
# Read-only headroom check; Recreate and unchanged requests avoid surge.
nodes = json.loads(n.command(['get', 'nodes', '-o', 'json']))['items']
if len(nodes) != 1 or not any(a['address'] == '92.205.62.239' for a in nodes[0]['status']['addresses']):
raise ValueError('wrong-node')
record['node_cpu_allocatable'] = nodes[0]['status']['allocatable']['cpu']
description = n.command(['describe', 'node', nodes[0]['metadata']['name']]).decode()
section = description.split('Allocated resources:', 1)[1]
cpu = re.search(r'^\s*cpu\s+(\S+)\s+\((\d+)%\)', section, re.MULTILINE)
if not cpu:
raise ValueError('cpu-headroom-unverified')
record['node_cpu_requested'] = cpu.group(1)
record['node_cpu_requested_percent'] = int(cpu.group(2))
record['no_surge_or_request_increase'] = True
target = original if args.rollback else target
operations = patch(d, target)
dry = send_patch(operations, True)
if digest(dry['spec']['template']) != digest(target):
raise ValueError('server-defaulting-differs')
record['server_dry_run_verified'] = True
record['result'] = 'dry-run-verified'; save()
if not (args.apply or args.rollback):
print(json.dumps(record)); return
# Recheck immediately before mutation; a concurrent new job is an abort.
record['jobs_at_apply'] = job_counts()
if any(record['jobs_at_apply'].values()):
raise ValueError('runner-became-busy')
record['result'] = 'apply-outcome-unverified'; save()
changed = send_patch(operations)
record.update(result='rollout-started', generation=changed['metadata']['generation']); save()
deadline = time.monotonic() + 360
while time.monotonic() < deadline:
after = deployment()
try:
new = ready_pod(after)
if new['metadata']['uid'] != pod['metadata']['uid'] and digest(after['spec']['template']) == digest(target):
break
except ValueError:
pass
time.sleep(3)
else:
raise ValueError('rollout-unverified')
record['after'] = boundary(new)
record['jobs_after'] = job_counts()
images = {c['name']: c['imageID'] for c in new['status']['containerStatuses']}
record['images_unchanged'] = images == p['images'] and all(c['imageID'] == p['images']['runner'] for c in new['status'].get('initContainerStatuses', []))
expected_other = 'reachable' if args.rollback else 'refused'
expected_listeners = p['original_listener_addresses_hex'] if args.rollback else ['0100007F:0947']
if (not record['after']['runner_loopback_ping'] or record['after']['other_pod_ping'] != expected_other or
record['after']['docker_listen_addresses_hex'] != expected_listeners or not record['images_unchanged']):
raise ValueError('native-boundary-unverified')
record['result'] = 'rollback-verified' if args.rollback else 'verified'; save()
print(json.dumps(record))
except Exception:
record['result'] = 'unverified'
record['raw_diagnostics_retained'] = False
save()
raise SystemExit('Runner change incomplete; inspect sanitized receipt and live state before retry.')
if __name__ == '__main__':
main()

View file

@ -0,0 +1,52 @@
# Runner Docker listener reduction
FORGE6-RUNNER-LOOPBACK-001, under FORGE-WP-0006-T03 and CUST-WP-0080 G12.
The October 13 inspection reproduced unauthenticated Docker ping from another
pod. The live DinD listener covered all pod interfaces and the namespace had no
NetworkPolicy. The declared runner consumer already used `127.0.0.1:2375`.
[Docker's daemon guidance](https://docs.docker.com/engine/daemon/remote-access/)
describes the security implications of network access to this control interface.
The source change is railiance-apps `706461b`, in
`manifests/forgejo-runner.yaml`. It binds Docker to loopback, changes DinD readiness
to a local exec ping, and pins all container/init image references to the exact
bytes already running. It changes no resources, registration, credential path,
replica count, PVC or workflow. Existing `Recreate` strategy avoids a surge pod.
Revision 1 only ran a preflight. That observed a dual-stack wildcard socket, so
revision 2 binds rollback verification to the observed original addresses rather
than assuming an IPv4 wildcard. Both proposals and preflight receipts remain.
Revision 2's conditional server dry-run matched exactly; three patch-boundary
tests passed. CPU requests were 3460m of 4 cores (86%); this change adds no requests.
The executor rechecks zero running/queued tasks immediately before the conditional
patch. This reduces but cannot atomically eliminate a concurrently arriving job.
Actual activation found no running/queued tasks before or after. Generation 4
became Ready with the original image digests. The runner's loopback ping passed;
another Forgejo pod's ping was refused, and `/proc/net/tcp` showed only
`0100007F:0947`. The executor exited 0. Receipt:
`docs/evidence/2026-10-13-netkingdom-security/forge-runner-loopback-activation.json`.
The private original template is retained, mode 0600, in ignored
`.private-backups/forge-runner-listener-20261013/original-template.json` and bound
by digest. Rollback requires the exact candidate template and Deployment UID:
```sh
python3 docs/changes/CUST-WP-0080/close_runner_docker_listener.py \
--proposal docs/changes/CUST-WP-0080/forge-runner-loopback-proposal.json \
--baseline .private-backups/forge-runner-listener-20261013/original-template.json \
--receipt docs/evidence/2026-10-13-netkingdom-security/forge-runner-loopback-rollback.json \
--rollback
```
Rollback restores the known prior exposure and is not automatic on a security
verification failure. Abort owner: railiance-forge, with railiance-apps owning
the source declaration. Never rerun apply because an observation timed out;
inspect the exact running session and live template first. An unexpected template
change refuses rollback instead of overwriting concurrent work. Reconcile source
if rollback is actually used. No rollback was performed in acceptance.
This closes the reproduced cross-pod Docker listener defect. Host-mode workflows
still share a privileged runner boundary. Job identity, source/workflow authority,
required checks, independent detection and credential isolation retain their
existing FORGE-WP-0006 / RMGR-WP-0021 / CUST-WP-0080 gates.

View file

@ -0,0 +1,25 @@
{
"proposal_id": "FORGE6-RUNNER-LOOPBACK-001",
"revision": 1,
"deployment_uid": "89c8a056-bcae-4940-acdd-2eb2768e108a",
"original_template_digest": "e9dbbea8192bbf805f4f5838db844d40ac9d3c331739952fd3df412cce09bcca",
"candidate_template_digest": "1ded8e6fc18d5911bf95d518e19a43cf855802e77582ae04e624fd589937e130",
"images": {
"dind": "docker.io/library/docker@sha256:aa3df78ecf320f5fafdce71c659f1629e96e9de0968305fe1de670e0ca9176ce",
"runner": "code.forgejo.org/forgejo/runner@sha256:5071e6832313bafe71577e05631bece88caff08fcfb372193e4a21941f4ed54b"
},
"source_sha256": {
"close_runner_docker_listener.py": "2ad56666680125f3e13dba4c47046618bb0ff0222d05dcae91699a204eae61e8",
"verify_receive_native.py": "c9120122c6f6f0b7c41fe4fa975e5a2a340e776885d852b861b8c13b7664adcd"
},
"authorization_ref": "canon/standards/build-mode-change-authorization_v0.1.md",
"target": "forgejo/Deployment forgejo-runner only; existing Docker and runner binaries pinned by digest",
"expected_result": "Runner remains Ready with same image bytes and loopback Docker ping; actual Docker listener is 127.0.0.1 only and a separate Forgejo pod cannot reach it. No running or queued tasks at apply. No replica, resource-request, credential or policy expansion.",
"diff_ref": "railiance-apps/manifests/forgejo-runner.yaml; exact conditional template patch and server dry-run",
"rollback": "Run same executor --rollback with this proposal, ignored original-template.json baseline and a new receipt, only against exact candidate template/UID. Restores original listener and image references; records restored exposure. No automatic rollback on a security-probe failure; abort owner reviews live state.",
"abort_owner": "railiance-forge, source owner railiance-apps; custodian-codex executes",
"cost_estimate_eur": 0,
"risk_estimate": "Brief runner unavailability during existing Recreate rollout and cold ephemeral Docker cache. Abort if running/queued work appears before apply. An external Docker consumer using the pod IP loses access; the declared runner uses loopback. A concurrent newly scheduled job remains a race; preserve any nonterminal outcome. Original private template retained for exact guarded rollback. No new resources/spend or data deletion requested.",
"scope_limit": "Reduces unauthenticated cross-pod Docker authority only. Host-mode jobs still share the privileged runner boundary; no full CI isolation, workflow authority, required-check or secret detection conformance inferred.",
"baseline_path": ".private-backups/forge-runner-listener-20261013/original-template.json"
}

View file

@ -0,0 +1,31 @@
{
"proposal_id": "FORGE6-RUNNER-LOOPBACK-001",
"revision": 2,
"deployment_uid": "89c8a056-bcae-4940-acdd-2eb2768e108a",
"original_template_digest": "e9dbbea8192bbf805f4f5838db844d40ac9d3c331739952fd3df412cce09bcca",
"candidate_template_digest": "1ded8e6fc18d5911bf95d518e19a43cf855802e77582ae04e624fd589937e130",
"images": {
"dind": "docker.io/library/docker@sha256:aa3df78ecf320f5fafdce71c659f1629e96e9de0968305fe1de670e0ca9176ce",
"runner": "code.forgejo.org/forgejo/runner@sha256:5071e6832313bafe71577e05631bece88caff08fcfb372193e4a21941f4ed54b"
},
"source_sha256": {
"close_runner_docker_listener.py": "3763a4f90ed964aad1bf5fc580318d891501dc6d908de8554a2cd7d35854e3e4",
"verify_receive_native.py": "c9120122c6f6f0b7c41fe4fa975e5a2a340e776885d852b861b8c13b7664adcd"
},
"authorization_ref": "canon/standards/build-mode-change-authorization_v0.1.md",
"target": "forgejo/Deployment forgejo-runner only; existing Docker and runner binaries pinned by digest",
"expected_result": "Runner remains Ready with same image bytes and loopback Docker ping; actual Docker listener is 127.0.0.1 only and a separate Forgejo pod cannot reach it. No running or queued tasks at apply. No replica, resource-request, credential or policy expansion.",
"diff_ref": "railiance-apps/manifests/forgejo-runner.yaml; exact conditional template patch and server dry-run",
"rollback": "Run same executor --rollback with this proposal, ignored original-template.json baseline and a new receipt, only against exact candidate template/UID. Restores original listener and image references; records restored exposure. No automatic rollback on a security-probe failure; abort owner reviews live state.",
"abort_owner": "railiance-forge, source owner railiance-apps; custodian-codex executes",
"cost_estimate_eur": 0,
"risk_estimate": "Brief runner unavailability during existing Recreate rollout and cold ephemeral Docker cache. Abort if running/queued work appears before apply. An external Docker consumer using the pod IP loses access; the declared runner uses loopback. A concurrent newly scheduled job remains a race; preserve any nonterminal outcome. Original private template retained for exact guarded rollback. No new resources/spend or data deletion requested.",
"scope_limit": "Reduces unauthenticated cross-pod Docker authority only. Host-mode jobs still share the privileged runner boundary; no full CI isolation, workflow authority, required-check or secret detection conformance inferred.",
"baseline_path": ".private-backups/forge-runner-listener-20261013/original-template.json",
"original_revision_sha256": "a2d29132c639f76bed0465a30119148d207f9242c29a2229a2e90cfc5a6aecac",
"original_listener_addresses_hex": [
"00000000000000000000000000000000:0947"
],
"preparation_refinement": "Read-only preflight showed Docker wildcard binding in /proc/net/tcp6, so rollback verification binds the observed original listener instead of assuming an IPv4-only socket. No live mutation in revision 1.",
"source_commit": "railiance-apps@706461b"
}

View file

@ -0,0 +1,55 @@
import copy
import importlib.util
import unittest
from pathlib import Path
s = importlib.util.spec_from_file_location('listener', Path(__file__).with_name('close_runner_docker_listener.py'))
m = importlib.util.module_from_spec(s); s.loader.exec_module(m)
class Tests(unittest.TestCase):
def fixture(self):
return {'metadata': {'labels': {'existing': 'preserved'}}, 'spec': {
'containers': [
{'name': 'dind', 'image': 'docker:27-dind', 'args': ['dockerd', '-H', 'tcp://0.0.0.0:2375', '--tls=false'],
'readinessProbe': {'tcpSocket': {'port': 2375}, 'periodSeconds': 10},
'resources': {}, 'securityContext': {'privileged': True}},
{'name': 'runner', 'image': 'runner:6.3.1', 'env': [{'name': 'DOCKER_HOST', 'value': 'tcp://127.0.0.1:2375'}]}],
'initContainers': [{'name': 'register', 'image': 'runner:6.3.1', 'env': [{'name': 'REGISTRATION_TOKEN', 'valueFrom': {'secretKeyRef': {'name': 'existing', 'key': 'token'}}}]}],
'volumes': [{'name': 'data', 'persistentVolumeClaim': {'claimName': 'existing'}}]}}
def test_changes_only_listener_probe_and_exact_images(self):
old = self.fixture(); frozen = copy.deepcopy(old)
new = m.candidate(old, {'dind': 'docker@sha256:existing', 'runner': 'runner@sha256:existing'})
self.assertEqual(old, frozen)
containers = new['spec']['containers']
self.assertEqual(containers[0]['args'][2], 'tcp://127.0.0.1:2375')
self.assertEqual(containers[0]['readinessProbe']['exec']['command'], m.PROBE)
self.assertNotIn('tcpSocket', containers[0]['readinessProbe'])
containers[0]['args'][2] = old['spec']['containers'][0]['args'][2]
containers[0]['readinessProbe'] = old['spec']['containers'][0]['readinessProbe']
for new_c, old_c in zip(containers, old['spec']['containers']):
new_c['image'] = old_c['image']
new['spec']['initContainers'][0]['image'] = old['spec']['initContainers'][0]['image']
self.assertEqual(new, old)
def test_refuses_unexpected_listener_probe_and_consumer(self):
for kind in ('listener', 'probe', 'consumer', 'container'):
with self.subTest(kind=kind):
value = self.fixture()
if kind == 'listener': value['spec']['containers'][0]['args'][2] = 'tcp://different:1234'
if kind == 'probe': value['spec']['containers'][0]['readinessProbe']['httpGet'] = {}
if kind == 'consumer': value['spec']['containers'][1]['env'][0]['value'] = 'tcp://other-host:2375'
if kind == 'container': value['spec']['containers'].append({'name': 'unknown'})
with self.assertRaises(ValueError): m.candidate(value, {'dind': 'pinned', 'runner': 'pinned'})
def test_patch_guards_exact_identity_revision_and_template(self):
old = {'metadata': {'uid': 'owned', 'resourceVersion': '12'}, 'spec': {'template': self.fixture()}}
patch = m.patch(old, {'desired': 'template'})
self.assertEqual([p['path'] for p in patch[:3]], ['/metadata/uid', '/metadata/resourceVersion', '/spec/template'])
self.assertTrue(all(p['op'] == 'test' for p in patch[:3]))
self.assertEqual(patch[2]['value'], old['spec']['template'])
self.assertEqual(patch[3], {'op': 'replace', 'path': '/spec/template', 'value': {'desired': 'template'}})
if __name__ == '__main__': unittest.main()