diff --git a/Makefile b/Makefile index deb9acd..5758ebf 100644 --- a/Makefile +++ b/Makefile @@ -131,21 +131,21 @@ backup: ## Backup S1 OS config to /opt/backup/railiance/infra/ (age-encrypted, r # ---- Ansible ---- ansible-bootstrap: ## Run base bootstrap play (users, ssh, ufw, sops-agent, custodian-agent) - cd ansible && ansible-playbook playbooks/bootstrap.yaml -u admin + cd ansible && ansible-playbook playbooks/bootstrap.yaml $(ANSIBLE_USER_FLAG) provision-custodian-agent: ## Deploy custodian agent SSH key to all managed hosts @python3 -c "import yaml; d=yaml.safe_load(open('ansible/inventory/group_vars/all.yaml')); k=d.get('custodian_agent_pubkey',''); exit(0 if k else 1)" \ || (echo "ERROR: custodian_agent_pubkey is empty. Run: cd ~/the-custodian && make custodian-keygen"; exit 1) - cd ansible && ansible-playbook playbooks/custodian-agent.yaml -u $(SSH_USER) + cd ansible && ansible-playbook playbooks/custodian-agent.yaml $(ANSIBLE_USER_FLAG) provision-custodian-agent-host: ## Deploy custodian agent key to one host: make provision-custodian-agent-host HOST=Railiance01 @test -n "$(HOST)" || (echo "Usage: make provision-custodian-agent-host HOST=Railiance01"; exit 1) - cd ansible && ansible-playbook playbooks/custodian-agent.yaml -u $(SSH_USER) \ + cd ansible && ansible-playbook playbooks/custodian-agent.yaml $(ANSIBLE_USER_FLAG) \ --limit "$(HOST)" bootstrap-ssh-ca: ## Deploy OpenBao SSH CA trust + auth_principals: make bootstrap-ssh-ca SSH_CA_PUBKEY=/path/to/ca_user.pub @test -n "$(SSH_CA_PUBKEY)" || (echo "Usage: make bootstrap-ssh-ca SSH_CA_PUBKEY=/path/to/ca_user.pub [HOST=Railiance01]"; exit 1) - cd ansible && ansible-playbook playbooks/bootstrap-ssh-ca.yaml -u $(SSH_USER) \ + cd ansible && ansible-playbook playbooks/bootstrap-ssh-ca.yaml $(ANSIBLE_USER_FLAG) \ -e ssh_ca_pubkey_path="$(SSH_CA_PUBKEY)" \ $(if $(HOST),--limit "$(HOST)",) @@ -167,7 +167,7 @@ deploy-stack: ## Print the full S1→S5 ordered deploy sequence (operator follow @echo "PRE-CONDITIONS" @echo " [ ] SSH key: ~/.ssh/id_ops" @echo " [ ] SOPS key: ~/.config/sops/age/keys.txt (or SOPS_AGE_KEY)" - @echo " [ ] ops-bridge: bridge up state-hub-coulombcore k3s-api-coulombcore" + @echo " [ ] ops-bridge: bridge up state-hub-coulombcore k3s-api-coulombcore k3s-api-railiance01" @echo "" @echo "S1 — Infrastructure Substrate (this repo)" @echo " make tf-plan && make tf-apply # provision server (skip if exists)" @@ -222,7 +222,10 @@ remote-set: ## Set origin to your Gitea repo (GITEA/OWNER/REPO vars) ANS_DIR := ansible INV_SCRIPT := $(ANS_DIR)/inventory_from_yaml.py PLAY := $(ANS_DIR)/playbooks/bootstrap.yaml -SSH_USER ?= admin +# Inventory servers.yaml sets ansible_user. Override only when needed: +# make converge SSH_USER=tegwick +SSH_USER ?= +ANSIBLE_USER_FLAG := $(if $(SSH_USER),-u $(SSH_USER),) # Load your SOPS key for decryption when running playbooks (optional if you use keys.txt) export SOPS_AGE_KEY := $(shell cat ~/.config/sops/age/keys.txt 2>/dev/null) @@ -234,50 +237,66 @@ ansible-help: ## Show common Ansible commands @echo " make converge # run baseline convergence on all hosts" @echo " make converge-host HOST=web-01# run on a single host" @echo " make converge-tags TAGS=base # run only tagged tasks" + @echo " make converge-firewall HOST=Railiance01 # UFW only (RAIL-HO-WP-0009)" @echo " make converge-check # dry-run (check mode)" @echo " make converge-diff # show config diffs" + @echo " make verify-host HOST=Railiance01" + @echo " make goss-status # last on-host timer result" ansible-inventory: ## Print the dynamic inventory Ansible will use cd $(ANS_DIR) && ansible-inventory --list | head -200 ansible-ping: ## Quick connectivity check (SSH + Python availability) - cd $(ANS_DIR) && ansible all -u $(SSH_USER) -m ping + cd $(ANS_DIR) && ansible all $(ANSIBLE_USER_FLAG) -m ping status: ## Show live security state of all hosts (UFW, fail2ban, SSH hardening) @echo "=== Connectivity ===" - cd $(ANS_DIR) && ansible all -u $(SSH_USER) -m ping + cd $(ANS_DIR) && ansible all $(ANSIBLE_USER_FLAG) -m ping @echo "=== UFW ===" - cd $(ANS_DIR) && ansible all -u $(SSH_USER) -m shell -a "ufw status" --become + cd $(ANS_DIR) && ansible all $(ANSIBLE_USER_FLAG) -m shell -a "ufw status" --become @echo "=== fail2ban ===" - cd $(ANS_DIR) && ansible all -u $(SSH_USER) -m shell -a "systemctl is-active fail2ban" + cd $(ANS_DIR) && ansible all $(ANSIBLE_USER_FLAG) -m shell -a "systemctl is-active fail2ban" @echo "=== SSH hardening ===" - cd $(ANS_DIR) && ansible all -u $(SSH_USER) -m shell -a "grep -iE '^(PermitRootLogin|PasswordAuthentication)' /etc/ssh/sshd_config" --become + cd $(ANS_DIR) && ansible all $(ANSIBLE_USER_FLAG) -m shell -a "grep -iE '^(PermitRootLogin|PasswordAuthentication)' /etc/ssh/sshd_config" --become @echo "" @echo "--- Hint: run 'make verify' for a structured pass/fail report ---" verify: ## Run Goss test suite against all hosts, commit TAP reports — exits non-zero on failure @echo "Running Goss baseline assertions..." - @cd $(ANS_DIR) && ansible-playbook playbooks/verify.yaml -u $(SSH_USER) || \ + @cd $(ANS_DIR) && ansible-playbook playbooks/verify.yaml $(ANSIBLE_USER_FLAG) || \ (echo "One or more assertions FAILED — see reports/ for TAP output." && exit 1) @echo "All assertions passed." @git add reports/ && \ git diff --cached --quiet && echo "No new reports to commit." || \ git commit -m "chore: Goss verification reports $$(date -u +%Y-%m-%dT%H%M%SZ)" +verify-host: ## Run Goss against one host: make verify-host HOST=Railiance01 + @test -n "$(HOST)" || (echo "Usage: make verify-host HOST=Railiance01"; exit 1) + @echo "Running Goss baseline assertions on $(HOST)..." + @cd $(ANS_DIR) && ansible-playbook playbooks/verify.yaml $(ANSIBLE_USER_FLAG) -l $(HOST) || \ + (echo "One or more assertions FAILED — see reports/ for TAP output." && exit 1) + +goss-status: ## Fetch last on-host Goss timer result (fails if FAILED flag present) + cd $(ANS_DIR) && ansible-playbook playbooks/goss-status.yaml $(ANSIBLE_USER_FLAG) + converge: ## Converge all hosts to the baseline (idempotent) - cd $(ANS_DIR) && ansible-playbook $(PLAY) -u $(SSH_USER) + cd $(ANS_DIR) && ansible-playbook $(PLAY) $(ANSIBLE_USER_FLAG) converge-host: ## Converge a single host: make converge-host HOST=core-01 @test -n "$(HOST)" || (echo "Usage: make converge-host HOST="; exit 1) - cd $(ANS_DIR) && ansible-playbook $(PLAY) -u $(SSH_USER) -l $(HOST) + cd $(ANS_DIR) && ansible-playbook $(PLAY) $(ANSIBLE_USER_FLAG) -l $(HOST) converge-tags: ## Run only certain tags: make converge-tags TAGS="base,ufw" @test -n "$(TAGS)" || (echo "Usage: make converge-tags TAGS=tag1,tag2"; exit 1) - cd $(ANS_DIR) && ansible-playbook $(PLAY) -u $(SSH_USER) --tags "$(TAGS)" + cd $(ANS_DIR) && ansible-playbook $(PLAY) $(ANSIBLE_USER_FLAG) --tags "$(TAGS)" + +converge-firewall: ## Apply only UFW tasks: make converge-firewall HOST=Railiance01 + @test -n "$(HOST)" || (echo "Usage: make converge-firewall HOST=Railiance01"; exit 1) + cd $(ANS_DIR) && ansible-playbook $(PLAY) $(ANSIBLE_USER_FLAG) -l $(HOST) --tags firewall converge-check: ## Dry-run (no changes), great for previews - cd $(ANS_DIR) && ansible-playbook $(PLAY) -u $(SSH_USER) --check + cd $(ANS_DIR) && ansible-playbook $(PLAY) $(ANSIBLE_USER_FLAG) --check converge-diff: ## Show file/templating diffs while applying changes - cd $(ANS_DIR) && ansible-playbook $(PLAY) -u $(SSH_USER) --diff + cd $(ANS_DIR) && ansible-playbook $(PLAY) $(ANSIBLE_USER_FLAG) --diff diff --git a/SCOPE.md b/SCOPE.md index 8bffd39..8056f22 100644 --- a/SCOPE.md +++ b/SCOPE.md @@ -86,12 +86,10 @@ higher layer (Kubernetes, platform, etc.) can run. higher-layer and future reef planning. `railiance-hosts` is **superseded** by this repo and carries a banner saying so; its retirement is pending in `railiance-master` -- **Open security work**: `RAIL-HO-WP-0009` — the base role declared the k3s API - open to Anywhere while the live host was source-restricted by hand, so - converging would have exposed the Kubernetes API. The allowlist is now - declarative (`k3s_api_allowed_sources` / `k3s_api_revoked_sources`) but - **has not yet been converged**, so the live host still carries two stale - grants to rotated operator addresses +- **Open security work**: `RAIL-HO-WP-0009` — k3s API is declared tunnel-only + (ADR-005). Live `Railiance01` still has three public 6443 grants until an + operator-approved `make converge-firewall HOST=Railiance01`. CoulombCore + UFW stays unmanaged (`ufw_manage: false`) --- diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 07c4ba2..cb9dbee 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -29,7 +29,8 @@ | task | RAIL-HO-WP-0008-T02 | todo | — | workplans/RAIL-HO-WP-0008-railiance01-resource-and-commercial-evidence.md | | task | RAIL-HO-WP-0008-T03 | todo | — | workplans/RAIL-HO-WP-0008-railiance01-resource-and-commercial-evidence.md | | task | RAIL-HO-WP-0009-T01 | done | — | workplans/RAIL-HO-WP-0009-firewall-declared-state-and-api-exposure.md | -| task | RAIL-HO-WP-0009-T02 | progress | — | workplans/RAIL-HO-WP-0009-firewall-declared-state-and-api-exposure.md | -| task | RAIL-HO-WP-0009-T03 | todo | — | workplans/RAIL-HO-WP-0009-firewall-declared-state-and-api-exposure.md | -| task | RAIL-HO-WP-0009-T04 | todo | — | workplans/RAIL-HO-WP-0009-firewall-declared-state-and-api-exposure.md | -| task | RAIL-HO-WP-0009-T05 | todo | — | workplans/RAIL-HO-WP-0009-firewall-declared-state-and-api-exposure.md | +| task | RAIL-HO-WP-0009-T02 | wait | — | workplans/RAIL-HO-WP-0009-firewall-declared-state-and-api-exposure.md | +| task | RAIL-HO-WP-0009-T03 | done | — | workplans/RAIL-HO-WP-0009-firewall-declared-state-and-api-exposure.md | +| task | RAIL-HO-WP-0009-T04 | done | — | workplans/RAIL-HO-WP-0009-firewall-declared-state-and-api-exposure.md | +| task | RAIL-HO-WP-0009-T05 | done | — | workplans/RAIL-HO-WP-0009-firewall-declared-state-and-api-exposure.md | +| task | RAIL-HO-WP-0009-T06 | done | — | workplans/RAIL-HO-WP-0009-firewall-declared-state-and-api-exposure.md | diff --git a/ansible/inventory/group_vars/all.yaml b/ansible/inventory/group_vars/all.yaml index 09283b7..9fb29fe 100644 --- a/ansible/inventory/group_vars/all.yaml +++ b/ansible/inventory/group_vars/all.yaml @@ -26,29 +26,37 @@ ops_bridge_user: tegwick ops_bridge_pubkey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKQmXbAVlEa8dzGx8Hk2S7AITpz6sMWdCN0MeMOzL82u ops-bridge@custodian" # --------------------------------------------------------------------------- -# k3s API firewall allowlist (6443/tcp) +# k3s API firewall (6443/tcp) — tunnel only (ADR-005) # -# The k3s API is NOT world-reachable. Only these sources may connect. -# Added 2026-08-11 after discovering the live host was source-restricted by hand -# while this repo still declared 6443 open to Anywhere — meaning a convergence -# run would have exposed the Kubernetes API to the internet. +# Public allowlist is empty. Reach the API over ops-bridge: +# bridge up k3s-api-railiance01 # local 16444 +# bridge up k3s-api-coulombcore # local 16443 # -# These are operator egress addresses, not secrets. They are dynamic: when your -# ISP rotates the lease, MOVE the old entry to k3s_api_revoked_sources rather -# than deleting the line, so convergence prunes the stale grant instead of -# leaving it standing for whoever the address gets reassigned to. -# -# Durable alternative worth considering: reach the API over the ops-bridge SSH -# tunnel instead (see railiance-infra/docs/deploy-stack.md, the -# k3s-api-coulombcore pattern) and allow no public sources at all. -k3s_api_allowed_sources: - - address: "89.244.90.236" - comment: "k3s-api-operator-current" - - address: "89.244.90.255" - comment: "k3s-api-operator-current" +# Revoked addresses are pruned on a firewall-tagged converge so rotated or +# retired grants do not remain standing. Do not add new public sources here +# without amending ADR-005. +k3s_api_allowed_sources: [] k3s_api_revoked_sources: + - address: "89.244.90.248" + comment: "hand grant added 2026-08-12/15; retired by ADR-005" + - address: "89.244.90.236" + comment: "rotated ISP lease; retired by ADR-005" + - address: "89.244.90.255" + comment: "rotated ISP lease; retired by ADR-005" - address: "89.244.90.246" comment: "rotated ISP lease, superseded 2026-08-11" - address: "85.132.220.102" - comment: "historic operator address, retired" + comment: "historic operator address; also the 2026-08-15 workstation lease" + +# Single-node clusters need no public VXLAN grant. Set peer addresses here +# only when a second node must exchange Flannel frames (RAIL-BS-WP-0007). +flannel_vxlan_allowed_sources: [] + +# HostEurope Nydus agent — provider dashboard, root-password reset, backups. +# Required by the VPS platform (hosteurope/260308-dependency-nydus.md). +# Source-restricting it would break the provider; Anywhere is intentional. +ufw_extra_allowed: + - port: "2224" + proto: tcp + comment: "nydus-ex-api dashboard agent" diff --git a/ansible/inventory/host_vars/CoulombCore.yml b/ansible/inventory/host_vars/CoulombCore.yml index b2be451..2ed4c5e 100644 --- a/ansible/inventory/host_vars/CoulombCore.yml +++ b/ansible/inventory/host_vars/CoulombCore.yml @@ -1,6 +1,12 @@ # Host-specific variables for CoulombCore (92.205.130.254) # k3s single-node cluster host — HostEurope +# Do not enable or rewrite UFW on this host. Live filter is iptables INPUT +# DROP with a Plesk-era accept list (UFW status: inactive). Enabling UFW +# here would take down 80/443 and the rest of the accepted surface unless +# every live accept is declared first. RAIL-HO-WP-0009-T03. +ufw_manage: false + # Swapfile (T01) swap_size_gb: 4 swap_swappiness: 10 diff --git a/ansible/inventory_from_yaml.py b/ansible/inventory_from_yaml.py index bfc8976..f43101c 100755 --- a/ansible/inventory_from_yaml.py +++ b/ansible/inventory_from_yaml.py @@ -18,12 +18,24 @@ def load_tf_outputs(): return {} def load_host_vars(name): - """Load host_vars/.yml if it exists.""" + """Load host_vars/.yml if it exists. + + The inventory script is ansible/inventory_from_yaml.py. Ansible does not + auto-load a host_vars directory next to a script inventory, so this has + to emit hostvars itself. Look in ansible/inventory/host_vars first (where + CoulombCore.yml actually lives), then the unused repo-root path. + """ script_dir = os.path.dirname(__file__) - path = os.path.join(script_dir, '..', 'inventory', 'host_vars', f'{name}.yml') - if os.path.exists(path): - with open(path) as f: - return yaml.safe_load(f) or {} + candidates = [ + os.path.join(script_dir, 'inventory', 'host_vars', f'{name}.yml'), + os.path.join(script_dir, 'inventory', 'host_vars', f'{name}.yaml'), + os.path.join(script_dir, '..', 'inventory', 'host_vars', f'{name}.yml'), + os.path.join(script_dir, '..', 'inventory', 'host_vars', f'{name}.yaml'), + ] + for path in candidates: + if os.path.exists(path): + with open(path) as f: + return yaml.safe_load(f) or {} return {} def main(): diff --git a/ansible/playbooks/bootstrap.yaml b/ansible/playbooks/bootstrap.yaml index 627b1ee..8104bea 100644 --- a/ansible/playbooks/bootstrap.yaml +++ b/ansible/playbooks/bootstrap.yaml @@ -5,8 +5,13 @@ - ../inventory/group_vars/secrets.sops.yaml roles: - role: base + tags: [base] - role: sops_agent + tags: [sops] - role: custodian_agent # injects ~/.ssh/id_custodian_agent.pub into authorized_keys + tags: [custodian_agent] - role: swapfile # provisions swap file (size + swappiness from host_vars) + tags: [swap] - role: resource_limits # nproc PAM caps + systemd user slice memory limits + tags: [resource_limits] # - role: wireguard # enable if you configure WireGuard variables diff --git a/ansible/playbooks/goss-status.yaml b/ansible/playbooks/goss-status.yaml new file mode 100644 index 0000000..4c0f21f --- /dev/null +++ b/ansible/playbooks/goss-status.yaml @@ -0,0 +1,32 @@ +--- +# Fetch the last on-host Goss timer result. Does not run a new check. +# Usage: ansible-playbook ansible/playbooks/goss-status.yaml +# make goss-status + +- hosts: all + become: true + gather_facts: false + tasks: + - name: Read last Goss timer status + ansible.builtin.slurp: + src: /var/lib/railiance/goss/last.status + register: goss_status + failed_when: false + + - name: Show last Goss timer status + ansible.builtin.debug: + msg: >- + {{ inventory_hostname }}: + {{ goss_status.content | default('') | b64decode | trim + if goss_status.content is defined + else 'no timer result yet' }} + + - name: Fail when the last on-host check reported FAILED + ansible.builtin.stat: + path: /var/lib/railiance/goss/FAILED + register: goss_failed + + - name: Report failed hosts + ansible.builtin.fail: + msg: "Goss baseline last run failed on {{ inventory_hostname }}" + when: goss_failed.stat.exists | default(false) diff --git a/ansible/roles/base/defaults/main.yml b/ansible/roles/base/defaults/main.yml index 05c7f1b..1f0b7c0 100644 --- a/ansible/roles/base/defaults/main.yml +++ b/ansible/roles/base/defaults/main.yml @@ -1,6 +1,12 @@ --- # Base role defaults. +# When false, this role will not enable or rewrite UFW. Use that for hosts +# whose live packet filter is not UFW (CoulombCore: iptables INPUT DROP with +# a Plesk-era accept list). Enabling UFW there is an availability decision, +# not a side effect of an unrelated converge. +ufw_manage: true + # Source addresses permitted to reach the k3s API (6443/tcp). # # WHY THIS EXISTS @@ -10,21 +16,45 @@ # config WEAKER than reality: re-running this role would have removed the # restriction and exposed the Kubernetes API to the internet. Found 2026-08-11. # -# The allowlist is therefore declared here and converged, not hand-edited. +# Operator addresses rotate (dynamic ISP leases). An allowlist is a treadmill: +# each rotation is either an outage or a stale grant to whoever the ISP +# reassigns the address to. RAIL-HO-WP-0009-T04 therefore keeps this list +# empty. Reach the API over the ops-bridge SSH tunnel +# (`k3s-api-railiance01`, local port 16444; `k3s-api-coulombcore`, 16443). +# See docs/adr/ADR-005-k3s-api-tunnel-only.md. # # Deliberately empty by default. A host that sets no sources gets NO public # access to 6443 — which is the safe failure. SSH (22) is unaffected, so a host # converged with an empty list is always recoverable. # -# Set the real values in inventory/group_vars/all.yaml. Each entry: +# Each entry, if any: # - address: "203.0.113.10" # comment: "k3s-api-operator-workstation" k3s_api_allowed_sources: [] # Source addresses whose k3s API access must be REMOVED on convergence. -# -# Operator addresses rotate (dynamic ISP leases). Without this, every rotation -# leaves a standing grant to an address the ISP has since reassigned to someone -# else. Move an address here when it stops being yours; convergence then prunes -# it rather than leaving it to accumulate. +# Move an address here when it stops being yours (or when the public +# allowlist is retired); convergence then prunes it. k3s_api_revoked_sources: [] + +# Source addresses permitted to send Flannel VXLAN (8472/udp). +# +# Empty by default. A single-node cluster does not need a public VXLAN +# grant; adding an unrestricted 8472/udp allow would expose the pod network +# to injection. Set this to the other nodes' addresses only when the cluster +# becomes multi-node (RAIL-BS-WP-0007 / ThreePhoenix HA). +# +# Each entry: +# - address: "203.0.113.20" +# comment: "flannel-vxlan-peer" +flannel_vxlan_allowed_sources: [] + +# Extra UFW allows that are not k3s. Used for provider agents that must stay +# reachable (HostEurope Nydus on 2224/tcp). Empty by default so a Hetzner +# host does not inherit a HostEurope-only hole. +# +# Each entry: +# - port: "2224" +# proto: tcp +# comment: "nydus-ex-api dashboard agent" +ufw_extra_allowed: [] diff --git a/ansible/roles/base/handlers/main.yml b/ansible/roles/base/handlers/main.yml index c86a3ee..bfd5d2c 100644 --- a/ansible/roles/base/handlers/main.yml +++ b/ansible/roles/base/handlers/main.yml @@ -1,4 +1,9 @@ --- +- name: Restart sshd + ansible.builtin.service: + name: ssh + state: restarted + - name: Restart fail2ban ansible.builtin.service: name: fail2ban diff --git a/ansible/roles/base/tasks/main.yml b/ansible/roles/base/tasks/main.yml index 182d1c6..b777a6d 100644 --- a/ansible/roles/base/tasks/main.yml +++ b/ansible/roles/base/tasks/main.yml @@ -1,5 +1,6 @@ --- - name: Ensure base packages + tags: [base, packages] ansible.builtin.package: name: - apt-transport-https @@ -15,6 +16,7 @@ update_cache: true - name: Harden SSH + tags: [base, ssh] ansible.builtin.copy: dest: /etc/ssh/sshd_config.d/10-hardening.conf owner: root @@ -24,13 +26,10 @@ PasswordAuthentication no PermitRootLogin no PubkeyAuthentication yes - -- name: Restart sshd - ansible.builtin.service: - name: ssh - state: restarted + notify: Restart sshd - name: Ensure .ssh directory exists for ops_bridge_user + tags: [base, ssh] ansible.builtin.file: path: "/home/{{ ops_bridge_user | default('tegwick') }}/.ssh" state: directory @@ -39,6 +38,7 @@ mode: '0700' - name: Inject ops-bridge public key into authorized_keys + tags: [base, ssh] ansible.posix.authorized_key: user: "{{ ops_bridge_user | default('tegwick') }}" key: "{{ ops_bridge_pubkey }}" @@ -47,27 +47,46 @@ when: ops_bridge_pubkey is defined and ops_bridge_pubkey | length > 0 - name: Configure UFW default incoming policy + tags: [base, firewall, ufw] ansible.builtin.ufw: state: enabled policy: deny direction: incoming + when: ufw_manage | bool -- name: Allow UFW routing (required for k3s flannel pod networking) +- name: Allow UFW routing when VXLAN peers are declared + tags: [base, firewall, ufw] ansible.builtin.ufw: policy: allow direction: routed + when: ufw_manage | bool and (flannel_vxlan_allowed_sources | length > 0) - name: Allow SSH in UFW + tags: [base, firewall, ufw] ansible.builtin.ufw: rule: allow name: OpenSSH + when: ufw_manage | bool -# k3s API access is source-restricted. See roles/base/defaults/main.yml for why -# this is declared rather than hand-applied. Order matters below: grants are -# added BEFORE the blanket rule is removed, so convergence never opens a window -# in which the operator cannot reach the API. +- name: Allow declared extra UFW ports + tags: [base, firewall, ufw] + ansible.builtin.ufw: + rule: allow + port: "{{ item.port }}" + proto: "{{ item.proto | default('tcp') }}" + comment: "{{ item.comment | default('extra-allow') }}" + loop: "{{ ufw_extra_allowed }}" + loop_control: + label: "{{ item.port }}/{{ item.proto | default('tcp') }}" + when: ufw_manage | bool + +# k3s API access is source-restricted and empty by default (tunnel-only). +# See roles/base/defaults/main.yml and docs/adr/ADR-005-k3s-api-tunnel-only.md. +# Order matters: remaining grants (if any) are added BEFORE the blanket rule +# is removed, so a non-empty allowlist never opens a window without API access. - name: Allow k3s API from approved operator sources only + tags: [base, firewall, ufw] ansible.builtin.ufw: rule: allow port: '6443' @@ -77,15 +96,19 @@ loop: "{{ k3s_api_allowed_sources }}" loop_control: label: "{{ item.address }}" + when: ufw_manage | bool - name: Remove blanket k3s API rule if present (must not be world-reachable) + tags: [base, firewall, ufw] ansible.builtin.ufw: rule: allow port: '6443' proto: tcp delete: true + when: ufw_manage | bool - name: Revoke k3s API access for retired operator sources + tags: [base, firewall, ufw] ansible.builtin.ufw: rule: allow port: '6443' @@ -95,29 +118,49 @@ loop: "{{ k3s_api_revoked_sources }}" loop_control: label: "{{ item.address }}" + when: ufw_manage | bool - name: Warn when no operator source is allowed to reach the k3s API + tags: [base, firewall, ufw] ansible.builtin.debug: msg: >- k3s_api_allowed_sources is empty, so 6443/tcp is closed to all external - sources on this host. This is the safe default, not an error. SSH is - unaffected and the host remains recoverable. Set the allowlist in - inventory/group_vars/all.yaml to restore API access. - when: k3s_api_allowed_sources | length == 0 + sources on this host. Reach the API over the ops-bridge tunnel + (k3s-api-railiance01 on 16444, k3s-api-coulombcore on 16443). SSH is + unaffected and the host remains recoverable. + when: ufw_manage | bool and (k3s_api_allowed_sources | length == 0) -- name: Allow Flannel VXLAN in UFW +- name: Allow Flannel VXLAN from declared cluster peers only + tags: [base, firewall, ufw] ansible.builtin.ufw: rule: allow port: '8472' proto: udp + from_ip: "{{ item.address }}" + comment: "{{ item.comment | default('flannel-vxlan-peer') }}" + loop: "{{ flannel_vxlan_allowed_sources }}" + loop_control: + label: "{{ item.address }}" + when: ufw_manage | bool + +- name: Remove blanket Flannel VXLAN rule if present (must not be world-reachable) + tags: [base, firewall, ufw] + ansible.builtin.ufw: + rule: allow + port: '8472' + proto: udp + delete: true + when: ufw_manage | bool - name: Enable fail2ban + tags: [base, fail2ban] ansible.builtin.service: name: fail2ban state: started enabled: true - name: Configure fail2ban SSH jail + tags: [base, fail2ban] ansible.builtin.copy: dest: /etc/fail2ban/jail.d/sshd.conf owner: root @@ -134,6 +177,7 @@ notify: Restart fail2ban - name: Set HISTCONTROL to ignorespace + tags: [base, histcontrol] ansible.builtin.copy: dest: /etc/profile.d/histcontrol.sh owner: root @@ -143,5 +187,6 @@ export HISTCONTROL=ignorespace - name: Set timezone + tags: [base, timezone] community.general.timezone: name: "{{ timezone | default('UTC') }}" diff --git a/ansible/roles/goss/files/goss-baseline-check.sh b/ansible/roles/goss/files/goss-baseline-check.sh new file mode 100755 index 0000000..94a917c --- /dev/null +++ b/ansible/roles/goss/files/goss-baseline-check.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Recurring declared-vs-live check. Installed by the goss role. +# Writes TAP + a one-line status; notifies only on pass/fail transitions. +set -euo pipefail + +GOSS_BIN="${GOSS_BIN:-/usr/local/bin/goss}" +GOSS_FILE="${GOSS_FILE:-/etc/goss/baseline.yaml}" +STATE_DIR="${STATE_DIR:-/var/lib/railiance/goss}" +NOTIFY_URL="${RAILIANCE_GOSS_NOTIFY_URL:-}" +HOST="$(hostname -s)" + +mkdir -p "${STATE_DIR}" +chmod 0755 "${STATE_DIR}" + +if [[ ! -x "${GOSS_BIN}" || ! -f "${GOSS_FILE}" ]]; then + echo "skip: goss binary or baseline missing" | tee "${STATE_DIR}/last.status" + exit 0 +fi + +set +e +"${GOSS_BIN}" -g "${GOSS_FILE}" validate --format tap > "${STATE_DIR}/last.tap" +rc=$? +set -e + +if [[ "${rc}" -eq 0 ]]; then + result=pass +else + result=fail +fi + +printf 'result=%s host=%s ts=%s rc=%s\n' \ + "${result}" "${HOST}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${rc}" \ + > "${STATE_DIR}/last.status" + +prev="" +if [[ -f "${STATE_DIR}/last.result" ]]; then + prev="$(cat "${STATE_DIR}/last.result")" +fi +echo "${result}" > "${STATE_DIR}/last.result" + +logger -t railiance-goss "baseline ${result} on ${HOST} (rc=${rc})" + +if [[ "${result}" == "fail" ]]; then + touch "${STATE_DIR}/FAILED" +else + rm -f "${STATE_DIR}/FAILED" +fi + +if [[ -n "${NOTIFY_URL}" && "${result}" != "${prev}" ]]; then + payload=$(printf \ + '{"summary":"Goss baseline %s on %s","event_type":"note","author":"railiance-goss-timer"}' \ + "${result}" "${HOST}") + curl -sS -m 10 -X POST "${NOTIFY_URL}" \ + -H 'Content-Type: application/json' \ + -d "${payload}" >/dev/null || \ + logger -t railiance-goss "notify failed for ${HOST} ${result}" +fi + +exit "${rc}" diff --git a/ansible/roles/goss/handlers/main.yml b/ansible/roles/goss/handlers/main.yml new file mode 100644 index 0000000..16b0951 --- /dev/null +++ b/ansible/roles/goss/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd daemon + ansible.builtin.systemd: + daemon_reload: true diff --git a/ansible/roles/goss/tasks/main.yml b/ansible/roles/goss/tasks/main.yml index cf62d7b..1e61d29 100644 --- a/ansible/roles/goss/tasks/main.yml +++ b/ansible/roles/goss/tasks/main.yml @@ -32,6 +32,39 @@ group: root mode: "0644" +- name: Install recurring Goss check wrapper + ansible.builtin.copy: + src: goss-baseline-check.sh + dest: /usr/local/sbin/goss-baseline-check + owner: root + group: root + mode: "0755" + +- name: Install Goss baseline systemd service + ansible.builtin.template: + src: railiance-goss-baseline.service.j2 + dest: /etc/systemd/system/railiance-goss-baseline.service + owner: root + group: root + mode: "0644" + notify: Reload systemd daemon + +- name: Install Goss baseline systemd timer + ansible.builtin.template: + src: railiance-goss-baseline.timer.j2 + dest: /etc/systemd/system/railiance-goss-baseline.timer + owner: root + group: root + mode: "0644" + notify: Reload systemd daemon + +- name: Enable hourly Goss baseline timer + ansible.builtin.systemd: + name: railiance-goss-baseline.timer + enabled: true + state: started + daemon_reload: true + - name: Run Goss assertions (TAP output) ansible.builtin.command: cmd: "{{ goss_bin }} -g {{ goss_dir }}/baseline.yaml validate --format tap" diff --git a/ansible/roles/goss/templates/railiance-goss-baseline.service.j2 b/ansible/roles/goss/templates/railiance-goss-baseline.service.j2 new file mode 100644 index 0000000..78a208d --- /dev/null +++ b/ansible/roles/goss/templates/railiance-goss-baseline.service.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Railiance declared-vs-live Goss baseline +Documentation=file:///etc/goss/baseline.yaml + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/goss-baseline-check +Nice=10 +# The wrapper records TAP even when assertions fail. +SuccessExitStatus=0 1 diff --git a/ansible/roles/goss/templates/railiance-goss-baseline.timer.j2 b/ansible/roles/goss/templates/railiance-goss-baseline.timer.j2 new file mode 100644 index 0000000..66b6d58 --- /dev/null +++ b/ansible/roles/goss/templates/railiance-goss-baseline.timer.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=Hourly Railiance Goss baseline (RAIL-HO-WP-0009-T05) + +[Timer] +OnBootSec=5min +OnUnitActiveSec=1h +RandomizedDelaySec=5min +Persistent=true +Unit=railiance-goss-baseline.service + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/resource_limits/tasks/main.yml b/ansible/roles/resource_limits/tasks/main.yml index 432f60d..deb9275 100644 --- a/ansible/roles/resource_limits/tasks/main.yml +++ b/ansible/roles/resource_limits/tasks/main.yml @@ -10,6 +10,7 @@ # user_memory_swap_max: systemd MemorySwapMax (default: 512M) - name: Set PAM nproc limits + tags: [resource_limits] ansible.builtin.template: src: nproc-limits.conf.j2 dest: /etc/security/limits.d/60-nproc-{{ resource_limit_user | default('tegwick') }}.conf @@ -18,6 +19,7 @@ mode: '0644' - name: Ensure systemd user slice override directory + tags: [resource_limits] ansible.builtin.file: path: "/etc/systemd/system/user-{{ resource_limit_uid | default(1000) }}.slice.d" state: directory @@ -26,6 +28,7 @@ mode: '0755' - name: Set systemd user slice memory limits + tags: [resource_limits] ansible.builtin.template: src: user-slice-limits.conf.j2 dest: "/etc/systemd/system/user-{{ resource_limit_uid | default(1000) }}.slice.d/limits.conf" diff --git a/ansible/roles/sops_agent/tasks/main.yml b/ansible/roles/sops_agent/tasks/main.yml index 7addb34..0a3f839 100644 --- a/ansible/roles/sops_agent/tasks/main.yml +++ b/ansible/roles/sops_agent/tasks/main.yml @@ -1,5 +1,6 @@ --- - name: Install age + tags: [sops] ansible.builtin.shell: | set -euo pipefail if ! command -v age >/dev/null; then @@ -9,12 +10,14 @@ executable: /bin/bash - name: Install sops + tags: [sops] ansible.builtin.get_url: url: https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 dest: /usr/local/bin/sops mode: '0755' - name: Create SOPS age dir + tags: [sops] ansible.builtin.file: path: /root/.config/sops/age state: directory diff --git a/ansible/roles/swapfile/tasks/main.yml b/ansible/roles/swapfile/tasks/main.yml index 730fbac..23a9157 100644 --- a/ansible/roles/swapfile/tasks/main.yml +++ b/ansible/roles/swapfile/tasks/main.yml @@ -6,17 +6,20 @@ # swap_swappiness: vm.swappiness value (default: 10) - name: Check if swapfile exists with correct size + tags: [swap] ansible.builtin.stat: path: /swapfile register: swapfile_stat - name: Allocate swapfile (fallocate) + tags: [swap] ansible.builtin.command: cmd: "fallocate -l {{ (swap_size_gb | default(4)) | int }}G /swapfile" creates: /swapfile when: not swapfile_stat.stat.exists - name: Set swapfile permissions + tags: [swap] ansible.builtin.file: path: /swapfile owner: root @@ -24,17 +27,20 @@ mode: '0600' - name: Format swapfile + tags: [swap] ansible.builtin.command: cmd: mkswap /swapfile when: not swapfile_stat.stat.exists - name: Enable swapfile + tags: [swap] ansible.builtin.command: cmd: swapon /swapfile when: not swapfile_stat.stat.exists ignore_errors: true # already active is not an error - name: Ensure swapfile in /etc/fstab + tags: [swap] ansible.builtin.lineinfile: path: /etc/fstab regexp: '^/swapfile' @@ -42,6 +48,7 @@ state: present - name: Set vm.swappiness at runtime + tags: [swap] ansible.posix.sysctl: name: vm.swappiness value: "{{ swap_swappiness | default(10) }}" @@ -49,6 +56,7 @@ reload: true - name: Persist vm.swappiness across reboots + tags: [swap] ansible.builtin.copy: dest: /etc/sysctl.d/60-swappiness.conf owner: root diff --git a/docs/adr/ADR-005-k3s-api-tunnel-only.md b/docs/adr/ADR-005-k3s-api-tunnel-only.md new file mode 100644 index 0000000..dba2b23 --- /dev/null +++ b/docs/adr/ADR-005-k3s-api-tunnel-only.md @@ -0,0 +1,58 @@ +# ADR-005 — k3s API is tunnel-only + +**Status:** Accepted +**Date:** 2026-08-15 +**Deciders:** implementation of RAIL-HO-WP-0009-T04 +**Workplans:** `RAIL-HO-WP-0009` + +--- + +## Context + +Operator addresses on this network rotate with the ISP lease. A public UFW +allowlist for `6443/tcp` is therefore a treadmill: + +- miss a rotation and `kubectl` breaks mid-session +- leave the old grant standing and it becomes a grant to a stranger +- hand-add the new address and the declaration drifts again + +That cycle produced this workplan. The live allowlist drifted by hand *during +the session that was fixing allowlist drift*, and again before the next +session (`89.244.90.248` appeared undeclared). On 2026-08-15 the workstation +egress address was `85.132.220.102` — already on the revoked list as a +"historic" operator address. + +`docs/deploy-stack.md` already documents API access over the ops-bridge SSH +tunnel for CoulombCore (`k3s-api-coulombcore`, local port 16443). The same +tunnel already exists for Railiance01 (`k3s-api-railiance01`, local port +16444). SSH itself stays public, so the host remains recoverable. + +## Decision + +The public k3s API allowlist is **empty**. Operator and agent `kubectl` access +uses the ops-bridge tunnels: + +| Cluster | Tunnel | Local port | Remote | +|-------------|-------------------------|------------|--------| +| CoulombCore | `k3s-api-coulombcore` | 16443 | 6443 | +| Railiance01 | `k3s-api-railiance01` | 16444 | 6443 | + +```bash +bridge up k3s-api-railiance01 +# kubeconfig server: https://127.0.0.1:16444 +``` + +Trade: every operator `kubectl` action depends on ops-bridge. That is accepted. +A rotating public allowlist is the worse dependency. + +Emergency break-glass remains SSH: `ssh railiance01 -- sudo k3s kubectl …`. +Do not re-open `6443/tcp` to Anywhere. + +## Consequences + +- `k3s_api_allowed_sources` stays `[]`. +- Former public grants live in `k3s_api_revoked_sources` so a firewall-tagged + converge deletes them. +- Goss asserts the 6443 allowlist size is exactly the declared length (zero) + and that no revoked address remains. +- Amending this ADR is required before adding any new public 6443 source. diff --git a/docs/conformance-loop.md b/docs/conformance-loop.md new file mode 100644 index 0000000..50f728f --- /dev/null +++ b/docs/conformance-loop.md @@ -0,0 +1,61 @@ +# Declared-vs-live conformance loop + +This is the S1 slice of the unowned **Q7 Governance and Change Management** +gap recorded in `railiance-platform/ArchitectureBlueprint.md` §5.3. The +family owner of the loop is **`railiance-master`**. This repo owns the host +check and the first reaction path; it does not own rapp/rail/reef schema +conformance (that is 10c, blocked on the family declaration validator). + +## What runs + +| Piece | Where | Cadence | +| --- | --- | --- | +| Rendered Goss baseline | `/etc/goss/baseline.yaml` from `goss/baseline.yaml.j2` | every `make verify` | +| On-host timer | `railiance-goss-baseline.timer` | hourly, plus 5 minutes after boot | +| Wrapper | `/usr/local/sbin/goss-baseline-check` | writes `/var/lib/railiance/goss/` | +| Operator collect | `make goss-status` | on demand | +| Operator run | `make verify` / `make verify-host HOST=…` | on demand; commits TAP to `reports/` | + +The firewall assertions are generated from the same inventory lists that +converge UFW (`k3s_api_allowed_sources`, `k3s_api_revoked_sources`, +`flannel_vxlan_allowed_sources`, `ufw_extra_allowed`). They assert: + +- each declared source is present +- the **count** of non-Anywhere 6443/8472 allows equals the declared length + (extra hand grants fail) +- no `ALLOW Anywhere` on 6443 or 8472 +- every revoked 6443 source is gone + +That is the check that would have caught `89.244.90.248` appearing on +Railiance01 after T01. + +## Where failures go + +1. **journald** — `logger -t railiance-goss` +2. **on-host flag** — `/var/lib/railiance/goss/FAILED` and `last.status` +3. **optional State Hub POST** — set `RAILIANCE_GOSS_NOTIFY_URL` on the host + to a State Hub `/progress/` endpoint. The wrapper notifies only on a + pass↔fail transition, so a known-fail host (CoulombCore, UFW inactive) + does not spam. +4. **git TAP** — `make verify` still commits `reports/goss--.tap` + +`make goss-status` fails the play if any host still has the `FAILED` flag. + +## Routing to Q7 + +`railiance-master` owns the conformance loop as a family capability +(ArchitectureBlueprint §5.3; RAILIANCE-WP-0016 item 10b). This file is the +handoff: + +- S1 check: **exists and is scheduled** in this repo +- S1 reaction: journald + flag + optional progress event +- Family reaction (page a human, open a task, block a rollout): **not here** + +Do not build a second alert path in `railiance-telemetry` for the same +"check failed → someone sees it" plumbing. Item 9b should reuse this. + +## Known expected fail + +`CoulombCore` has UFW inactive. The baseline asserts `Status: active`. The +timer will fail there until an explicit decision enables UFW or the host is +removed from the verify inventory. That failure is evidence, not noise. diff --git a/docs/convergence.md b/docs/convergence.md index 008592c..41c11fc 100644 --- a/docs/convergence.md +++ b/docs/convergence.md @@ -9,7 +9,9 @@ This process is called **convergence**. When you run `make converge`, Ansible connects to all declared hosts and applies baseline roles: - **User setup** → ensures the `admin` user exists with your SSH key and passwordless sudo -- **Firewall** → configures `ufw` with sensible defaults (deny incoming, allow SSH) +- **Firewall** → configures `ufw` with deny incoming, SSH, declared extras + (Nydus), and a tunnel-only k3s API (ADR-005). Use + `make converge-firewall HOST=Railiance01` to apply only those tasks. - **Hardening** → basic SSH daemon hardening, disable root login, disable password auth - **Tooling** → installs essential packages (htop, vim, git, curl, fail2ban, etc.) - **SOPS agent** → ensures decryption tooling (`age`, `sops`) is available on the host diff --git a/docs/deploy-stack.md b/docs/deploy-stack.md index 2d808ae..96803d5 100644 --- a/docs/deploy-stack.md +++ b/docs/deploy-stack.md @@ -120,7 +120,8 @@ After S2 is up, establish the persistent tunnels from the workstation: ```bash bridge up state-hub-coulombcore # state-hub HTTP (port 18000 remote) bridge up state-hub-mcp-coulombcore # state-hub MCP (port 18001 remote) -bridge up k3s-api-coulombcore # k3s API (port 16443 local) +bridge up k3s-api-coulombcore # CoulombCore k3s API (port 16443 local) +bridge up k3s-api-railiance01 # Railiance01 k3s API (port 16444 local) ``` Verify: `bridge status` shows all three connected. diff --git a/docs/evidence/RAIL-HO-WP-0009-T03-declared-vs-live-audit.md b/docs/evidence/RAIL-HO-WP-0009-T03-declared-vs-live-audit.md new file mode 100644 index 0000000..41b5b9b --- /dev/null +++ b/docs/evidence/RAIL-HO-WP-0009-T03-declared-vs-live-audit.md @@ -0,0 +1,141 @@ +# RAIL-HO-WP-0009-T03 — declared-vs-live audit + +Audited 2026-08-15 from the workstation against `Railiance01` +(`92.205.62.239`) and `CoulombCore` (`92.205.130.254`). SSH as `tegwick`. +This is a record of what was found; absence of drift is recorded too. + +## Method + +Compared `ansible/roles/base/tasks/main.yml`, sibling bootstrap roles, and +`spec/server-baseline.yaml` to live `ufw status`, `sshd -T`, fail2ban, +sudoers, listening sockets, systemd user-slice, nproc, swappiness, and +timezone. + +## Matches declaration + +| Surface | Railiance01 | CoulombCore | +| --- | --- | --- | +| sshd drop-in `10-hardening.conf` | present; root/password off, pubkey on | same | +| `sshd -T` effective | matches drop-in | matches drop-in | +| fail2ban `sshd` jail | active | active | +| `/etc/profile.d/histcontrol.sh` | `ignorespace` | `ignorespace` | +| sudoers `tegwick` NOPASSWD | present | present | +| OpenSSH UFW | ALLOW Anywhere | UFW inactive (see below) | + +## Divergences + +### 1. k3s API allowlist (Railiance01) — same defect class as T01 + +Live 6443 grants on 2026-08-15: + +| From | Declared? | +| --- | --- | +| `89.244.90.236` | was declared; now revoked by ADR-005 | +| `89.244.90.255` | was declared; now revoked by ADR-005 | +| `89.244.90.248` | **never declared** — third hand grant | + +No blanket `Anywhere` rule. Stale grants from T02 (`.246`, `.102`) are gone. + +`.248` appeared after the 2026-08-12 session that was already fixing +allowlist drift. Goss as of T01 would not have failed: it asserted declared +sources present, not that the live set equalled the declared set. + +Workstation egress on this audit was `85.132.220.102` — already on the +revoked list. That is the allowlist treadmill in one line. + +### 2. Flannel VXLAN declaration weaker than needed (T06) + +`8472/udp` listens on `0.0.0.0` on both hosts. Railiance01 has **no** UFW +rule for it (default deny incoming covers it). The role used to declare +`8472/udp` ALLOW with no source restriction, so a converge would have +*opened* VXLAN to the internet. Fixed: empty `flannel_vxlan_allowed_sources`, +blanket rule deleted if present. + +### 3. Port 2224/tcp Anywhere — intended, now declared + +Both hosts run `nydus-ex-api` on `0.0.0.0:2224`. Railiance01 UFW allows it +Anywhere with comment `nydus-ex-api dashboard agent`. This is the HostEurope +Nydus agent (dashboard, root-password reset, backups); see +`hosteurope/260308-dependency-nydus.md`. Removing it would break the +provider. Declared as `ufw_extra_allowed`. Source-restricting it is not +possible without HostEurope's egress list. + +### 4. CoulombCore: UFW inactive, but not "no firewall" + +`ufw status` → `Status: inactive`. Live filter is `iptables` `INPUT DROP` +with a Plesk-era accept list: 22, 80, 443, 2224, mail, 3306, 5432, 53, +445/139, and **tcp 49152-65535**. 6443 and 8472 are **not** in that accept +list, so they are dropped even with UFW off. + +Converging the base role would enable UFW, allow only SSH + (now empty) k3s ++ nydus, and take down 80/443 and the rest of the accepted surface. +`ufw_manage: false` on CoulombCore prevents that. Enabling UFW there is its +own decision and needs a declared accept list first. + +### 5. Resource limits and swap — applied on CoulombCore, never on Railiance01 + +| | Railiance01 | CoulombCore | +| --- | --- | --- | +| `user-1000.slice` MemoryMax | absent | `1500M` / swap `512M` (Ansible) | +| PAM nproc | no `limits.d` files | `60-nproc-tegwick.conf` | +| swappiness | 60 (kernel default) | 10 | +| timezone | `Etc/UTC` | `Etc/UTC` | + +The user-slice cap can OOM agent workloads (T02). Do **not** apply it to +Railiance01 without a separate decision. Role tags now isolate it +(`resource_limits`, `swap`, `timezone`). + +`inventory/group_vars/all.yaml` sets `timezone: Europe/Berlin` but that file +is not loaded by the Ansible inventory script. The role default is UTC, which +matches live. Leave it; do not start loading that file as a side effect. + +### 6. Inventory host_vars path bug + +`ansible/inventory_from_yaml.py` loaded `inventory/host_vars/.yml` +(does not exist). The real file is `ansible/inventory/host_vars/CoulombCore.yml`. +Ansible does not auto-load host_vars next to a script inventory, so +`ufw_manage`, swap, and resource-limit overrides would not reach the play. +Fixed: the script now reads `ansible/inventory/host_vars/` first. + +### 7. sshd restart was an unconditioned task + +`Restart sshd` ran on every converge, which is why `--check` reported an +sshd restart among the 11 changes. It is now a handler notified only when +the hardening drop-in changes. + +### 8. Undeclared listening sockets (not opened in UFW) + +On Railiance01, not in this role, UFW default-deny so not currently public: + +- `*:6443` k3s-server (the API this workplan closes) +- `92.205.62.239:2379` / `:2380` etcd on the public address — S2 concern +- `*:10250` kubelet — S2 concern +- `0.0.0.0:18081` Python reverse proxy to `127.0.0.1:18765` (issue-core), + started 2026-07-26. No UFW rule. Out of scope here; record only. + +On CoulombCore, `*:6443` and `0.0.0.0:8472` listen; iptables INPUT DROP +without an accept for those ports is what currently protects them, not UFW. + +### 9. Routed policy + +Railiance01 UFW default routed is **deny**. The role used to set routed +**allow** for Flannel. Routed allow is now applied only when +`flannel_vxlan_allowed_sources` is non-empty (multi-node). Single-node +converge will not flip it. + +## Decisions taken in this task + +- Declare Nydus 2224; do not remove it. +- Do not enable UFW on CoulombCore (`ufw_manage: false`). +- Do not apply Railiance01 resource limits / swappiness / timezone as part + of this workplan. +- Tag the base role and sibling bootstrap roles so firewall can be scoped. +- Fix the host_vars loader. + +## Still operator decisions (not this workplan) + +- Enable UFW on CoulombCore, with a declared accept list that includes 80/443. +- Apply or drop the declared user-slice memory cap on Railiance01. +- Whether etcd/kubelet listening on the public address should be rebound + (`railiance-cluster`). +- What the `0.0.0.0:18081` issue-core proxy is for, and whether it stays. diff --git a/docs/k3s-api-access.md b/docs/k3s-api-access.md new file mode 100644 index 0000000..41fa29d --- /dev/null +++ b/docs/k3s-api-access.md @@ -0,0 +1,48 @@ +# k3s API access + +The Kubernetes API is not on the public internet. See +[ADR-005](adr/ADR-005-k3s-api-tunnel-only.md). + +## Day-to-day (workstation) + +```bash +bridge up k3s-api-railiance01 # https://127.0.0.1:16444 → Railiance01 :6443 +bridge up k3s-api-coulombcore # https://127.0.0.1:16443 → CoulombCore :6443 +bridge status +``` + +Point the kubeconfig `server` at the matching local port. The default +`~/.kube/config` and `~/.kube/config-hosteurope` currently both target +`https://127.0.0.1:16443` (CoulombCore). For Railiance01 use 16444. + +```bash +# Railiance01 +KUBECONFIG=~/.kube/config-railiance01 kubectl get nodes +# or, once: +# kubectl --kubeconfig ~/.kube/config-railiance01 config set-cluster default --server=https://127.0.0.1:16444 +``` + +SSH is unaffected. If the tunnel is down, recover over SSH: + +```bash +ssh tegwick@92.205.62.239 -- sudo k3s kubectl get nodes +``` + +## What not to do + +- Do not `ufw allow 6443/tcp` from the current public IP. +- Do not add addresses to `k3s_api_allowed_sources` without amending ADR-005. +- When an old grant must die, put it in `k3s_api_revoked_sources` and run + `make converge-firewall HOST=Railiance01`. + +## Converge the firewall only + +The base role is tagged. Firewall changes no longer drag sshd restarts or +user-slice memory caps with them: + +```bash +make converge-firewall HOST=Railiance01 +``` + +CoulombCore sets `ufw_manage: false`. That host's live filter is iptables, not +UFW; this target will skip UFW tasks there. diff --git a/docs/verification.md b/docs/verification.md index cb36ad0..c9ec24d 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -10,11 +10,13 @@ a reproducible, CI-friendly pass/fail verdict. of every managed node. It covers: - **Firewall** — UFW active, default deny inbound, required ports allowed - (SSH 22/tcp, Flannel VXLAN 8472/udp). The k3s API (6443/tcp) is - **source-restricted**, not world-open: only addresses in - `k3s_api_allowed_sources` may reach it, and `k3s_api_revoked_sources` is - pruned on convergence. A host with an empty allowlist has 6443 closed to all - external sources — the safe failure, recoverable over SSH. + (SSH 22/tcp; HostEurope Nydus 2224/tcp). The k3s API (6443/tcp) is + **tunnel-only** (ADR-005): `k3s_api_allowed_sources` is empty, and + `k3s_api_revoked_sources` is pruned on a firewall-tagged converge. Flannel + VXLAN (8472/udp) is omitted while the cluster is single-node; peer + addresses go in `flannel_vxlan_allowed_sources` when a second node appears. + Hosts with `ufw_manage: false` (CoulombCore) are not rewritten. A host with + an empty 6443 allowlist is recoverable over SSH. - **SSH daemon** — root login disabled, password auth disabled, pubkey auth enabled - **Services** — ufw, fail2ban, ssh.socket enabled and running - **Packages** — ufw, fail2ban, git, curl, vim, htop (age and sops installed as binaries) @@ -56,7 +58,8 @@ you need a structured, automatable check. ## Goss test file -`goss/baseline.yaml` contains one Goss assertion per spec item. The mapping is: +`goss/baseline.yaml.j2` is rendered per host from the same inventory lists +that converge UFW. The mapping is: | spec section | Goss resource | |---|---| @@ -73,8 +76,11 @@ you need a structured, automatable check. 1. Add the desired state to `spec/server-baseline.yaml` 2. Add the Ansible task to `ansible/roles/base/tasks/main.yml` -3. Add the Goss assertion to `goss/baseline.yaml` -4. Run `make converge && make verify` to confirm +3. Add the Goss assertion to `goss/baseline.yaml.j2` +4. Run `make converge-firewall` and `make verify-host HOST=…` to confirm + +An hourly on-host timer (`railiance-goss-baseline.timer`) reruns the last +rendered baseline. See `docs/conformance-loop.md`. ## Reports diff --git a/goss/baseline.yaml.j2 b/goss/baseline.yaml.j2 index 3ad29f9..a1833a5 100644 --- a/goss/baseline.yaml.j2 +++ b/goss/baseline.yaml.j2 @@ -4,9 +4,10 @@ # # THIS IS A TEMPLATE. It is rendered per host from inventory variables, so the # firewall assertions below are generated from the SAME declaration that -# converges the host (k3s_api_allowed_sources in group_vars/all.yaml). That is -# deliberate: a hand-written assertion drifts from the declaration it is meant -# to check, which is exactly how RAIL-HO-WP-0009 happened. +# converges the host (k3s_api_allowed_sources, flannel_vxlan_allowed_sources, +# ufw_extra_allowed in group_vars). That is deliberate: a hand-written +# assertion drifts from the declaration it is meant to check, which is exactly +# how RAIL-HO-WP-0009 happened. package: ufw: @@ -60,16 +61,33 @@ command: {% for src in k3s_api_allowed_sources | default([]) %} - '/6443\/tcp\s+ALLOW\s+{{ src.address | regex_escape }}/' {% endfor %} - - /8472\/udp.*ALLOW/ +{% for src in flannel_vxlan_allowed_sources | default([]) %} + - '/8472\/udp\s+ALLOW\s+{{ src.address | regex_escape }}/' +{% endfor %} +{% for rule in ufw_extra_allowed | default([]) %} + - '/{{ rule.port }}\/{{ rule.proto | default("tcp") }}\s+ALLOW/' +{% endfor %} - # The k3s API must never be reachable from Anywhere. The previous assertion - # matched /6443\/tcp.*ALLOW/, which passes identically whether the API is - # restricted to one operator address or open to the entire internet — it - # asserted that the port was allowed, not from whom. + # Exact allowlist size: extra hand grants must fail, not only missing ones. + # The previous assertion matched /6443\/tcp.*ALLOW/, which passes identically + # whether the API is restricted to one operator address or open to the entire + # internet — it asserted that the port was allowed, not from whom. "ufw status | grep -Ec '6443/tcp[[:space:]]+ALLOW[[:space:]]+Anywhere' || true": exit-status: 0 stdout: - "0" + "ufw status | grep -E '6443/tcp[[:space:]]+ALLOW' | grep -vc 'Anywhere' || true": + exit-status: 0 + stdout: + - "{{ k3s_api_allowed_sources | default([]) | length }}" + "ufw status | grep -Ec '8472/udp[[:space:]]+ALLOW[[:space:]]+Anywhere' || true": + exit-status: 0 + stdout: + - "0" + "ufw status | grep -E '8472/udp[[:space:]]+ALLOW' | grep -vc 'Anywhere' || true": + exit-status: 0 + stdout: + - "{{ flannel_vxlan_allowed_sources | default([]) | length }}" {% for src in k3s_api_revoked_sources | default([]) %} # Revoked operator source must not retain access: {{ src.comment | default('') }} diff --git a/spec/server-baseline.yaml b/spec/server-baseline.yaml index 57cac72..9a45d03 100644 --- a/spec/server-baseline.yaml +++ b/spec/server-baseline.yaml @@ -6,7 +6,7 @@ # When you change something here, update the Ansible roles AND the Goss tests. # Format: human-readable YAML, kept technology-neutral. -version: "1.0" +version: "1.1" applies_to: all # override per node group if needed # --------------------------------------------------------------------------- @@ -17,6 +17,8 @@ firewall: status: active default_incoming: deny default_outgoing: allow + # Hosts with ufw_manage: false (CoulombCore) keep their existing packet + # filter. Do not enable UFW there as a side effect of an unrelated converge. rules: - name: OpenSSH # UFW app name; resolves to 22/tcp action: allow @@ -24,10 +26,17 @@ firewall: port: 6443 proto: tcp action: allow + sources: [] # tunnel-only (ADR-005); public allowlist stays empty - name: flannel-vxlan port: 8472 proto: udp action: allow + sources: [] # omit while single-node; peer addresses only when multi-node + - name: nydus-ex-api + port: 2224 + proto: tcp + action: allow + sources: anywhere # HostEurope provider agent; required by the VPS platform # --------------------------------------------------------------------------- # SSH daemon diff --git a/workplans/RAIL-HO-WP-0009-firewall-declared-state-and-api-exposure.md b/workplans/RAIL-HO-WP-0009-firewall-declared-state-and-api-exposure.md index 1e90164..84457cb 100644 --- a/workplans/RAIL-HO-WP-0009-firewall-declared-state-and-api-exposure.md +++ b/workplans/RAIL-HO-WP-0009-firewall-declared-state-and-api-exposure.md @@ -8,7 +8,7 @@ status: active owner: codex topic_slug: railiance created: "2026-08-12" -updated: "2026-08-12" +updated: "2026-08-15" related_repos: - railiance-cluster - railiance-platform @@ -98,7 +98,7 @@ session (`89.244.90.236` granted; `kubectl` verified, node Ready v1.35.1+k3s1). ```task id: RAIL-HO-WP-0009-T02 -status: progress +status: wait priority: high state_hub_task_id: "7d91dfc2-481b-4943-90e9-bb1b814a23dc" ``` @@ -141,9 +141,22 @@ That is worth recording as evidence rather than as a footnote: the allowlist drifted again, by hand, *during the very session that was fixing allowlist drift*. It is the strongest available argument for T04. +**Progress 2026-08-15.** Tags now exist (`make converge-firewall HOST=Railiance01`). +Live 6443 grants on this date: `.236`, `.255`, and an undeclared `.248`. T04 +emptied the declared allowlist, so this task's remaining work is the +operator-approved prune of those three grants. SSH stays; kubectl goes via +`k3s-api-railiance01` (`:16444`). CoulombCore is not touched +(`ufw_manage: false`). + +Waiting on operator approval. Command: + +```bash +make converge-firewall HOST=Railiance01 +``` + ```task id: RAIL-HO-WP-0009-T03 -status: todo +status: done priority: high state_hub_task_id: "3e835b96-736c-458c-91eb-04bfd4c7e0e7" ``` @@ -179,9 +192,24 @@ ever applied — an sshd restart, `MemoryMax=1500M` / `MemorySwapMax=512M` on cap in particular could OOM running agent workloads. The base role has **no tags**, so convergence cannot be scoped. Add tags as part of this task. +**Done 2026-08-15.** Full audit: +`docs/evidence/RAIL-HO-WP-0009-T03-declared-vs-live-audit.md`. + +- sshd / fail2ban / HISTCONTROL / sudoers match on both hosts. +- 2224/tcp is HostEurope Nydus; declared as `ufw_extra_allowed`. +- CoulombCore UFW is inactive; live filter is iptables INPUT DROP (6443/8472 + not accepted). Set `ufw_manage: false` so a converge cannot enable UFW there. +- Base role and sibling bootstrap roles now have tags (`firewall`, `ssh`, + `packages`, `fail2ban`, `swap`, `resource_limits`, `sops`). sshd restart is + a handler. Routed-allow applies only when VXLAN peers are declared. +- Inventory script now loads `ansible/inventory/host_vars/` (it was looking at + a path that does not exist). +- Not in this workplan: Railiance01 resource limits, CoulombCore UFW enable, + etcd/kubelet bind addresses, the `0.0.0.0:18081` issue-core proxy. + ```task id: RAIL-HO-WP-0009-T04 -status: todo +status: done priority: medium state_hub_task_id: "908630e8-e245-47f5-a060-3949250f522c" ``` @@ -195,10 +223,18 @@ the same pattern to `railiance01` and reduce the public allowlist to nothing. Decide explicitly rather than by default: this trades convenience for exposure, and the tunnel becomes a dependency of every operator action. +**Done 2026-08-15.** Decision recorded in +`docs/adr/ADR-005-k3s-api-tunnel-only.md`. `k3s_api_allowed_sources` is empty; +former public grants (including the undeclared `.248` and the 2026-08-15 +workstation lease `.102`) are in `k3s_api_revoked_sources`. Operator access +is `bridge up k3s-api-railiance01` (local 16444). Live prune of the three +remaining grants is T02. + ```task id: RAIL-HO-WP-0009-T06 -status: todo +status: done priority: high +state_hub_task_id: "764adf9e-cb60-4edd-86bf-f0c170e65cd2" ``` **Flannel VXLAN is declared open to Anywhere.** Found 2026-08-12 by the newly @@ -219,9 +255,15 @@ Note the sequencing risk: `RAIL-BS-WP-0007` (ThreePhoenix HA, multi-node) will need working VXLAN between nodes. Resolve this before that lands, or the fix will be made under delivery pressure. +**Done 2026-08-15.** `flannel_vxlan_allowed_sources` defaults to empty; the +role no longer adds a blanket 8472/udp allow and deletes one if present. +Goss asserts 8472 is never `ALLOW Anywhere` and that the non-Anywhere count +equals the declared peer list. When ThreePhoenix adds a second node, put its +address in that list before converging firewall. + ```task id: RAIL-HO-WP-0009-T05 -status: todo +status: done priority: medium state_hub_task_id: "81af8470-286c-440a-bbe7-b47a17ee6750" ``` @@ -251,6 +293,21 @@ Result on `Railiance01`: 32 assertions, 31 pass, one genuine failure (T06). What remains for this task is the *loop* — scheduling it and routing failures somewhere they are seen. +**Done 2026-08-15.** The check now also asserts exact allowlist *size*, so an +extra hand grant fails. The loop: + +- hourly `railiance-goss-baseline.timer` on each host after `make verify` +- TAP + `/var/lib/railiance/goss/last.status` + `FAILED` flag +- optional State Hub POST on pass↔fail via `RAILIANCE_GOSS_NOTIFY_URL` +- `make goss-status` / `make verify-host` + +Routed to `railiance-master` (ArchitectureBlueprint §5.3, Q7 owner) in +`docs/conformance-loop.md`. Family-level reaction (page a human, block a +rollout) is not this repo. + ## Outcome -Pending. T01 done; the live host is reachable but not yet converged. +T01, T03, T04, T05, T06 done. T02 waits on operator approval to run +`make converge-firewall HOST=Railiance01`, which will delete the three live +6443 grants and make the host match ADR-005. CoulombCore UFW is intentionally +not enabled.