diff --git a/engagements/pilots/eng-coulomb-railiance01-ho-001/.kaizen/metrics/host-operator/executions.jsonl b/engagements/pilots/eng-coulomb-railiance01-ho-001/.kaizen/metrics/host-operator/executions.jsonl index 5d0823f..99d83e2 100644 --- a/engagements/pilots/eng-coulomb-railiance01-ho-001/.kaizen/metrics/host-operator/executions.jsonl +++ b/engagements/pilots/eng-coulomb-railiance01-ho-001/.kaizen/metrics/host-operator/executions.jsonl @@ -1,2 +1,3 @@ {"agent": "host-operator", "duty": "standard_review", "engagement_id": "eng-coulomb-railiance01-ho-001", "execution_time_s": 5.0, "phase": "staffing", "quality_score": 0.7, "success": true, "timestamp": "2026-07-16T10:08:57Z"} {"agent": "host-operator", "duty": "deep_assessment", "engagement_id": "eng-coulomb-railiance01-ho-001", "execution_time_s": 900.0, "phase": "operating", "quality_score": 0.9, "success": true, "timestamp": "2026-07-16T10:42:51Z"} +{"agent": "host-operator", "duty": "deep_assessment", "engagement_id": "eng-coulomb-railiance01-ho-001", "execution_time_s": 1200.0, "phase": "operating", "quality_score": 0.88, "success": true, "timestamp": "2026-07-16T11:05:17Z"} diff --git a/engagements/pilots/eng-coulomb-railiance01-ho-001/.kaizen/metrics/host-operator/summary.json b/engagements/pilots/eng-coulomb-railiance01-ho-001/.kaizen/metrics/host-operator/summary.json index ea7b7a2..2cd58b0 100644 --- a/engagements/pilots/eng-coulomb-railiance01-ho-001/.kaizen/metrics/host-operator/summary.json +++ b/engagements/pilots/eng-coulomb-railiance01-ho-001/.kaizen/metrics/host-operator/summary.json @@ -1,9 +1,9 @@ { "agent": "host-operator", - "avg_execution_time_s": 452.5, - "avg_quality_score": 0.8, - "execution_count": 2, - "last_execution": "2026-07-16T10:42:51Z", + "avg_execution_time_s": 701.667, + "avg_quality_score": 0.827, + "execution_count": 3, + "last_execution": "2026-07-16T11:05:17Z", "success_rate": 1.0, "trend": { "quality_score": "stable", diff --git a/engagements/pilots/eng-coulomb-railiance01-ho-001/commercial/ledger.jsonl b/engagements/pilots/eng-coulomb-railiance01-ho-001/commercial/ledger.jsonl index 1e4497d..b2fe942 100644 --- a/engagements/pilots/eng-coulomb-railiance01-ho-001/commercial/ledger.jsonl +++ b/engagements/pilots/eng-coulomb-railiance01-ho-001/commercial/ledger.jsonl @@ -1,3 +1,4 @@ {"apiVersion":"kaizen.agentic/v1","kind":"KaiLedgerEntry","id":"kai-20260716-quote-open","account":"coulomb-ops-kai","engagement_id":"eng-coulomb-railiance01-ho-001","type":"note","product":"quote_snapshot","capability_tier":4,"amount_kai":0,"currency":"KAI","created_at":"2026-07-16T08:00:00Z","metadata":{"total_quoted_kai":72800,"phase":"staffing","note":"Month-1 estimate recorded; no charge until fund/ramp"}} {"access_surcharge_product": "read_only", "account": "coulomb-ops-kai", "amount_kai": 1600, "apiVersion": "kaizen.agentic/v1", "capability_tier": 4, "created_at": "2026-07-16T10:08:57Z", "currency": "KAI", "engagement_id": "eng-coulomb-railiance01-ho-001", "id": "kai-20260716T100857Z-standard_review", "kind": "KaiLedgerEntry", "metadata": {"access_class": "read_only", "phase": "staffing", "success": true, "target": "railiance01"}, "product": "standard_review", "session_ref": "reports/2026-07-16-standard-review.md", "type": "duty_charge"} {"access_surcharge_product": "host_observe", "account": "coulomb-ops-kai", "amount_kai": 4900, "apiVersion": "kaizen.agentic/v1", "capability_tier": 4, "created_at": "2026-07-16T10:42:51Z", "currency": "KAI", "engagement_id": "eng-coulomb-railiance01-ho-001", "id": "kai-20260716T104251Z-deep_assessment", "kind": "KaiLedgerEntry", "metadata": {"access_class": "host_observe", "phase": "operating", "success": true, "target": "railiance01"}, "product": "deep_assessment", "session_ref": "reports/2026-07-16-deep-assessment.md", "type": "duty_charge"} +{"access_surcharge_product": "privileged_ops", "account": "coulomb-ops-kai", "amount_kai": 5200, "apiVersion": "kaizen.agentic/v1", "capability_tier": 4, "created_at": "2026-07-16T11:05:17Z", "currency": "KAI", "engagement_id": "eng-coulomb-railiance01-ho-001", "id": "kai-20260716T110517Z-deep_assessment", "kind": "KaiLedgerEntry", "metadata": {"access_class": "privileged_ops", "phase": "operating", "success": true, "target": "railiance01"}, "product": "deep_assessment", "session_ref": "reports/2026-07-16-deep-assessment.md", "type": "duty_charge"} diff --git a/engagements/pilots/eng-coulomb-railiance01-ho-001/reports/2026-07-16-deep-assessment.md b/engagements/pilots/eng-coulomb-railiance01-ho-001/reports/2026-07-16-deep-assessment.md index 25a3dc8..74d410b 100644 --- a/engagements/pilots/eng-coulomb-railiance01-ho-001/reports/2026-07-16-deep-assessment.md +++ b/engagements/pilots/eng-coulomb-railiance01-ho-001/reports/2026-07-16-deep-assessment.md @@ -8,6 +8,6 @@ ## Summary -T10 ramp-up complete: Critical memory/load, k3s API unavailable, RU all done, phase operating +Approved P1-P4 done: 4G swap, journal vacuum, apt upgrade, UFW k3s API allowlist; k3s Ready; RAM still tight _Billing metadata only in commercial/ledger.jsonl; no secrets in this report._ diff --git a/engagements/pilots/eng-coulomb-railiance01-ho-001/reports/2026-07-16-privileged-remediation.md b/engagements/pilots/eng-coulomb-railiance01-ho-001/reports/2026-07-16-privileged-remediation.md new file mode 100644 index 0000000..7854a67 --- /dev/null +++ b/engagements/pilots/eng-coulomb-railiance01-ho-001/reports/2026-07-16-privileged-remediation.md @@ -0,0 +1,47 @@ +# Privileged remediation report — railiance01 — 2026-07-16 + +**Engagement:** eng-coulomb-railiance01-ho-001 +**Approval:** `vault/session-log/2026-07-16-privileged-approval.md` +**Outcome:** success (with residual capacity risk) + +## Summary + +| ID | Action | Result | +|----|--------|--------| +| P1 | 4 GiB `/swapfile`, fstab persist | **Done** — swap active (~1.9 GiB used post-change) | +| P2 | journald vacuum → 500 M | **Done** — freed ~3.5 GiB archived journals; journal now ~461 M | +| P3 | `apt-get upgrade` (noninteractive) | **Done** — packages updated; kernel still current (no reboot required) | +| P4 | UFW: drop world 6443/8472; allow 6443 from operator IPs | **Done** | + +## Before → after (host signals) + +| Metric | Before (ramp-up) | After remediation | +|--------|------------------|-------------------| +| Swap | none | 4 GiB file, ~1.9 GiB used | +| MemAvailable | often <150 Mi | ~625 Mi | +| PSI memory full avg60 | ~24% | ~8% | +| Load 1m | ~12 | ~6 | +| /var/log/journal | ~4.1 Gi | ~461 Mi | +| k3s node | API often unavailable | **Ready** control-plane | +| UFW 6443 | Anywhere | 89.244.90.246, 85.132.220.102 only | +| UFW 8472 | Anywhere | **removed** (single-node) | + +## Side effects / notes + +- `fwupd.service` failed to restart during upgrade (non-blocking for cluster) +- needrestart: deferred restarts for cloud-final, dbus, getty, logind, unattended-upgrades; user sessions still on older sshd binaries until re-login +- **RAM is still 3.8 GiB** — swap masks OOM risk but is not a substitute for more memory under sustained pressure +- If remote kubectl from another IP fails, add: `ufw allow from to any port 6443 proto tcp` +- UFW backups: `/etc/ufw/user.rules.bak.20260716` (and user6) + +## Explicitly not done + +- Hardware RAM upgrade (provider console) +- Host reboot (kernel KSTA=1, not required) + +## Recommended follow-up + +1. Plan RAM upgrade when convenient +2. Re-login SSH sessions to pick up new binaries +3. Watch swap usage; if chronically full, capacity still insufficient +4. Maintain operator IP allowlist when admin IPs change diff --git a/engagements/pilots/eng-coulomb-railiance01-ho-001/vault/handoff/risks.md b/engagements/pilots/eng-coulomb-railiance01-ho-001/vault/handoff/risks.md index fa8ecfa..480bd79 100644 --- a/engagements/pilots/eng-coulomb-railiance01-ho-001/vault/handoff/risks.md +++ b/engagements/pilots/eng-coulomb-railiance01-ho-001/vault/handoff/risks.md @@ -1,13 +1,10 @@ # Outstanding risks — eng-coulomb-railiance01-ho-001 -Populated during operate; finalise at ramp-down (RD-03). - | Risk | Severity | Mitigation / owner | Status | |------|----------|--------------------|--------| -| RAM undersized (3.8G, no swap) for k3s+platform density | Critical | Human capacity decision (RAM and/or swap); reduce concurrent load | open | -| k3s API ServiceUnavailable under pressure | Critical | Stabilize memory; re-check API; avoid heavy kubectl while degraded | open | -| Load average ≫ 2 cores | High | Same as capacity; schedule non-urgent work off-host | open | -| journald ~4.1G | Medium | Approved journal vacuum | open | -| Pending security package updates | Medium | Approved OS security pass | open | -| UFW allows 6443/8472 from Anywhere | Medium | Policy review + possible firewall_change | open | -| Reverse-forward port 18765 collisions | Low–Medium | Bridge/tunnel cleanup | open | +| RAM still 3.8G; swap heavily used (~1.9G/4G) | High | Provider RAM upgrade | open | +| Load still elevated (~6 on 2 cores) | Medium | Reduce concurrent workloads; capacity | open | +| Stale user sessions after apt upgrade | Low | Re-login SSH | open | +| UFW 6443 allowlist may miss new admin IPs | Medium | Add `ufw allow from to any port 6443 proto tcp` | open | +| 8472 closed externally | Low | Re-open if multi-node flannel peers appear | accepted (single-node) | +| fwupd.service failed restart during upgrade | Low | Investigate if firmware tooling needed | open | diff --git a/engagements/pilots/eng-coulomb-railiance01-ho-001/vault/memory.md b/engagements/pilots/eng-coulomb-railiance01-ho-001/vault/memory.md index 0b0f417..db6eeb5 100644 --- a/engagements/pilots/eng-coulomb-railiance01-ho-001/vault/memory.md +++ b/engagements/pilots/eng-coulomb-railiance01-ho-001/vault/memory.md @@ -3,7 +3,7 @@ agent: host-operator engagement_id: eng-coulomb-railiance01-ho-001 project: coulomb-railiance01 last_updated: '2026-07-16' -session_count: 3 +session_count: 4 confidentiality: client_owned --- @@ -32,33 +32,34 @@ forward-deployed host-operator pilot (KAIZEN-WP-0009). ## OS & Patch State -- Ubuntu 24.04.3 LTS (noble); kernel 6.8.0-87-generic (needrestart KSTA=1 — kernel current) -- Many packages upgradable (security-relevant: bind9-*, curl, ca-certificates, dpkg, …) -- `unattended-upgrades.service` flagged by needrestart for restart -- **No package upgrades applied** this session (gated) +- Ubuntu 24.04 LTS (noble); kernel 6.8.0-87-generic (needrestart KSTA=1 — no reboot required after 2026-07-16 upgrade) +- **2026-07-16:** full `apt-get upgrade` applied (approved); certs/ca updated; some service restarts deferred +- unattended-upgrades / logind may still want restart; re-login SSH sessions recommended + ## Security Posture -- UFW **active**: default deny in; allow OpenSSH 22, k3s API **6443/tcp Anywhere**, flannel **8472/udp Anywhere** -- Listeners include SSH, k3s components, localhost bridge ports (18000/18001), gitea stack processes -- Journal noise: reverse-forward port 18765 already in use (tunnel contention) -- Public 6443 exposure: policy review recommended (firewall_change gated) -- First snapshot: 2026-07-16 (see health report) +- UFW active: OpenSSH Anywhere; **6443 only from operator IPs** 89.244.90.246 and 85.132.220.102 +- **8472/udp world-open removed** (single-node; re-add if multi-node flannel peers needed) +- UFW backup: /etc/ufw/user.rules.bak.20260716 +- 2026-07-16 package upgrades applied (security-relevant packages included) + ## Load & Workload Envelope -| Field | Value (2026-07-16 sample) | -|-------|---------------------------| +| Field | Post-remediation 2026-07-16 | +|-------|----------------------------| | Cores | 2 | -| Load 1/5/15 | ~12 / 11 / 16 (**saturated**) | -| RAM | 3.8 GiB; no swap; MemAvailable often <0.5 GiB | -| PSI memory full | avg60 ~24%, avg300 ~29% | -| Disk / | 54% of 96G | -| /var/log | ~5.7G (journal ~4.1G) | -| Heavy workloads | k3s server, gitea, temporal-server, activity-core worker/API, state-hub edge uvicorn, traefik, coredns | -| Saturation incident | 2026-07-16 — memory+load critical; k3s API ServiceUnavailable | +| Load 1m | ~6 (was ~12) — still elevated | +| RAM | 3.8 GiB; MemAvailable ~625 Mi | +| Swap | **4 GiB /swapfile**, ~1.9 GiB used | +| PSI memory full avg60 | ~8% (was ~24%) | +| Disk / | ~55% | +| /var/log/journal | ~461 Mi (was ~4.1 Gi) | +| k3s node | Ready | + +Envelope: swap is emergency cushion; plan RAM upgrade if swap stays high. -**Envelope intent:** treat load > 4 sustained or MemAvailable < 300Mi as **Watch/Critical**; escalate capacity. ## Accumulated Findings @@ -83,22 +84,31 @@ forward-deployed host-operator pilot (KAIZEN-WP-0009). ## Recurring Findings -- Memory pressure / no swap · first seen 2026-07-16 · frequency 1 -- Load ≫ cores · first seen 2026-07-16 · frequency 1 +- Memory pressure on 3.8G host · first seen 2026-07-16 · mitigated by swap but capacity still tight +- Load ≫ cores · improved after remediation; still watch + ## Cleared Issues -_None yet (no remediation applied)._ +- No swap · cleared 2026-07-16 via /swapfile 4G + fstab +- journald 4G bloat · vacuumed to ~500M target 2026-07-16 +- World-open k3s API 6443 · restricted to operator IPs 2026-07-16 +- World-open flannel 8472 · removed 2026-07-16 (single-node) +- Pending package upgrades · apt upgrade applied 2026-07-16 + ## Open Threads -- Human decision: add RAM and/or temporary swap -- Human approval: journal vacuum; security package upgrades; UFW source restriction for 6443 -- Re-check k3s API health after memory improves -- Align inventory hostname with kernel hostname / DNS naming +- Plan hardware/provider RAM increase (swap is temporary relief) +- Re-login stale SSH sessions after package upgrade +- Add UFW allow for new admin IPs when they change +- Monitor if 8472 needed for any multi-node peer (currently none) + ## Session Log - 2026-07-16 · railiance01 · standard_review · T09 wire-up smoke: prepare+close-session path verified (no host access) · ok - 2026-07-16 · railiance01 · first live observe: Critical memory/load; k3s API unavailable; RU checklist complete · ok - 2026-07-16 · railiance01 · deep_assessment · T10 ramp-up complete: Critical memory/load, k3s API unavailable, RU all done, phase operating · ok +- 2026-07-16 · railiance01 · privileged remediation P1–P4 applied (swap, journal, apt, ufw) · ok +- 2026-07-16 · railiance01 · deep_assessment · Approved P1-P4 done: 4G swap, journal vacuum, apt upgrade, UFW k3s API allowlist; k3s Ready; RAM still tight · ok diff --git a/engagements/pilots/eng-coulomb-railiance01-ho-001/vault/session-log/2026-07-16-privileged-approval.md b/engagements/pilots/eng-coulomb-railiance01-ho-001/vault/session-log/2026-07-16-privileged-approval.md new file mode 100644 index 0000000..4c6c919 --- /dev/null +++ b/engagements/pilots/eng-coulomb-railiance01-ho-001/vault/session-log/2026-07-16-privileged-approval.md @@ -0,0 +1,32 @@ +# Human approval — privileged remediation batch + +**Engagement:** eng-coulomb-railiance01-ho-001 +**Date:** 2026-07-16 +**Approver:** Bernd (operator) — via session instruction: *“approve the suggested tasks… implement them”* +**Agent:** host-operator (grok session) + +## Approved items + +| ID | Action | Class | +|----|--------|--------| +| P1 | Emergency swap file (~4G) on root FS | privileged_ops | +| P2 | journald vacuum to ~500M | privileged_ops | +| P3 | OS package upgrades (apt) + needrestart awareness | package_upgrade | +| P4 | Tighten UFW: remove world-open 6443/8472; allow k3s API from known operator IPs | firewall_change | + +## Explicit non-goals + +- Hardware RAM upgrade (provider console — not shell-implementable here) +- Unilateral workload kill / pod deletion +- Reboot unless needrestart shows kernel requires it **and** operator re-confirms mid-session + +## Rollback notes + +- P1: `swapoff` + remove swapfile + fstab line +- P2: logs not recoverable +- P3: package-specific rollback / restore +- P4: re-add `ufw allow 6443/tcp` and `8472/udp` if access breaks + +## Status + +Approved for immediate implementation under host_observe + gated classes above.