tegwick
e72c5de92f
Fresh read-only railiance01 capture (06:08Z) via the Custodian collector, 7d Prometheus usage, per-workload proposals with owners, and a node budget. The rule-based proposal reconciles at +140m only with serialized release/build/HPA events (-340m all concurrent); no procurement. Vergabe stays 60m provisional pending RAPPS-WP-0014-T04. T03 stays in progress: 54 changes lack receiving tasks in 26 owner repos. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 8340@bnt-lap001 Assistant-Session: 2e192b3d-b3e8-4403-94d4-d55a2c8b476c
45 lines
4 KiB
Python
45 lines
4 KiB
Python
import json, subprocess, sys, importlib.util
|
|
from collections import defaultdict
|
|
spec = importlib.util.spec_from_file_location("o", "/home/worsch/railiance-cluster/tools/observe_cluster_resources.py")
|
|
o = importlib.util.module_from_spec(spec); spec.loader.exec_module(o)
|
|
tele = json.load(open(sys.argv[1])); out = sys.argv[2]
|
|
pods = json.loads(subprocess.run(["kubectl","get","pods","-A","-o","json"],capture_output=True,text=True,check=True).stdout)["items"]
|
|
def vec(k):
|
|
d = {}
|
|
for r in tele["queries"][k]["response"]["data"]["result"]:
|
|
m = r["metric"]; d[(m.get("namespace"), m.get("pod"))] = float(r["value"][1])
|
|
return d
|
|
V = {k: vec(k) for k in ["cpu_p95_m_by_pod","cpu_peak_m_by_pod","mem_peak_bytes_by_pod","mem_p95_bytes_by_pod","throttle_ratio_by_pod","restarts_by_pod"]}
|
|
W = defaultdict(lambda: {"pods":[], "cpu_req_m":0,"mem_req_mi":0.0,"cpu_lim_m":0,"mem_lim_mi":0.0,"containers_without_cpu_limit":0,"containers_without_mem_limit":0,"containers_without_cpu_request":0,
|
|
"cpu_p95_m_sum":0.0,"cpu_peak_m_sum":0.0,"mem_peak_mi_sum":0.0,"mem_p95_mi_sum":0.0,"throttle_max":None,"restarts_7d":0.0,"pods_without_metrics":[],"phases":set(),"kinds":set()})
|
|
for p in pods:
|
|
md, sp, st = p["metadata"], p["spec"], p.get("status",{})
|
|
if st.get("phase") in ("Succeeded","Failed"): continue
|
|
ns, name = md["namespace"], md["name"]; wl = o.workload_name(p); w = W[(ns, wl)]
|
|
w["pods"].append(name); w["phases"].add(st.get("phase")); w["kinds"].update(r["kind"] for r in md.get("ownerReferences") or [])
|
|
c, m = o.pod_effective_requests(p); w["cpu_req_m"] += c; w["mem_req_mi"] += m/2**20
|
|
for ct in sp.get("containers", []):
|
|
lim = (ct.get("resources") or {}).get("limits") or {}; req = (ct.get("resources") or {}).get("requests") or {}
|
|
if "cpu" in lim: w["cpu_lim_m"] += o.cpu_millicores(lim["cpu"])
|
|
else: w["containers_without_cpu_limit"] += 1
|
|
if "memory" in lim: w["mem_lim_mi"] += o.bytes_value(lim["memory"])/2**20
|
|
else: w["containers_without_mem_limit"] += 1
|
|
if not req.get("cpu"): w["containers_without_cpu_request"] += 1
|
|
k = (ns, name)
|
|
if k not in V["cpu_p95_m_by_pod"]: w["pods_without_metrics"].append(name)
|
|
w["cpu_p95_m_sum"] += V["cpu_p95_m_by_pod"].get(k,0); w["cpu_peak_m_sum"] += V["cpu_peak_m_by_pod"].get(k,0)
|
|
w["mem_peak_mi_sum"] += V["mem_peak_bytes_by_pod"].get(k,0)/2**20; w["mem_p95_mi_sum"] += V["mem_p95_bytes_by_pod"].get(k,0)/2**20
|
|
t = V["throttle_ratio_by_pod"].get(k)
|
|
if t is not None and t == t: w["throttle_max"] = max(w["throttle_max"] or 0, t)
|
|
w["restarts_7d"] += V["restarts_by_pod"].get(k,0)
|
|
rows = []
|
|
for (ns, wl), w in W.items():
|
|
w = {**w, "namespace": ns, "workload": wl, "phases": sorted(x for x in w["phases"] if x), "kinds": sorted(w["kinds"])}
|
|
for f in ("cpu_p95_m_sum","cpu_peak_m_sum","mem_peak_mi_sum","mem_p95_mi_sum","restarts_7d","mem_req_mi","mem_lim_mi"): w[f] = round(w[f],1)
|
|
if w["throttle_max"] is not None: w["throttle_max"] = round(w["throttle_max"],3)
|
|
rows.append(w)
|
|
rows.sort(key=lambda r: (-r["cpu_req_m"], r["namespace"]))
|
|
json.dump({"captured_at": tele["captured_at"], "note": "Per-workload sums of per-pod 7d p95/peak (upper bound for multi-pod workloads; sum of p95 >= p95 of sum). Current running pods only; terminated Job pods appear only in namespace totals.", "workloads": rows}, open(out,"w"), indent=1)
|
|
print(f"{'ns/workload':55} req lim p95 peak memreq memlim mempk thr rst")
|
|
for r in rows: print(f"{r['namespace']+'/'+r['workload']:55} {r['cpu_req_m']:4} {r['cpu_lim_m']:5}{'*' if r['containers_without_cpu_limit'] else ' '}{r['cpu_p95_m_sum']:6} {r['cpu_peak_m_sum']:6} {r['mem_req_mi']:6.0f} {r['mem_lim_mi']:6.0f} {r['mem_peak_mi_sum']:6.0f} {r['throttle_max']} {r['restarts_7d']} {'NOMET' if r['pods_without_metrics'] else ''}")
|
|
print("TOTAL req", sum(r['cpu_req_m'] for r in rows), "p95sum", round(sum(r['cpu_p95_m_sum'] for r in rows)), "peaksum", round(sum(r['cpu_peak_m_sum'] for r in rows)), "memreq", round(sum(r['mem_req_mi'] for r in rows)), "mempk", round(sum(r['mem_peak_mi_sum'] for r in rows)))
|