321 lines
18 KiB
Python
321 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Retain CUST-WP-0071 sizing evidence; no network, resizing or task emission."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from datetime import datetime, timedelta, timezone
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import tempfile
|
|
|
|
SCHEMA = "custodian.weekly-allocation.v1"
|
|
METRICS = (
|
|
"cpu_p95_m", "cpu_peak_m", "memory_p95_bytes", "memory_peak_bytes",
|
|
"cpu_throttle_ratio", "latency_p95_seconds", "errors", "request_count",
|
|
"restarts", "oom_events", "cpu_sample_count", "memory_sample_count",
|
|
)
|
|
REQUIRED = tuple(m for m in METRICS if not m.endswith("sample_count"))
|
|
|
|
|
|
def timestamp(value):
|
|
result = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
if result.tzinfo is None:
|
|
raise ValueError("timestamps must include timezone")
|
|
return result.astimezone(timezone.utc)
|
|
|
|
|
|
def duration(value):
|
|
match = re.fullmatch(r"([1-9][0-9]*)([smhdw])", value)
|
|
if not match:
|
|
raise ValueError(f"unsupported duration: {value}")
|
|
return int(match[1]) * {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800}[match[2]]
|
|
|
|
|
|
def digest(value):
|
|
return hashlib.sha256(json.dumps(value, sort_keys=True, allow_nan=False).encode()).hexdigest()
|
|
|
|
|
|
def finite(value):
|
|
number = float(value)
|
|
if not math.isfinite(number) or number < 0:
|
|
raise ValueError("metric must be finite and nonnegative")
|
|
return number
|
|
|
|
|
|
def metric_series(telemetry):
|
|
"""Do not distribute namespace aggregates over workloads, or sum percentiles."""
|
|
series, queries = {}, {}
|
|
seconds = duration(telemetry["window"])
|
|
for name, query in sorted(telemetry.get("queries", {}).items()):
|
|
response = query.get("response", {})
|
|
data = response.get("data", {})
|
|
queries[name] = {
|
|
"promql": query.get("promql"), "response_status": response.get("status"),
|
|
"result_type": data.get("resultType"), "warnings": response.get("warnings", []),
|
|
"error": response.get("error"), "response_sha256": digest(response),
|
|
"evaluation_times": [],
|
|
}
|
|
if name not in METRICS or response.get("status") != "success" or data.get("resultType") != "vector":
|
|
continue
|
|
for sample in data.get("result", []):
|
|
labels = sample.get("metric", {})
|
|
namespace, workload = labels.get("namespace"), labels.get("workload")
|
|
# Additional dimensions are ambiguous: do not silently collapse pod/job series.
|
|
if not namespace or set(labels) - {"namespace", "workload", "__name__"}:
|
|
queries[name].setdefault("rejected", []).append("unaggregated or absent identity labels")
|
|
continue
|
|
key = (namespace, workload)
|
|
values = series.setdefault(key, {})
|
|
if name in values:
|
|
values[name] = {"value": None, "reason": "ambiguous duplicate series"}
|
|
continue
|
|
try:
|
|
epoch, raw = sample["value"]
|
|
end = datetime.fromtimestamp(finite(epoch), timezone.utc)
|
|
number = finite(raw)
|
|
except (KeyError, ValueError, TypeError, OverflowError):
|
|
values[name] = {"value": None, "reason": "invalid sample"}
|
|
continue
|
|
queries[name]["evaluation_times"].append(end.isoformat())
|
|
values[name] = {
|
|
"value": number, "window_start": (end - timedelta(seconds=seconds)).isoformat(),
|
|
"window_end": end.isoformat(), "query": name,
|
|
"capture_alignment_seconds": (end - timestamp(telemetry["captured_at"])).total_seconds(),
|
|
}
|
|
return series, queries
|
|
|
|
|
|
def recommendation_id(resource, namespace, workload):
|
|
return "allocation-" + digest([resource, namespace, workload, "sizing-review"])[:24]
|
|
|
|
|
|
def build_report(cluster, allocation, telemetry, inventory, provenance, *, as_of,
|
|
previous=None, reviews=None, max_age_hours=48):
|
|
now = timestamp(as_of)
|
|
if cluster.get("schema_version") != "railiance.cluster-resource-observation.v1":
|
|
raise ValueError("unsupported cluster schema")
|
|
if allocation.get("schema") != "custodian.allocation-reconcile.v1":
|
|
raise ValueError("unsupported allocation schema")
|
|
resource = cluster["resource_id"]
|
|
if inventory.get("id") != resource or allocation.get("host", {}).get("cluster_resource_id") != resource:
|
|
raise ValueError("resource identity mismatch")
|
|
if allocation.get("count_host_and_cluster_cpus_once") is not True:
|
|
raise ValueError("allocation must deduplicate host and cluster CPUs")
|
|
if cluster["captured_at"] != allocation["captured_at"]:
|
|
raise ValueError("cluster and reconcile snapshots differ")
|
|
if provenance.get("source_observation") != cluster["captured_at"]:
|
|
raise ValueError("provenance does not identify cluster snapshot")
|
|
seconds = duration(telemetry["window"])
|
|
if seconds != 604800:
|
|
raise ValueError("weekly report requires a seven-day telemetry window")
|
|
step = duration(telemetry["step"])
|
|
series, queries = metric_series(telemetry)
|
|
captured = {"cluster": cluster["captured_at"], "telemetry": telemetry["captured_at"]}
|
|
freshness = {}
|
|
for name, value in captured.items():
|
|
age = (now - timestamp(value)).total_seconds()
|
|
freshness[name] = {"age_seconds": age, "status": "future" if age < 0 else "stale" if age > max_age_hours * 3600 else "fresh"}
|
|
query_signature = digest({name: q["promql"] for name, q in queries.items()})
|
|
end = timestamp(telemetry["captured_at"])
|
|
comparison_reason = "no previous retained report"
|
|
compatible = False
|
|
if previous:
|
|
prev_end = timestamp(previous["period"]["capture_end"])
|
|
compatible = (
|
|
previous.get("schema") == SCHEMA and previous.get("resource_id") == resource
|
|
and previous["period"].get("seconds") == seconds
|
|
and previous["period"].get("step_seconds") == step
|
|
and previous.get("query_signature") == query_signature
|
|
and abs((end - prev_end).total_seconds() - seconds) <= step
|
|
)
|
|
comparison_reason = "comparable consecutive capture windows" if compatible else "nonconsecutive windows or changed query/resource/step contract"
|
|
previous_rows = {(r["namespace"], r["workload"]): r for r in (previous or {}).get("workloads", [])}
|
|
owners = {(r["namespace"], r["workload"]): r for r in allocation.get("workloads", [])}
|
|
if len(owners) != len(allocation.get("workloads", [])):
|
|
raise ValueError("duplicate allocation workload identity")
|
|
reviews = reviews or {}
|
|
rows, recommendations = [], []
|
|
seen = set()
|
|
for workload in sorted(cluster.get("utilization", {}).get("workloads", []), key=lambda x: (x["namespace"], x["workload"])):
|
|
key = workload["namespace"], workload["workload"]
|
|
if key in seen:
|
|
raise ValueError("duplicate cluster workload identity")
|
|
seen.add(key)
|
|
owner = owners.get(key, {})
|
|
measurements = {m: series.get(key, {}).get(m, {"value": None, "reason": "no workload series"}) for m in METRICS}
|
|
gaps = [m for m in REQUIRED if measurements[m]["value"] is None]
|
|
metric_stale = [m for m, v in measurements.items() if v["value"] is not None and not 0 <= (now - timestamp(v["window_end"])).total_seconds() <= max_age_hours * 3600]
|
|
metric_unaligned = [m for m, v in measurements.items() if v["value"] is not None and abs(v["capture_alignment_seconds"]) > step]
|
|
rid = recommendation_id(resource, *key)
|
|
reasons = []
|
|
if gaps:
|
|
reasons.append("missing workload metrics: " + ", ".join(gaps))
|
|
if metric_stale or any(v["status"] != "fresh" for v in freshness.values()):
|
|
reasons.append("stale or future observation")
|
|
if metric_unaligned:
|
|
reasons.append("metric evaluation windows differ from capture by more than one step")
|
|
if owner.get("owner", "unknown") == "unknown":
|
|
reasons.append("unknown owner")
|
|
if workload.get("requests", {}).get("cpu_millicores") == 0:
|
|
reasons.append("zero CPU request requires owner review")
|
|
review = reviews.get(rid)
|
|
decision, confidence, evidence = "investigate", "low", []
|
|
if review:
|
|
if review.get("decision") not in {"keep", "increase", "decrease", "investigate"}:
|
|
raise ValueError("invalid owner review decision")
|
|
required = ("owner", "evidence_refs", "receiving_task", "configuration", "rollback_ref", "reviewed_at", "period_end", "coverage_ref", "service_acceptance_ref", "margin_ref")
|
|
if any(not review.get(k) for k in required):
|
|
raise ValueError("owner review missing evidence, receiving task or configuration/rollback")
|
|
if review["owner"] != owner.get("owner") or review["period_end"] != telemetry["captured_at"]:
|
|
reasons.append("owner review does not match this owner/window")
|
|
elif timestamp(review["reviewed_at"]) > now:
|
|
reasons.append("owner review is future-dated")
|
|
elif not reasons:
|
|
decision, confidence, evidence = review["decision"], "owner-reviewed", review["evidence_refs"]
|
|
else:
|
|
reasons.append("owner proposal retained; incomplete evidence prevents accepted sizing recommendation")
|
|
else:
|
|
reasons.append("no owner-reviewed sizing decision")
|
|
row = {
|
|
"namespace": key[0], "workload": key[1], "owner": owner.get("owner", "unknown"),
|
|
"service": owner.get("service", "unknown"), "tenant": owner.get("tenant", "unknown"),
|
|
"pods": workload.get("pods"), "requests": workload.get("requests"),
|
|
"limits": workload.get("limits"), "instantaneous_observed": workload.get("observed"),
|
|
"metrics": measurements, "missing_metrics": gaps,
|
|
"namespace_context_ref": key[0] if (key[0], None) in series else None,
|
|
"raw_scrape_coverage": "unknown; evaluation counts do not prove source coverage",
|
|
"recommendation_id": rid, "week_on_week": {}, "comparison_reasons": {},
|
|
}
|
|
prior = previous_rows.get(key)
|
|
for metric in METRICS:
|
|
current = measurements[metric]["value"]
|
|
old_measurement = (prior or {}).get("metrics", {}).get(metric, {})
|
|
old = old_measurement.get("value")
|
|
metric_compatible = False
|
|
reason = comparison_reason
|
|
if compatible and current is not None and old is not None:
|
|
try:
|
|
old_end, old_start = timestamp(old_measurement["window_end"]), timestamp(old_measurement["window_start"])
|
|
new_end, new_start = timestamp(measurements[metric]["window_end"]), timestamp(measurements[metric]["window_start"])
|
|
metric_compatible = (
|
|
(old_end - old_start).total_seconds() == seconds
|
|
and (new_end - new_start).total_seconds() == seconds
|
|
and abs((new_end - old_end).total_seconds() - seconds) <= step
|
|
and abs((new_start - old_end).total_seconds()) <= step
|
|
and abs(measurements[metric]["capture_alignment_seconds"]) <= step
|
|
and abs((old_end - timestamp(previous["period"]["capture_end"])).total_seconds()) <= step
|
|
)
|
|
reason = "comparable consecutive metric windows" if metric_compatible else "metric evaluation windows are not consecutive or capture-aligned"
|
|
except (KeyError, ValueError, TypeError):
|
|
reason = "missing or invalid previous metric window"
|
|
elif compatible:
|
|
reason = "missing current or previous metric"
|
|
row["week_on_week"][metric] = current - old if metric_compatible else None
|
|
row["comparison_reasons"][metric] = reason
|
|
rows.append(row)
|
|
prior_recommendation = next((r for r in (previous or {}).get("recommendations", []) if r["id"] == rid), {})
|
|
recommendations.append({
|
|
"id": rid, "decision": decision, "confidence": confidence, "owner": row["owner"],
|
|
"reasons": reasons, "evidence_refs": evidence, "owner_proposal": review,
|
|
"receiving_task": review.get("receiving_task") if review else prior_recommendation.get("receiving_task"),
|
|
"previous_decision": prior_recommendation.get("decision"),
|
|
"owner_outcome": review.get("outcome") if review else None,
|
|
"delivery": "not emitted", "acknowledgment": None,
|
|
})
|
|
if seen != set(owners):
|
|
raise ValueError("cluster and allocation workload inventories differ")
|
|
return {
|
|
"schema": SCHEMA, "workplan": "CUST-WP-0071-T05", "as_of": as_of,
|
|
"adapter_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
|
|
"resource_id": resource, "period": {"capture_end": telemetry["captured_at"], "seconds": seconds, "step_seconds": step,
|
|
"note": "Actual per-query windows are retained on samples; capture time is not a common query evaluation time."},
|
|
"freshness": freshness, "max_age_hours": max_age_hours, "query_signature": query_signature,
|
|
"queries": queries, "collector_provenance": provenance,
|
|
"source_sha256": {"cluster": digest(cluster), "allocation": digest(allocation), "telemetry": digest(telemetry), "inventory": digest(inventory)},
|
|
"node": {"capacity": cluster.get("capacity"), "allocation": cluster.get("allocation"),
|
|
"scheduled_request_cpu_m": allocation.get("scheduled_request_cpu_m"),
|
|
"pending_unscheduled_cpu_m": allocation.get("pending_unscheduled_cpu_m"),
|
|
"reservation_residual_cpu_m": allocation.get("residual_cpu_m"),
|
|
"release_preflight": allocation.get("state_hub_preflight_observation"),
|
|
"operating_margin": None, "release_margin_acceptance": "not established by reservation residual",
|
|
"host_cluster_counted_once": True},
|
|
"portfolio": {k: inventory.get(k) for k in ("id", "schema_version", "ownership", "capacity", "relationships", "evidence")},
|
|
"workloads": rows,
|
|
"namespace_context": {ns: values for (ns, workload), values in sorted(series.items(), key=lambda x: str(x[0])) if workload is None},
|
|
"comparison": {"compatible": compatible, "reason": comparison_reason,
|
|
"previous_report_sha256": digest(previous) if previous else None,
|
|
"disappeared_workloads": [list(key) for key in sorted(set(previous_rows) - seen)]},
|
|
"measurement_gaps": cluster.get("measurement_gaps", []) + allocation.get("measurement_gaps", []),
|
|
"recommendations": recommendations,
|
|
"limitations": ["No representative pilot acceptance inferred from telemetry.",
|
|
"Namespace percentiles and peaks are not workload metrics or additive node peaks.",
|
|
"Portfolio capacity is dated evidence, not additional host capacity.",
|
|
"No procurement, sizing changes, owner-task creation, delivery or acknowledgment performed.",
|
|
"Installed executor, scoped reads, independent missed-run handling and first unattended receipt remain T05 gates."],
|
|
}
|
|
|
|
|
|
def retain(path, report):
|
|
"""Never overwrite a retained week; identical retries succeed."""
|
|
encoded = json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n"
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = None
|
|
try:
|
|
with tempfile.NamedTemporaryFile(mode="w", dir=path.parent, delete=False) as stream:
|
|
temporary = Path(stream.name)
|
|
stream.write(encoded)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
try:
|
|
os.link(temporary, path)
|
|
except FileExistsError:
|
|
if path.is_symlink() or path.read_text() != encoded:
|
|
raise ValueError("retained report differs; choose a new explicit revision path")
|
|
finally:
|
|
if temporary is not None:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--request", type=Path, help="Executor-owned JSON request; paths relative to its directory")
|
|
for name in ("cluster", "allocation", "telemetry", "inventory", "provenance"):
|
|
parser.add_argument("--" + name, type=Path)
|
|
parser.add_argument("--previous", type=Path)
|
|
parser.add_argument("--reviews", type=Path)
|
|
parser.add_argument("--as-of")
|
|
parser.add_argument("--max-age-hours", type=float, default=48)
|
|
parser.add_argument("--output", type=Path)
|
|
args = parser.parse_args()
|
|
try:
|
|
if args.request:
|
|
request = json.loads(args.request.read_text())
|
|
if request.get("schema") != "custodian.weekly-allocation-request.v1":
|
|
raise ValueError("unsupported executor request schema")
|
|
if any(getattr(args, key) for key in ("cluster", "allocation", "telemetry", "inventory", "provenance", "previous", "reviews", "as_of", "output")):
|
|
raise ValueError("request cannot be mixed with input/output overrides")
|
|
for key in ("cluster", "allocation", "telemetry", "inventory", "provenance", "previous", "reviews", "output"):
|
|
value = request.get(key)
|
|
setattr(args, key, args.request.parent / value if value else None)
|
|
args.as_of = request.get("as_of")
|
|
if any(not getattr(args, key) for key in ("cluster", "allocation", "telemetry", "inventory", "provenance", "as_of", "output")):
|
|
raise ValueError("executor input paths, as_of and retention output must be explicitly bound")
|
|
if not math.isfinite(args.max_age_hours) or args.max_age_hours <= 0:
|
|
raise ValueError("max-age-hours must be positive and finite")
|
|
inputs = {name: json.loads(getattr(args, name).read_text()) for name in ("cluster", "allocation", "telemetry", "inventory", "provenance")}
|
|
report = build_report(**inputs, as_of=args.as_of, max_age_hours=args.max_age_hours,
|
|
previous=json.loads(args.previous.read_text()) if args.previous else None,
|
|
reviews=json.loads(args.reviews.read_text()) if args.reviews else None)
|
|
retain(args.output, report)
|
|
except (OSError, ValueError, KeyError, TypeError) as exc:
|
|
parser.exit(2, f"weekly allocation report refused: {exc}\n")
|
|
print(json.dumps({"status": "retained", "path": str(args.output), "sha256": digest(report),
|
|
"workloads": len(report["workloads"]), "delivery": "not emitted"}))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|