tegwick
dca6683a8a
Adds runtime: remote to the synthetic pilot driver (external base URL and disposable database, restart via the driver's namespaced ServiceAccount, UTC phase windows for external cgroup sampling) and the target replay profile pinned to the deployed release digest and 3 Gunicorn workers. Five focused tests added; 22 tests pass, ruff clean. For railiance-apps RAPPS-WP-0014-T04. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 8340@bnt-lap001 Assistant-Session: 2e192b3d-b3e8-4403-94d4-d55a2c8b476c
1091 lines
44 KiB
Python
1091 lines
44 KiB
Python
#!/usr/bin/env python3
|
|
"""Provisional HTTP workload against Gunicorn and a disposable PostgreSQL.
|
|
|
|
Runtimes: ``host`` and ``container`` start everything locally. ``remote`` drives
|
|
an already running isolated synthetic deployment (for example a throwaway
|
|
Kubernetes namespace): the base URL, database URL and optional restart pod
|
|
selector come from ``SYNTHETIC_BASE_URL``, ``SYNTHETIC_DATABASE_URL`` and
|
|
``SYNTHETIC_RESTART_SELECTOR``. Remote resource sampling is external; phase UTC
|
|
windows are retained so samples can be aligned afterwards.
|
|
"""
|
|
|
|
import argparse
|
|
import concurrent.futures
|
|
import hashlib
|
|
import http.cookiejar
|
|
import json
|
|
import math
|
|
import os
|
|
import platform
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
|
|
def summarize(rows):
|
|
durations = sorted(row["seconds"] for row in rows)
|
|
return {
|
|
"requests": len(rows),
|
|
"errors": sum(not r["ok"] for r in rows),
|
|
"p95_seconds": durations[math.ceil(len(durations) * 0.95) - 1] if durations else None,
|
|
}
|
|
|
|
|
|
LEGACY_OPERATIONS = [
|
|
"list",
|
|
"search",
|
|
"tender-create",
|
|
"detail",
|
|
"lot-create",
|
|
"task-create",
|
|
"document-upload",
|
|
"document-download",
|
|
]
|
|
DEPENDENCIES = {
|
|
"list": None,
|
|
"search": None,
|
|
"tender-create": None,
|
|
"detail": "tender-create",
|
|
"tender-edit": "tender-create",
|
|
"lot-create": "tender-create",
|
|
"lot-edit": "lot-create",
|
|
"task-create": "tender-create",
|
|
"task-edit": "task-create",
|
|
"document-upload": "tender-create",
|
|
"document-download": "document-upload",
|
|
}
|
|
|
|
|
|
def operation_plan(operations):
|
|
if not isinstance(operations, list) or not operations:
|
|
raise ValueError("operations must be a nonempty ordered list")
|
|
seen = set()
|
|
for operation in operations:
|
|
if operation not in DEPENDENCIES:
|
|
raise ValueError(f"unknown operation: {operation}")
|
|
dependency = DEPENDENCIES[operation]
|
|
if dependency and dependency not in seen:
|
|
raise ValueError(f"{operation} requires earlier {dependency}")
|
|
if operation in seen and (operation.endswith("-create") or operation == "document-upload"):
|
|
raise ValueError(f"only one {operation} per cycle; repeat reads/edits to tune mix")
|
|
seen.add(operation)
|
|
return list(operations)
|
|
|
|
|
|
def phase_plan(config):
|
|
phases = config.get(
|
|
"phases",
|
|
[
|
|
{
|
|
"name": "ordinary",
|
|
"kind": "load",
|
|
"duration_seconds": config["duration_seconds"],
|
|
"think_seconds": config["think_seconds"],
|
|
}
|
|
],
|
|
)
|
|
if not isinstance(phases, list) or not phases:
|
|
raise ValueError("phases must be a nonempty list")
|
|
names = set()
|
|
result = []
|
|
for item in phases:
|
|
item = dict(item)
|
|
name = item.get("name")
|
|
if not isinstance(name, str) or not name or name in names or name in ("idle", "startup"):
|
|
raise ValueError("phase names must be unique and not idle/startup")
|
|
names.add(name)
|
|
if item.get("kind") == "restart":
|
|
if config.get("runtime") not in ("container", "remote"):
|
|
raise ValueError("restart phase requires container or remote runtime")
|
|
elif item.get("kind") == "load":
|
|
for key in ("duration_seconds", "think_seconds"):
|
|
value = item.get(key)
|
|
if type(value) not in (int, float) or not math.isfinite(value) or value < 0:
|
|
raise ValueError(f"invalid phase {key}")
|
|
if not item["duration_seconds"]:
|
|
raise ValueError("load duration must be positive")
|
|
item["operations"] = operation_plan(
|
|
item.get("operations", config.get("operations", LEGACY_OPERATIONS))
|
|
)
|
|
else:
|
|
raise ValueError("phase kind must be load or restart")
|
|
result.append(item)
|
|
if not any(p["kind"] == "load" for p in result):
|
|
raise ValueError("at least one load phase required")
|
|
return result
|
|
|
|
|
|
def expected_requests(completed, operations, users):
|
|
return sum(completed) * len(operations) + users * 2
|
|
|
|
|
|
def wait_ready(base_url, timeout=30):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(base_url + "/health/", timeout=1) as response:
|
|
if response.status == 200:
|
|
return
|
|
except OSError:
|
|
pass
|
|
time.sleep(0.1)
|
|
raise RuntimeError("application readiness timeout")
|
|
|
|
|
|
def container_url(name):
|
|
binding = subprocess.check_output(
|
|
["docker", "port", name, "8000/tcp"], text=True, timeout=10
|
|
).strip()
|
|
host, port = binding.rsplit(":", 1)
|
|
if host != "127.0.0.1" or not port.isdigit():
|
|
raise ValueError("synthetic runtime must have one loopback HTTP binding")
|
|
return f"http://127.0.0.1:{port}"
|
|
|
|
|
|
KUBE_SA = Path("/var/run/secrets/kubernetes.io/serviceaccount")
|
|
|
|
|
|
def kube_request(method, path, sa_dir=KUBE_SA):
|
|
"""Minimal in-cluster API call using the pod's own namespaced ServiceAccount."""
|
|
host = os.environ["KUBERNETES_SERVICE_HOST"]
|
|
port = os.environ.get("KUBERNETES_SERVICE_PORT", "443")
|
|
import ssl
|
|
|
|
context = ssl.create_default_context(cafile=str(sa_dir / "ca.crt"))
|
|
req = urllib.request.Request(
|
|
f"https://{host}:{port}{path}",
|
|
method=method,
|
|
headers={"Authorization": "Bearer " + (sa_dir / "token").read_text().strip()},
|
|
)
|
|
with urllib.request.urlopen(req, timeout=10, context=context) as response:
|
|
return json.loads(response.read())
|
|
|
|
|
|
def ready_pods(listing):
|
|
return [
|
|
item["metadata"]["name"]
|
|
for item in listing.get("items", [])
|
|
if not item["metadata"].get("deletionTimestamp")
|
|
and any(
|
|
c["type"] == "Ready" and c["status"] == "True"
|
|
for c in item.get("status", {}).get("conditions", [])
|
|
)
|
|
]
|
|
|
|
|
|
def restart_kubernetes_pod(base_url, selector, timeout=180, sa_dir=KUBE_SA, request=None):
|
|
"""Delete the single selected pod in this namespace and wait for a new ready one."""
|
|
request = request or (lambda method, path: kube_request(method, path, sa_dir))
|
|
namespace = (sa_dir / "namespace").read_text().strip()
|
|
query = urllib.parse.urlencode({"labelSelector": selector})
|
|
pods = f"/api/v1/namespaces/{namespace}/pods"
|
|
before = ready_pods(request("GET", f"{pods}?{query}"))
|
|
if len(before) != 1:
|
|
raise RuntimeError("restart requires exactly one ready selected pod")
|
|
request("DELETE", f"{pods}/{before[0]}")
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
listing = request("GET", f"{pods}?{query}")
|
|
names = [i["metadata"]["name"] for i in listing.get("items", [])]
|
|
if before[0] not in names and ready_pods(listing):
|
|
wait_ready(base_url, timeout=max(1, deadline - time.monotonic()))
|
|
return base_url
|
|
time.sleep(0.5)
|
|
raise RuntimeError("replacement pod readiness timeout")
|
|
|
|
|
|
def settings_text(media, issue_db, allowed_hosts=("127.0.0.1",)):
|
|
return (
|
|
"from vergabe_teilnahme.settings.base import *\n"
|
|
f"DEBUG = False\nALLOWED_HOSTS = {list(allowed_hosts)!r}\nNETKINGDOM_ENABLED = False\n"
|
|
f"MEDIA_ROOT = {media!r}\n"
|
|
f"ISSUE_FACADE_LOCAL_DB = {issue_db!r}\n"
|
|
'ISSUE_FACADE_GITEA = None\nFORCE_SCRIPT_NAME = None\nMEDIA_URL = "/media/"\n'
|
|
'EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend"\n'
|
|
"SESSION_COOKIE_SECURE = False\nCSRF_COOKIE_SECURE = False\n"
|
|
)
|
|
|
|
|
|
def restart_container(name):
|
|
subprocess.run(
|
|
["docker", "restart", "--time", "5", name],
|
|
check=True,
|
|
timeout=20,
|
|
stdout=subprocess.DEVNULL,
|
|
)
|
|
# Docker can allocate a different random published port after restart.
|
|
endpoint = container_url(name)
|
|
wait_ready(endpoint)
|
|
return endpoint
|
|
|
|
|
|
def validate(config):
|
|
for key in (
|
|
"users",
|
|
"iterations",
|
|
"tenders",
|
|
"document_bytes",
|
|
"workers",
|
|
"timeout_seconds",
|
|
):
|
|
if type(config[key]) is not int or config[key] < 1:
|
|
raise ValueError(f"{key} must be a positive integer")
|
|
for key in (
|
|
"think_seconds",
|
|
"ordinary_p95_seconds",
|
|
"duration_seconds",
|
|
"idle_seconds",
|
|
):
|
|
if not isinstance(config[key], (int, float)) or not math.isfinite(config[key]):
|
|
raise ValueError(f"{key} must be finite")
|
|
if config["think_seconds"] < 0 or config["ordinary_p95_seconds"] <= 0:
|
|
raise ValueError("invalid timing parameter")
|
|
if config["duration_seconds"] <= 0 or config["idle_seconds"] < 0:
|
|
raise ValueError("duration must be positive and idle nonnegative")
|
|
if config["document_bytes"] < 8:
|
|
raise ValueError("document_bytes must be at least 8")
|
|
if config.get("runtime", "host") not in ("host", "container", "remote"):
|
|
raise ValueError("unsupported runtime")
|
|
if config.get("runtime") == "container":
|
|
if (
|
|
not isinstance(config["app_cpu_limit"], (int, float))
|
|
or not 0 < config["app_cpu_limit"] <= 8
|
|
):
|
|
raise ValueError("app_cpu_limit must be positive and at most 8")
|
|
if not config.get("app_image") or not config.get("app_memory_limit"):
|
|
raise ValueError("container image and memory limit required")
|
|
phase_plan(config)
|
|
return config
|
|
|
|
|
|
def stop_process(server):
|
|
"""Bounded teardown of a process group, including Gunicorn workers."""
|
|
if server is None:
|
|
return
|
|
try:
|
|
os.killpg(server.pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
pass
|
|
try:
|
|
server.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
try:
|
|
os.killpg(server.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
server.wait(timeout=5)
|
|
|
|
|
|
def guarded_sampler(function, errors):
|
|
try:
|
|
function()
|
|
except Exception as exc:
|
|
errors.append({"sampler": function.__name__, "error_type": type(exc).__name__})
|
|
|
|
|
|
def sampling_gaps(samples, db_samples, errors, remote=False):
|
|
gaps = list(errors)
|
|
if remote:
|
|
return gaps
|
|
if any(s.get("sample_error") for s in samples):
|
|
gaps.append({"sampler": "application", "reason": "sample failed"})
|
|
good = [s for s in samples if "monotonic_seconds" in s]
|
|
if len(good) < 2 or not db_samples:
|
|
gaps.append({"sampler": "coverage", "reason": "insufficient samples"})
|
|
if any(
|
|
b["monotonic_seconds"] - a["monotonic_seconds"] > 3
|
|
for a, b in zip(good, good[1:])
|
|
if a.get("phase") == b.get("phase")
|
|
):
|
|
gaps.append({"sampler": "application", "reason": "sampling interval exceeds 3 seconds"})
|
|
return gaps
|
|
|
|
|
|
def run(config, root, failure_output=None):
|
|
# Never consume a deployment database, media tree, session or credential.
|
|
os.environ["DATABASE_URL"] = os.environ["SYNTHETIC_DATABASE_URL"]
|
|
os.environ["SECRET_KEY"] = "synthetic-disposable-not-a-deployment-key"
|
|
os.environ["DJANGO_SETTINGS_MODULE"] = "synthetic_settings"
|
|
container_mode = config.get("runtime", "host") == "container"
|
|
remote_mode = config.get("runtime") == "remote"
|
|
media = "/tmp/synthetic-media" if container_mode else str(root / "media")
|
|
issue_db = "/tmp/synthetic-issues.db" if container_mode else str(root / "issues.db")
|
|
(root / "synthetic_settings.py").write_text(settings_text(media, issue_db))
|
|
sys.path.insert(0, str(root))
|
|
import django
|
|
|
|
django.setup()
|
|
from django.core.management import call_command
|
|
from django.db import connections
|
|
|
|
from vergabe_teilnahme.apps.accounts.models import Mitarbeiter
|
|
from vergabe_teilnahme.apps.aufgaben.models import Aufgabe
|
|
from vergabe_teilnahme.apps.ausschreibungen.models import Ausschreibung
|
|
from vergabe_teilnahme.apps.dokumente.models import Dokument
|
|
from vergabe_teilnahme.apps.lose.models import Los
|
|
|
|
call_command("migrate", verbosity=0)
|
|
for i in range(config["tenders"]):
|
|
Ausschreibung.objects.create(titel=f"Synthetic tender {i}", ausschreiber="Synthetic")
|
|
users = []
|
|
password = uuid.uuid4().hex
|
|
for i in range(config["users"]):
|
|
user = Mitarbeiter.objects.create_user(username=f"synthetic-{i}", password=password)
|
|
users.append(user.username)
|
|
with socket.socket() as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
port = sock.getsockname()[1]
|
|
base_url = f"http://127.0.0.1:{port}"
|
|
env = dict(os.environ, PYTHONPATH=str(root) + os.pathsep + str(Path.cwd()))
|
|
server_log = (root / "server.log").open("w")
|
|
server = None
|
|
app_container = None
|
|
sampling_done = threading.Event()
|
|
sampler_threads = []
|
|
sampler_errors = []
|
|
sampling_lock = threading.Lock()
|
|
try:
|
|
startup_started = time.monotonic()
|
|
if remote_mode:
|
|
base_url = os.environ["SYNTHETIC_BASE_URL"].rstrip("/")
|
|
elif container_mode:
|
|
app_container = os.environ["SYNTHETIC_CONTAINER"] + "-app"
|
|
root.chmod(0o755)
|
|
subprocess.run(
|
|
[
|
|
"docker",
|
|
"run",
|
|
"--detach",
|
|
"--pull=never",
|
|
"--name",
|
|
app_container,
|
|
"--network",
|
|
os.environ["SYNTHETIC_NETWORK"],
|
|
"--publish",
|
|
"127.0.0.1::8000",
|
|
"--cpus",
|
|
str(config["app_cpu_limit"]),
|
|
"--memory",
|
|
config["app_memory_limit"],
|
|
"--pids-limit",
|
|
"256",
|
|
"--mount",
|
|
(
|
|
f"type=bind,src={root / 'synthetic_settings.py'},"
|
|
"dst=/synthetic_settings.py,readonly"
|
|
),
|
|
"--env",
|
|
"PYTHONPATH=/",
|
|
"--env",
|
|
"DJANGO_SETTINGS_MODULE=synthetic_settings",
|
|
"--env",
|
|
"SECRET_KEY=synthetic-disposable-not-a-deployment-key",
|
|
"--env",
|
|
"DATABASE_URL=postgres://synthetic:synthetic-local-only@db:5432/synthetic",
|
|
config["app_image"],
|
|
"gunicorn",
|
|
"vergabe_teilnahme.wsgi:application",
|
|
"--bind",
|
|
"0.0.0.0:8000",
|
|
"--workers",
|
|
str(config["workers"]),
|
|
"--timeout",
|
|
"30",
|
|
],
|
|
check=True,
|
|
timeout=30,
|
|
stdout=subprocess.DEVNULL,
|
|
)
|
|
base_url = container_url(app_container)
|
|
else:
|
|
server = subprocess.Popen(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"gunicorn",
|
|
"vergabe_teilnahme.wsgi:application",
|
|
"--bind",
|
|
f"127.0.0.1:{port}",
|
|
"--workers",
|
|
str(config["workers"]),
|
|
"--timeout",
|
|
"30",
|
|
],
|
|
start_new_session=True,
|
|
env=env,
|
|
stdout=server_log,
|
|
stderr=server_log,
|
|
)
|
|
samples = []
|
|
db_samples = []
|
|
phase = {"name": "startup", "kind": "startup"}
|
|
|
|
def sample():
|
|
ticks_per_second = os.sysconf("SC_CLK_TCK")
|
|
while not sampling_done.is_set():
|
|
if phase.get("kind") == "restart":
|
|
samples.append(
|
|
{
|
|
"at": datetime.now(UTC).isoformat(),
|
|
"phase": phase["name"],
|
|
"expected_unavailable": True,
|
|
"reason": "planned restart",
|
|
}
|
|
)
|
|
sampling_done.wait(0.5)
|
|
continue
|
|
if container_mode:
|
|
with sampling_lock:
|
|
value = subprocess.check_output(
|
|
[
|
|
"docker",
|
|
"exec",
|
|
app_container,
|
|
"sh",
|
|
"-c",
|
|
(
|
|
"cat /sys/fs/cgroup/cpu.stat; echo MEMORY; "
|
|
"cat /sys/fs/cgroup/memory.current"
|
|
),
|
|
],
|
|
text=True,
|
|
timeout=5,
|
|
)
|
|
cpu_text, memory_text = value.split("MEMORY\n")
|
|
counters = {
|
|
k: int(v) for k, v in (line.split() for line in cpu_text.splitlines())
|
|
}
|
|
samples.append(
|
|
{
|
|
"at": datetime.now(UTC).isoformat(),
|
|
"monotonic_seconds": time.monotonic(),
|
|
"phase": phase["name"],
|
|
"cpu_seconds": counters["usage_usec"] / 1e6,
|
|
"rss_bytes": int(memory_text.strip()),
|
|
"memory_kind": "cgroup-memory-current",
|
|
"cpu_stat": counters,
|
|
"pids": [app_container],
|
|
}
|
|
)
|
|
sampling_done.wait(0.5)
|
|
continue
|
|
pids = [server.pid]
|
|
try:
|
|
pids += [
|
|
int(x)
|
|
for x in Path(f"/proc/{server.pid}/task/{server.pid}/children")
|
|
.read_text()
|
|
.split()
|
|
]
|
|
cpu, rss = 0, 0
|
|
for pid in pids:
|
|
fields = Path(f"/proc/{pid}/stat").read_text().split(") ", 1)[1].split()
|
|
cpu += (int(fields[11]) + int(fields[12])) / ticks_per_second
|
|
rss += int(fields[21]) * os.sysconf("SC_PAGE_SIZE")
|
|
samples.append(
|
|
{
|
|
"at": datetime.now(UTC).isoformat(),
|
|
"monotonic_seconds": time.monotonic(),
|
|
"phase": phase["name"],
|
|
"cpu_seconds": cpu,
|
|
"rss_bytes": rss,
|
|
"pids": pids,
|
|
}
|
|
)
|
|
except (OSError, ValueError):
|
|
samples.append({"at": datetime.now(UTC).isoformat(), "sample_error": True})
|
|
sampling_done.wait(0.5)
|
|
|
|
def sample_db():
|
|
while not sampling_done.is_set():
|
|
value = subprocess.check_output(
|
|
[
|
|
"docker",
|
|
"stats",
|
|
"--no-stream",
|
|
"--format",
|
|
"{{json .}}",
|
|
os.environ["SYNTHETIC_CONTAINER"],
|
|
],
|
|
text=True,
|
|
timeout=5,
|
|
)
|
|
db_samples.append(
|
|
{
|
|
"at": datetime.now(UTC).isoformat(),
|
|
"phase": phase["name"],
|
|
"docker_stats": json.loads(value),
|
|
}
|
|
)
|
|
sampling_done.wait(1)
|
|
|
|
sampler = threading.Thread(
|
|
target=guarded_sampler, args=(sample, sampler_errors), daemon=True
|
|
)
|
|
db_sampler = threading.Thread(
|
|
target=guarded_sampler, args=(sample_db, sampler_errors), daemon=True
|
|
)
|
|
if not remote_mode:
|
|
sampler_threads.extend([sampler, db_sampler])
|
|
sampler.start()
|
|
db_sampler.start()
|
|
wait_ready(base_url)
|
|
startup_seconds = time.monotonic() - startup_started
|
|
phase.update(name="idle", kind="idle")
|
|
idle_window = {"name": "idle", "kind": "idle", "started_at": datetime.now(UTC).isoformat()}
|
|
time.sleep(config["idle_seconds"])
|
|
idle_window["finished_at"] = datetime.now(UTC).isoformat()
|
|
plan = phase_plan(config)
|
|
active_plan = []
|
|
phase_results = []
|
|
transport_rebinds = []
|
|
phase_completed = {}
|
|
|
|
def worker(i):
|
|
jar = http.cookiejar.CookieJar()
|
|
|
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
def redirect_request(self, *args, **kwargs):
|
|
return None
|
|
|
|
opener = urllib.request.build_opener(
|
|
urllib.request.HTTPCookieProcessor(jar), NoRedirect()
|
|
)
|
|
rows = []
|
|
|
|
def request(operation, method, path, data=None, expected=200, transfer=False):
|
|
started = time.perf_counter()
|
|
try:
|
|
headers = {}
|
|
body = None
|
|
if method == "post":
|
|
token = next((c.value for c in jar if c.name == "csrftoken"), "")
|
|
fields = dict(data or {}, csrfmiddlewaretoken=token)
|
|
if transfer:
|
|
boundary = "synthetic-" + uuid.uuid4().hex
|
|
body = b""
|
|
for name, value in fields.items():
|
|
body += (
|
|
f"--{boundary}\r\nContent-Disposition: form-data; "
|
|
f'name="{name}"\r\n\r\n{value}\r\n'
|
|
).encode()
|
|
body += (
|
|
(
|
|
f"--{boundary}\r\nContent-Disposition: form-data; "
|
|
'name="datei"; filename="synthetic.pdf"\r\n'
|
|
"Content-Type: application/pdf\r\n\r\n"
|
|
).encode()
|
|
+ b"%PDF-1.4"
|
|
+ b"x" * (config["document_bytes"] - 8)
|
|
+ f"\r\n--{boundary}--\r\n".encode()
|
|
)
|
|
headers["Content-Type"] = f"multipart/form-data; boundary={boundary}"
|
|
else:
|
|
body = urllib.parse.urlencode(fields).encode()
|
|
headers["Referer"] = base_url + path
|
|
req = urllib.request.Request(
|
|
base_url + path, data=body, headers=headers, method=method.upper()
|
|
)
|
|
try:
|
|
response = opener.open(req, timeout=config["timeout_seconds"])
|
|
except urllib.error.HTTPError as exc:
|
|
response = exc
|
|
received = len(response.read())
|
|
status = response.status
|
|
ok = status == expected
|
|
if transfer and method == "get":
|
|
ok = ok and received == config["document_bytes"]
|
|
response.close()
|
|
except Exception:
|
|
status, ok, received = None, False, 0
|
|
rows.append(
|
|
{
|
|
"phase": phase["name"],
|
|
"user": i,
|
|
"operation": operation,
|
|
"transfer": transfer,
|
|
"seconds": time.perf_counter() - started,
|
|
"status": status,
|
|
"ok": ok,
|
|
"response_bytes": received,
|
|
}
|
|
)
|
|
return ok
|
|
|
|
request("login-form", "get", "/accounts/login/")
|
|
barrier.wait(timeout=30)
|
|
request(
|
|
"login",
|
|
"post",
|
|
"/accounts/login/",
|
|
{"username": users[i], "password": password},
|
|
302,
|
|
)
|
|
for n in range(config["iterations"]):
|
|
if time.monotonic() >= deadline:
|
|
break
|
|
title = f"Workload-{phase['name']}-{i}-{n}"
|
|
tender = lot = task = document = None
|
|
base = None
|
|
for index, operation in enumerate(active_plan):
|
|
dependency = DEPENDENCIES[operation]
|
|
entity = {
|
|
"tender-create": tender,
|
|
"lot-create": lot,
|
|
"task-create": task,
|
|
"document-upload": document,
|
|
}
|
|
if dependency and entity[dependency] is None:
|
|
rows.append(
|
|
{
|
|
"phase": phase["name"],
|
|
"user": i,
|
|
"operation": operation,
|
|
"transfer": operation.startswith("document-"),
|
|
"seconds": 0,
|
|
"status": None,
|
|
"ok": False,
|
|
"response_bytes": 0,
|
|
"reason": "dependency operation failed",
|
|
}
|
|
)
|
|
continue
|
|
if operation == "list":
|
|
request(operation, "get", "/ausschreibungen/")
|
|
elif operation == "search":
|
|
request(operation, "get", "/ausschreibungen/?q=Synthetic")
|
|
elif operation == "tender-create":
|
|
if request(
|
|
operation,
|
|
"post",
|
|
"/ausschreibungen/neu/",
|
|
{"titel": title, "ausschreiber": "Synthetic"},
|
|
302,
|
|
):
|
|
tender = Ausschreibung.objects.filter(titel=title).first()
|
|
base = f"/ausschreibungen/{tender.pk}/" if tender else None
|
|
elif operation == "detail":
|
|
request(operation, "get", base)
|
|
elif operation == "tender-edit":
|
|
changed = f"{title} edited-{index}"
|
|
request(
|
|
operation,
|
|
"post",
|
|
base + "bearbeiten/",
|
|
{"titel": changed, "ausschreiber": "Synthetic edited"},
|
|
302,
|
|
)
|
|
tender.refresh_from_db()
|
|
rows[-1]["ok"] &= tender.titel == changed
|
|
elif operation == "lot-create":
|
|
request(
|
|
operation,
|
|
"post",
|
|
base + "lose/neu/",
|
|
{"losnummer": "1", "lostitel": "Synthetic lot"},
|
|
302,
|
|
)
|
|
lot = Los.objects.filter(ausschreibung=tender).first()
|
|
elif operation == "lot-edit":
|
|
changed = f"Edited lot-{index}"
|
|
request(
|
|
operation,
|
|
"post",
|
|
base + f"lose/{lot.pk}/bearbeiten/",
|
|
{"losnummer": "1", "lostitel": changed},
|
|
302,
|
|
)
|
|
lot.refresh_from_db()
|
|
rows[-1]["ok"] &= lot.lostitel == changed
|
|
elif operation == "task-create":
|
|
request(
|
|
operation,
|
|
"post",
|
|
base + "aufgaben/neu/",
|
|
{"titel": "Synthetic task", "typ": "fachlich", "prioritaet": 2},
|
|
302,
|
|
)
|
|
task = Aufgabe.objects.filter(ausschreibung=tender).first()
|
|
elif operation == "task-edit":
|
|
changed = f"Edited task-{index}"
|
|
request(
|
|
operation,
|
|
"post",
|
|
base + f"aufgaben/{task.pk}/bearbeiten/",
|
|
{"titel": changed, "typ": "fachlich", "prioritaet": 1},
|
|
302,
|
|
)
|
|
task.refresh_from_db()
|
|
rows[-1]["ok"] &= task.titel == changed and task.prioritaet == 1
|
|
elif operation == "document-upload":
|
|
request(
|
|
operation,
|
|
"post",
|
|
base + "dokumente/hochladen/",
|
|
{"kategorie": "intern", "version": "1.0"},
|
|
302,
|
|
True,
|
|
)
|
|
document = Dokument.objects.filter(ausschreibung=tender).first()
|
|
elif operation == "document-download":
|
|
request(operation, "get", document.datei.url, transfer=True)
|
|
completed[i] += 1
|
|
time.sleep(active_phase["think_seconds"])
|
|
connections.close_all()
|
|
return rows
|
|
|
|
started = datetime.now(UTC).isoformat()
|
|
before = time.perf_counter()
|
|
rows = []
|
|
for active_phase in plan:
|
|
phase.update(name=active_phase["name"], kind=active_phase["kind"])
|
|
phase_started = time.monotonic()
|
|
phase_started_at = datetime.now(UTC).isoformat()
|
|
if active_phase["kind"] == "restart":
|
|
old_url = base_url
|
|
with sampling_lock:
|
|
if remote_mode:
|
|
base_url = restart_kubernetes_pod(
|
|
base_url, os.environ["SYNTHETIC_RESTART_SELECTOR"]
|
|
)
|
|
else:
|
|
base_url = restart_container(app_container)
|
|
transport_rebinds.append(
|
|
{"phase": active_phase["name"], "before": old_url, "after": base_url}
|
|
)
|
|
phase_results.append(
|
|
{
|
|
"name": active_phase["name"],
|
|
"kind": "restart",
|
|
"seconds": time.monotonic() - phase_started,
|
|
"started_at": phase_started_at,
|
|
"finished_at": datetime.now(UTC).isoformat(),
|
|
"ready": True,
|
|
}
|
|
)
|
|
continue
|
|
active_plan = active_phase["operations"]
|
|
deadline = time.monotonic() + active_phase["duration_seconds"]
|
|
completed = [0] * config["users"]
|
|
barrier = threading.Barrier(config["users"])
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=config["users"]) as pool:
|
|
phase_rows = [
|
|
r for group in pool.map(worker, range(config["users"])) for r in group
|
|
]
|
|
phase_completed[active_phase["name"]] = completed
|
|
phase_ordinary = summarize([r for r in phase_rows if not r["transfer"]])
|
|
phase_transfers = summarize([r for r in phase_rows if r["transfer"]])
|
|
expected = expected_requests(completed, active_plan, config["users"])
|
|
phase_results.append(
|
|
{
|
|
"name": active_phase["name"],
|
|
"kind": "load",
|
|
"seconds": time.monotonic() - phase_started,
|
|
"started_at": phase_started_at,
|
|
"finished_at": datetime.now(UTC).isoformat(),
|
|
"operations": active_plan,
|
|
"expected_requests": expected,
|
|
"actual_requests": len(phase_rows),
|
|
"complete": all(completed) and len(phase_rows) == expected,
|
|
"ordinary": phase_ordinary,
|
|
"document_transfer": phase_transfers,
|
|
}
|
|
)
|
|
rows.extend(phase_rows)
|
|
ordinary = summarize([r for r in rows if not r["transfer"]])
|
|
transfers = summarize([r for r in rows if r["transfer"]])
|
|
complete = all(p.get("complete", p.get("ready", False)) for p in phase_results)
|
|
result = {
|
|
"schema_version": 2,
|
|
"startup_seconds": startup_seconds,
|
|
"idle_window": idle_window,
|
|
"transport_rebinds": transport_rebinds,
|
|
"phase_results": phase_results,
|
|
"started_at": started,
|
|
"finished_at": datetime.now(UTC).isoformat(),
|
|
"duration_seconds": time.perf_counter() - before,
|
|
"config": config,
|
|
"environment": {
|
|
"python": platform.python_version(),
|
|
"django": django.get_version(),
|
|
"platform": platform.platform(),
|
|
"database": os.environ.get(
|
|
"SYNTHETIC_DATABASE_DESCRIPTION", "disposable PostgreSQL 16 container"
|
|
),
|
|
"transport": os.environ.get(
|
|
"SYNTHETIC_TRANSPORT_DESCRIPTION", "loopback HTTP Gunicorn"
|
|
),
|
|
"workers": config["workers"],
|
|
},
|
|
"ordinary": ordinary,
|
|
"document_transfer": transfers,
|
|
"complete": complete,
|
|
"provisional_pass": complete
|
|
and ordinary["errors"] == 0
|
|
and transfers["errors"] == 0
|
|
and all(
|
|
p["ordinary"]["p95_seconds"] <= config["ordinary_p95_seconds"]
|
|
for p in phase_results
|
|
if p["kind"] == "load"
|
|
),
|
|
"limitations": [
|
|
"No browser, ingress, TLS or SSO",
|
|
"No CPU/memory allocation recommendation or human acceptance",
|
|
"No deployed-release hardware or shared-service contention acceptance",
|
|
],
|
|
"operations": rows,
|
|
"runtime_samples": samples,
|
|
"database_samples": db_samples,
|
|
"completed_iterations": phase_completed,
|
|
}
|
|
sampling_done.set()
|
|
for thread in sampler_threads:
|
|
thread.join(timeout=7)
|
|
gaps = sampling_gaps(samples, db_samples, sampler_errors, remote=remote_mode)
|
|
if remote_mode:
|
|
result["resource_sampling"] = "external; align with phase UTC windows"
|
|
if any(thread.is_alive() for thread in sampler_threads):
|
|
gaps.append({"sampler": "cleanup", "reason": "thread did not stop"})
|
|
result["measurement_gaps"] = gaps
|
|
result["measurement_complete"] = not gaps
|
|
if container_mode:
|
|
result["app_container"] = json.loads(
|
|
subprocess.check_output(
|
|
["docker", "inspect", "--format", "{{json .}}", app_container],
|
|
text=True,
|
|
timeout=10,
|
|
)
|
|
)
|
|
# Retain only public runtime identifiers/limits, never full container environment.
|
|
info = result.pop("app_container")
|
|
result["app_runtime"] = {
|
|
"image_id": info["Image"],
|
|
"nano_cpus": info["HostConfig"]["NanoCpus"],
|
|
"memory_bytes": info["HostConfig"]["Memory"],
|
|
"memory_kind": "cgroup-memory-current includes page cache",
|
|
"oom_killed": info["State"]["OOMKilled"],
|
|
"dockerfile_sha256": hashlib.sha256(Path("Dockerfile").read_bytes()).hexdigest(),
|
|
}
|
|
return result
|
|
except Exception as exc:
|
|
if failure_output is not None:
|
|
failure = {
|
|
"schema_version": 2,
|
|
"status": "failed",
|
|
"complete": False,
|
|
"provisional_pass": False,
|
|
"measurement_complete": False,
|
|
"error_type": type(exc).__name__,
|
|
"phase": locals().get("phase", {"name": "startup"}),
|
|
"config": config,
|
|
"phase_results": locals().get("phase_results", []),
|
|
"operations": locals().get("rows", []),
|
|
"runtime_samples": locals().get("samples", []),
|
|
"database_samples": locals().get("db_samples", []),
|
|
"measurement_gaps": [{"reason": "runtime failure; partial evidence only"}],
|
|
"driver_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
|
|
}
|
|
failure_output.write_text(json.dumps(failure, indent=2) + "\n")
|
|
raise
|
|
finally:
|
|
sampling_done.set()
|
|
for thread in sampler_threads:
|
|
if thread.ident is not None:
|
|
thread.join(timeout=7)
|
|
stop_process(server)
|
|
if app_container is not None:
|
|
subprocess.run(
|
|
["docker", "rm", "--force", "--volumes", app_container],
|
|
timeout=15,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
server_log.close()
|
|
connections.close_all()
|
|
|
|
|
|
def host_facts(config):
|
|
return {
|
|
"logical_cpus": os.cpu_count(),
|
|
"affinity_cpus": len(os.sched_getaffinity(0)),
|
|
"cpu_model": next(
|
|
(
|
|
x.split(":", 1)[1].strip()
|
|
for x in Path("/proc/cpuinfo").read_text().splitlines()
|
|
if x.startswith("model name")
|
|
),
|
|
"unknown",
|
|
),
|
|
"memory_total": next(
|
|
x for x in Path("/proc/meminfo").read_text().splitlines() if x.startswith("MemTotal:")
|
|
),
|
|
"cgroup_cpu_max": Path("/sys/fs/cgroup/cpu.max").read_text().strip()
|
|
if Path("/sys/fs/cgroup/cpu.max").exists()
|
|
else "unavailable",
|
|
"app_cpu_quota": config.get("app_cpu_limit", "not imposed by harness"),
|
|
}
|
|
|
|
|
|
def main_remote(config, args):
|
|
"""Drive an existing isolated synthetic deployment; never a shared database."""
|
|
for name in ("SYNTHETIC_BASE_URL", "SYNTHETIC_DATABASE_URL"):
|
|
if not os.environ.get(name):
|
|
raise SystemExit(f"{name} is required for remote runtime")
|
|
with tempfile.TemporaryDirectory(prefix="vergabe-synthetic-") as directory:
|
|
result = run(config, Path(directory), args.output)
|
|
result["source_revision"] = os.environ.get("SYNTHETIC_SOURCE_REVISION", "unrecorded")
|
|
result["driver_sha256"] = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()
|
|
result["host"] = host_facts(config)
|
|
result["config_sha256"] = hashlib.sha256(args.config.read_bytes()).hexdigest()
|
|
args.output.write_text(json.dumps(result, indent=2) + "\n")
|
|
keys = ("ordinary", "document_transfer", "complete", "provisional_pass")
|
|
print(json.dumps({k: result[k] for k in keys}))
|
|
return 0 if result["provisional_pass"] and result["measurement_complete"] else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--config", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
config = validate(json.loads(args.config.read_text()))
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
if config.get("runtime") == "remote":
|
|
sys.exit(main_remote(config, args))
|
|
name = "vergabe-synthetic-" + uuid.uuid4().hex[:12]
|
|
network = name + "-network"
|
|
network_args = []
|
|
try:
|
|
if config.get("runtime") == "container":
|
|
subprocess.run(
|
|
["docker", "network", "create", network],
|
|
check=True,
|
|
timeout=15,
|
|
stdout=subprocess.DEVNULL,
|
|
)
|
|
os.environ["SYNTHETIC_NETWORK"] = network
|
|
network_args = ["--network", network, "--network-alias", "db"]
|
|
# Cached image only, loopback random host port, no existing volume or database.
|
|
subprocess.run(
|
|
[
|
|
"docker",
|
|
"run",
|
|
"--detach",
|
|
"--pull=never",
|
|
"--name",
|
|
name,
|
|
"--publish",
|
|
"127.0.0.1::5432",
|
|
"--env",
|
|
"POSTGRES_DB=synthetic",
|
|
"--env",
|
|
"POSTGRES_USER=synthetic",
|
|
"--env",
|
|
"POSTGRES_PASSWORD=synthetic-local-only",
|
|
*network_args,
|
|
"postgres:16-alpine",
|
|
],
|
|
check=True,
|
|
stdout=subprocess.DEVNULL,
|
|
)
|
|
for _ in range(100):
|
|
ready = subprocess.run(
|
|
["docker", "exec", name, "pg_isready", "-U", "synthetic"],
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
if ready.returncode == 0:
|
|
break
|
|
time.sleep(0.1)
|
|
else:
|
|
raise RuntimeError("isolated PostgreSQL startup failed")
|
|
port = (
|
|
subprocess.check_output(["docker", "port", name, "5432/tcp"], text=True)
|
|
.strip()
|
|
.split(":")[-1]
|
|
)
|
|
os.environ["SYNTHETIC_CONTAINER"] = name
|
|
os.environ["SYNTHETIC_DATABASE_URL"] = (
|
|
f"postgres://synthetic:synthetic-local-only@127.0.0.1:{port}/synthetic"
|
|
)
|
|
with tempfile.TemporaryDirectory(prefix="vergabe-synthetic-") as directory:
|
|
result = run(config, Path(directory), args.output)
|
|
result["postgres_image_id"] = subprocess.check_output(
|
|
["docker", "inspect", "--format", "{{.Image}}", name], text=True
|
|
).strip()
|
|
result["postgres_final_stats"] = json.loads(
|
|
subprocess.check_output(
|
|
["docker", "stats", "--no-stream", "--format", "{{json .}}", name],
|
|
text=True,
|
|
)
|
|
)
|
|
finally:
|
|
subprocess.run(
|
|
["docker", "rm", "--force", "--volumes", name],
|
|
timeout=15,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
if network_args:
|
|
subprocess.run(
|
|
["docker", "network", "rm", network],
|
|
check=True,
|
|
timeout=15,
|
|
stdout=subprocess.DEVNULL,
|
|
)
|
|
result["source_revision"] = subprocess.check_output(
|
|
["git", "rev-parse", "HEAD"], text=True
|
|
).strip()
|
|
result["driver_sha256"] = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()
|
|
app_files = sorted(Path("vergabe_teilnahme").rglob("*.py")) + [
|
|
Path("uv.lock"),
|
|
Path("pyproject.toml"),
|
|
]
|
|
result["app_source_sha256"] = hashlib.sha256(
|
|
b"".join(str(p).encode() + b"\0" + p.read_bytes() for p in app_files)
|
|
).hexdigest()
|
|
result["host"] = {
|
|
"logical_cpus": os.cpu_count(),
|
|
"affinity_cpus": len(os.sched_getaffinity(0)),
|
|
"cpu_model": next(
|
|
(
|
|
x.split(":", 1)[1].strip()
|
|
for x in Path("/proc/cpuinfo").read_text().splitlines()
|
|
if x.startswith("model name")
|
|
),
|
|
"unknown",
|
|
),
|
|
"memory_total": next(
|
|
x for x in Path("/proc/meminfo").read_text().splitlines() if x.startswith("MemTotal:")
|
|
),
|
|
"cgroup_cpu_max": Path("/sys/fs/cgroup/cpu.max").read_text().strip()
|
|
if Path("/sys/fs/cgroup/cpu.max").exists()
|
|
else "unavailable",
|
|
"app_cpu_quota": config.get("app_cpu_limit", "not imposed by harness"),
|
|
}
|
|
result["config_sha256"] = hashlib.sha256(args.config.read_bytes()).hexdigest()
|
|
args.output.write_text(json.dumps(result, indent=2) + "\n")
|
|
print(
|
|
json.dumps(
|
|
{
|
|
k: result[k]
|
|
for k in (
|
|
"ordinary",
|
|
"document_transfer",
|
|
"complete",
|
|
"provisional_pass",
|
|
)
|
|
}
|
|
)
|
|
)
|
|
raise SystemExit(0 if result["provisional_pass"] and result["measurement_complete"] else 1)
|