diff --git a/SCOPE.md b/SCOPE.md index 3dfde1d..6090795 100644 --- a/SCOPE.md +++ b/SCOPE.md @@ -202,3 +202,10 @@ sample passes the two-user ordinary p95<=2s/zero-error criterion; it does not es human acceptance or the production60m CPU request. Coverage and provisional values are in `docs/synthetic-pilot.md` and `docs/evidence/2026-10-19-synthetic-sizing/`. No deployment or publication is performed by these tools. + +The follow-up container profile uses the owner Dockerfile at the existing 1000m CPU +and 1Gi memory limits: the same two-user mix passes locally with ordinary p95 of +0.412s and zero errors. It records cgroup throttling and memory separately from +PostgreSQL. This supports retaining those limits for this provisional scenario; +it does not validate a 60m reservation under contention or predict human demand. +Sampler gaps refuse sizing, and bounded cleanup applies on errors as well as success. diff --git a/config/synthetic-pilot-container.json b/config/synthetic-pilot-container.json new file mode 100644 index 0000000..47fdff5 --- /dev/null +++ b/config/synthetic-pilot-container.json @@ -0,0 +1,16 @@ +{ + "users": 2, + "workers": 2, + "iterations": 100000, + "tenders": 100, + "document_bytes": 1048576, + "think_seconds": 0.2, + "timeout_seconds": 30, + "ordinary_p95_seconds": 2.0, + "duration_seconds": 180, + "idle_seconds": 30, + "runtime": "container", + "app_image": "vergabe-synthetic:local-11f7c8a", + "app_cpu_limit": 1.0, + "app_memory_limit": "1g" +} diff --git a/docs/evidence/2026-10-19-synthetic-sizing/README.md b/docs/evidence/2026-10-19-synthetic-sizing/README.md index 5e44ffd..9502745 100644 --- a/docs/evidence/2026-10-19-synthetic-sizing/README.md +++ b/docs/evidence/2026-10-19-synthetic-sizing/README.md @@ -42,3 +42,34 @@ UTC boundaries are retained as observed and may differ due to wall-clock adjustm Validation: six focused configuration/percentile/sizing tests; Ruff clean for maintained scripts; complete real HTTP run passed all3028 operations. No customer records, external identity, live credentials or production writes were used. + + +## Existing-limit follow-up + +Owner Dockerfile runtime image: `sha256:57da46165ab113d0395e6885a7215a99a8866fe33731150c2985f6115138ea8b`. +Inspected app limits:1000000000 NanoCPUs and1073741824 bytes. No OOM kill. +The same configuration (two users/workers,100 initial tenders,1MiB documents, +0.2s think time,30s idle and180s load) completed258 cycles,2068 requests: +1552 ordinary requests p95 **0.411574s**,516 transfers p95 **0.158373s**,zero errors. +Measurement collection completed with no gaps. Closed-loop throughput falls versus +unbounded execution; the latency criterion still passes at the existing CPU limit. + +Application mean CPU940m; sampled p95 1037m reflects period-boundary measurement, +not a higher configured limit.1449/1807 periods were throttled (about80%); throttled +time counter increased117195212µs. Cgroup memory.current peaked425.59MiB including +page cache, which is not directly comparable with earlier summedRSS. Separate DB +CPU snapshots p95 were60.63% of one core; database demand is outside the app limit. + +Recommendation: retain the existing1000m/1Gi limits for this provisional scenario; +do not adopt the earlier unbounded2900m envelope as a production increase. The60m +request remains unproven under contention. Actual reservation changes require owner +headroom/demand review; new users/corpus/mix, bursts, restart, ingress/SSO or hardware +changes require another measurement. No production settings changed. + +`container-sizing-run.json.gz` retains raw operations, resource samples, exact +maintained-driver digest and runtime provenance. `container-comparison.json` +is reproduced by `scripts/compare_synthetic_limits.py` after decompressing that +input. `container-preflight-failure.json` retains the failed internal-network setup +and verified cleanup; the successful rerun uses a dedicated bridge with only +loopback published ports, not a claimed egress firewall. Maintained source includes +bounded failure cleanup and explicit sampler-gap refusal; ten focused tests pass. diff --git a/docs/evidence/2026-10-19-synthetic-sizing/container-comparison.json b/docs/evidence/2026-10-19-synthetic-sizing/container-comparison.json new file mode 100644 index 0000000..6e4c4e2 --- /dev/null +++ b/docs/evidence/2026-10-19-synthetic-sizing/container-comparison.json @@ -0,0 +1,60 @@ +{ + "status": "provisional-existing-limit-comparison", + "ordinary": { + "requests": 1552, + "errors": 0, + "p95_seconds": 0.41157431800093036 + }, + "document_transfer": { + "requests": 516, + "errors": 0, + "p95_seconds": 0.15837287099930109 + }, + "complete": true, + "provisional_pass": true, + "measurement_complete": true, + "measurement_gaps": [], + "config": { + "users": 2, + "workers": 2, + "iterations": 100000, + "tenders": 100, + "document_bytes": 1048576, + "think_seconds": 0.2, + "timeout_seconds": 30, + "ordinary_p95_seconds": 2.0, + "duration_seconds": 180, + "idle_seconds": 30, + "runtime": "container", + "app_image": "vergabe-synthetic:local-11f7c8a", + "app_cpu_limit": 1.0, + "app_memory_limit": "1g" + }, + "completed_iterations": [ + 129, + 129 + ], + "app_runtime": { + "image_id": "sha256:57da46165ab113d0395e6885a7215a99a8866fe33731150c2985f6115138ea8b", + "nano_cpus": 1000000000, + "memory_bytes": 1073741824, + "memory_kind": "cgroup-memory-current includes page cache", + "oom_killed": false, + "dockerfile_sha256": "154d116566361c632dced1f83678cac5c111b1ac17344421cdba4f06213fa498" + }, + "application_measured": { + "sample_intervals": 256, + "cpu_p95_m": 1037.0409363759723, + "cpu_mean_m": 940.156508347305, + "cgroup_memory_peak_mib": 425.5859375, + "throttled_periods": 1449, + "periods": 1807, + "throttled_usec": 117195212 + }, + "database_separate": { + "samples": 88, + "cpu_p95_percent": 60.63, + "memory_snapshots_kind": "docker usage; full intervals retained raw" + }, + "interpretation": "No production limit increase implied. The inspected limit is tested only for this local synthetic mix. 60m request is not validated as adequate under contention. Memory includes page cache; comparison to earlier summedRSS is not equivalent. No human/ingress/SSO acceptance." +} diff --git a/docs/evidence/2026-10-19-synthetic-sizing/container-preflight-failure.json b/docs/evidence/2026-10-19-synthetic-sizing/container-preflight-failure.json new file mode 100644 index 0000000..9e0ecaf --- /dev/null +++ b/docs/evidence/2026-10-19-synthetic-sizing/container-preflight-failure.json @@ -0,0 +1,9 @@ +{ + "status": "failed-before-workload", + "reason": "Docker internal network suppressed published PostgreSQL loopback port", + "affected_runtime": "vergabe-synthetic-28b8611b54cf", + "workload_requests": 0, + "cleanup_verified": true, + "verification": "docker ps -a and docker network ls filters returned no matching objects", + "correction": "dedicated bridge with loopback-only published ports; no egress isolation claim" +} diff --git a/docs/evidence/2026-10-19-synthetic-sizing/container-sizing-run.json.gz b/docs/evidence/2026-10-19-synthetic-sizing/container-sizing-run.json.gz new file mode 100644 index 0000000..a481fc0 Binary files /dev/null and b/docs/evidence/2026-10-19-synthetic-sizing/container-sizing-run.json.gz differ diff --git a/docs/synthetic-pilot.md b/docs/synthetic-pilot.md index 4c40da2..cad1b5a 100644 --- a/docs/synthetic-pilot.md +++ b/docs/synthetic-pilot.md @@ -80,3 +80,37 @@ folded into the app recommendation. No resource setting is applied by this tool. The configured cycle creates rather than edits tender/lot/task records. Editing, startup timing, restart, a distinct burst and target-platform quota/throttling remain coverage gaps before full pilot sizing acceptance. + +## Existing-limit container comparison + +Build the existing owner runtime target, then use the separate configurable +container profile. Neither command publishes an image or modifies a deployment: + +```sh +docker build --target runtime -t vergabe-synthetic:local-11f7c8a . +.venv/bin/python scripts/synthetic_pilot.py \ + --config config/synthetic-pilot-container.json \ + --output /tmp/vergabe-container-sizing.json +``` + +The profile retains the same workload assumptions and adds `runtime: container`, +`app_image`, `app_cpu_limit: 1.0`, and `app_memory_limit: 1g`. This tests the accepted +1000m CPU limit and 1Gi memory limit; a 60m request is scheduling reservation, not +an execution quota. The image is built from the owner Dockerfile, with its locked +runtime dependencies. App and database are attached to a uniquely named bridge; +only random loopback host ports are published. This is data/process isolation, +not an egress firewall. Synthetic settings disable SSO and external issue routing. + +Container samples read `cpu.stat` (including throttling counters) and +`memory.current` inside its cgroup. For raw-schema compatibility the latter uses +`rss_bytes` with `memory_kind: cgroup-memory-current`; it includes page cache and +is not comparable directly with the earlier host summed-RSS measurement. The +short sampling exec processes also consume a small amount of the container quota. +The database remains separately sampled, outside the application's CPU limit. + +All runtime paths use bounded `finally` cleanup; host Gunicorn runs in its own +process group for TERM/KILL teardown. Sampler failures are retained in +`measurement_gaps`, make `measurement_complete` false, and return a nonzero CLI +status even if HTTP latency passes. The envelope calculator refuses both gaps +and capped runs: a capped run must be compared directly against its limit, not +extrapolated into a recommendation to raise that limit. diff --git a/scripts/compare_synthetic_limits.py b/scripts/compare_synthetic_limits.py new file mode 100644 index 0000000..330426b --- /dev/null +++ b/scripts/compare_synthetic_limits.py @@ -0,0 +1,69 @@ +import argparse +import json +import math +from pathlib import Path + +parser = argparse.ArgumentParser( + description="Compare measured application cgroup limits; never apply resources" +) +parser.add_argument("input", type=Path) +parser.add_argument("--output", type=Path, required=True) +parser.add_argument("--expected-cpu-m", type=int, default=1000) +parser.add_argument("--expected-memory-mib", type=int, default=1024) +args = parser.parse_args() +v = json.loads(args.input.read_text()) +if not v.get("measurement_complete") or v.get("measurement_gaps"): + raise ValueError("measurement gaps prevent a limit comparison") +if v["app_runtime"]["nano_cpus"] != args.expected_cpu_m * 1000000: + raise ValueError("actual CPU limit does not match expected limit") +if v["app_runtime"]["memory_bytes"] != args.expected_memory_mib * 1024**2: + raise ValueError("actual memory limit does not match expected limit") +samples = [s for s in v["runtime_samples"] if s.get("phase") == "ordinary" and "cpu_seconds" in s] +rates = [ + (b["cpu_seconds"] - a["cpu_seconds"]) / (b["monotonic_seconds"] - a["monotonic_seconds"]) * 1000 + for a, b in zip(samples, samples[1:]) +] + + +def p95(values): + return sorted(values)[math.ceil(0.95 * len(values)) - 1] + + +start, end = samples[0]["cpu_stat"], samples[-1]["cpu_stat"] +db = [s["docker_stats"] for s in v["database_samples"] if s["phase"] == "ordinary"] +report = { + "status": "provisional-existing-limit-comparison", + "ordinary": v["ordinary"], + "document_transfer": v["document_transfer"], + "complete": v["complete"], + "provisional_pass": v["provisional_pass"], + "measurement_complete": v["measurement_complete"], + "measurement_gaps": v["measurement_gaps"], + "config": v["config"], + "completed_iterations": v["completed_iterations"], + "app_runtime": v["app_runtime"], + "application_measured": { + "sample_intervals": len(rates), + "cpu_p95_m": p95(rates), + "cpu_mean_m": (samples[-1]["cpu_seconds"] - samples[0]["cpu_seconds"]) + / (samples[-1]["monotonic_seconds"] - samples[0]["monotonic_seconds"]) + * 1000, + "cgroup_memory_peak_mib": max(s["rss_bytes"] for s in samples) / 1024**2, + "throttled_periods": end["nr_throttled"] - start["nr_throttled"], + "periods": end["nr_periods"] - start["nr_periods"], + "throttled_usec": end["throttled_usec"] - start["throttled_usec"], + }, + "database_separate": { + "samples": len(db), + "cpu_p95_percent": p95([float(s["CPUPerc"].rstrip("%")) for s in db]), + "memory_snapshots_kind": "docker usage; full intervals retained raw", + }, + "interpretation": ( + "No production limit increase implied. The inspected limit is tested only for this " + "local synthetic mix. 60m request is not validated as adequate under contention. " + "Memory includes page cache; comparison to earlier summedRSS is not equivalent. " + "No human/ingress/SSO acceptance." + ), +} +args.output.write_text(json.dumps(report, indent=2) + "\n") +print(json.dumps(report, indent=2)) diff --git a/scripts/summarize_synthetic_sizing.py b/scripts/summarize_synthetic_sizing.py index 7ab520a..da59270 100644 --- a/scripts/summarize_synthetic_sizing.py +++ b/scripts/summarize_synthetic_sizing.py @@ -14,6 +14,12 @@ def percentile(values): def recommend(result, margin): + if result.get("measurement_gaps") or result.get("measurement_complete") is False: + raise ValueError("sampling gaps prevent a sizing recommendation") + if result.get("app_runtime"): + raise ValueError( + "quota-capped run requires direct limit comparison, not unbounded extrapolation" + ) samples = result["runtime_samples"] cpu = [] memory = [] diff --git a/scripts/synthetic_pilot.py b/scripts/synthetic_pilot.py index 99ef103..dab01ec 100644 --- a/scripts/synthetic_pilot.py +++ b/scripts/synthetic_pilot.py @@ -2,7 +2,6 @@ """Provisional loopback HTTP workload using Gunicorn and disposable PostgreSQL.""" import argparse -import atexit import concurrent.futures import hashlib import http.cookiejar @@ -10,6 +9,7 @@ import json import math import os import platform +import signal import socket import subprocess import sys @@ -58,19 +58,69 @@ def validate(config): raise ValueError("duration must be positive and idle nonnegative") if config["document_bytes"] < 8: raise ValueError("document_bytes must be at least 8") + if config.get("runtime", "host") not in ("host", "container"): + raise ValueError("unsupported runtime") + if config.get("runtime") == "container": + if ( + not isinstance(config["app_cpu_limit"], (int, float)) + or not 0 < config["app_cpu_limit"] <= 8 + ): + raise ValueError("app_cpu_limit must be positive and at most 8") + if not config.get("app_image") or not config.get("app_memory_limit"): + raise ValueError("container image and memory limit required") return config +def stop_process(server): + """Bounded teardown of a process group, including Gunicorn workers.""" + if server is None: + return + try: + os.killpg(server.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + server.wait(timeout=5) + except subprocess.TimeoutExpired: + try: + os.killpg(server.pid, signal.SIGKILL) + except ProcessLookupError: + pass + server.wait(timeout=5) + + +def guarded_sampler(function, errors): + try: + function() + except Exception as exc: + errors.append({"sampler": function.__name__, "error_type": type(exc).__name__}) + + +def sampling_gaps(samples, db_samples, errors): + gaps = list(errors) + if any(s.get("sample_error") for s in samples): + gaps.append({"sampler": "application", "reason": "sample failed"}) + good = [s for s in samples if "monotonic_seconds" in s] + if len(good) < 2 or not db_samples: + gaps.append({"sampler": "coverage", "reason": "insufficient samples"}) + if any(b["monotonic_seconds"] - a["monotonic_seconds"] > 3 for a, b in zip(good, good[1:])): + gaps.append({"sampler": "application", "reason": "sampling interval exceeds 3 seconds"}) + return gaps + + def run(config, root): # Never consume a deployment database, media tree, session or credential. os.environ["DATABASE_URL"] = os.environ["SYNTHETIC_DATABASE_URL"] os.environ["SECRET_KEY"] = "synthetic-disposable-not-a-deployment-key" os.environ["DJANGO_SETTINGS_MODULE"] = "synthetic_settings" + container_mode = config.get("runtime", "host") == "container" + media = "/tmp/synthetic-media" if container_mode else str(root / "media") + issue_db = "/tmp/synthetic-issues.db" if container_mode else str(root / "issues.db") (root / "synthetic_settings.py").write_text( "from vergabe_teilnahme.settings.base import *\n" 'DEBUG = False\nALLOWED_HOSTS = ["127.0.0.1"]\nNETKINGDOM_ENABLED = False\n' - f"MEDIA_ROOT = {str(root / 'media')!r}\n" - f"ISSUE_FACADE_LOCAL_DB = {str(root / 'issues.db')!r}\n" + f"MEDIA_ROOT = {media!r}\n" + f"ISSUE_FACADE_LOCAL_DB = {issue_db!r}\n" 'ISSUE_FACADE_GITEA = None\nFORCE_SCRIPT_NAME = None\nMEDIA_URL = "/media/"\n' 'EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend"\n' "SESSION_COOKIE_SECURE = False\nCSRF_COOKIE_SECURE = False\n" @@ -100,267 +150,408 @@ def run(config, root): base_url = f"http://127.0.0.1:{port}" env = dict(os.environ, PYTHONPATH=str(root) + os.pathsep + str(Path.cwd())) server_log = (root / "server.log").open("w") - server = subprocess.Popen( - [ - sys.executable, - "-m", - "gunicorn", - "vergabe_teilnahme.wsgi:application", - "--bind", - f"127.0.0.1:{port}", - "--workers", - str(config["workers"]), - "--timeout", - "30", - ], - env=env, - stdout=server_log, - stderr=server_log, - ) - atexit.register(lambda: server.terminate() if server.poll() is None else None) - for _ in range(100): - try: - with urllib.request.urlopen(base_url + "/health/", timeout=1) as response: - if response.status == 200: - break - except OSError: - if server.poll() is not None: - raise RuntimeError("Gunicorn startup failed; see isolated server log") - time.sleep(0.1) - else: - server.terminate() - raise RuntimeError("Gunicorn readiness timeout") - samples = [] - db_samples = [] + server = None + app_container = None sampling_done = threading.Event() - phase = {"name": "idle"} - - def sample(): - ticks_per_second = os.sysconf("SC_CLK_TCK") - while not sampling_done.is_set(): - pids = [server.pid] - try: - pids += [ - int(x) - for x in Path(f"/proc/{server.pid}/task/{server.pid}/children") - .read_text() - .split() - ] - cpu, rss = 0, 0 - for pid in pids: - fields = Path(f"/proc/{pid}/stat").read_text().split(") ", 1)[1].split() - cpu += (int(fields[11]) + int(fields[12])) / ticks_per_second - rss += int(fields[21]) * os.sysconf("SC_PAGE_SIZE") - samples.append( - { - "at": datetime.now(UTC).isoformat(), - "monotonic_seconds": time.monotonic(), - "phase": phase["name"], - "cpu_seconds": cpu, - "rss_bytes": rss, - "pids": pids, - } - ) - except (OSError, ValueError): - samples.append({"at": datetime.now(UTC).isoformat(), "sample_error": True}) - sampling_done.wait(0.5) - - def sample_db(): - while not sampling_done.is_set(): - value = subprocess.check_output( + sampler_threads = [] + sampler_errors = [] + try: + if container_mode: + app_container = os.environ["SYNTHETIC_CONTAINER"] + "-app" + root.chmod(0o755) + subprocess.run( [ "docker", - "stats", - "--no-stream", - "--format", - "{{json .}}", - os.environ["SYNTHETIC_CONTAINER"], + "run", + "--detach", + "--pull=never", + "--name", + app_container, + "--network", + os.environ["SYNTHETIC_NETWORK"], + "--publish", + "127.0.0.1::8000", + "--cpus", + str(config["app_cpu_limit"]), + "--memory", + config["app_memory_limit"], + "--pids-limit", + "256", + "--mount", + ( + f"type=bind,src={root / 'synthetic_settings.py'}," + "dst=/synthetic_settings.py,readonly" + ), + "--env", + "PYTHONPATH=/", + "--env", + "DJANGO_SETTINGS_MODULE=synthetic_settings", + "--env", + "SECRET_KEY=synthetic-disposable-not-a-deployment-key", + "--env", + "DATABASE_URL=postgres://synthetic:synthetic-local-only@db:5432/synthetic", + config["app_image"], + "gunicorn", + "vergabe_teilnahme.wsgi:application", + "--bind", + "0.0.0.0:8000", + "--workers", + str(config["workers"]), + "--timeout", + "30", ], - text=True, + check=True, + timeout=30, + stdout=subprocess.DEVNULL, ) - db_samples.append( - { - "at": datetime.now(UTC).isoformat(), - "phase": phase["name"], - "docker_stats": json.loads(value), - } - ) - sampling_done.wait(1) - - sampler = threading.Thread(target=sample, daemon=True) - db_sampler = threading.Thread(target=sample_db, daemon=True) - sampler.start() - db_sampler.start() - time.sleep(config["idle_seconds"]) - phase["name"] = "ordinary" - deadline = time.monotonic() + config["duration_seconds"] - completed = [0] * config["users"] - barrier = threading.Barrier(config["users"]) - - def worker(i): - jar = http.cookiejar.CookieJar() - - class NoRedirect(urllib.request.HTTPRedirectHandler): - def redirect_request(self, *args, **kwargs): - return None - - opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar), NoRedirect()) - rows = [] - - def request(operation, method, path, data=None, expected=200, transfer=False): - started = time.perf_counter() - try: - headers = {} - body = None - if method == "post": - token = next((c.value for c in jar if c.name == "csrftoken"), "") - fields = dict(data or {}, csrfmiddlewaretoken=token) - if transfer: - boundary = "synthetic-" + uuid.uuid4().hex - body = b"" - for name, value in fields.items(): - body += (f'--{boundary}\r\nContent-Disposition: form-data; ' - f'name="{name}"\r\n\r\n{value}\r\n').encode() - body += ( - (f'--{boundary}\r\nContent-Disposition: form-data; ' - 'name="datei"; filename="synthetic.pdf"\r\n' - 'Content-Type: application/pdf\r\n\r\n').encode() - + b"%PDF-1.4" - + b"x" * (config["document_bytes"] - 8) - + f"\r\n--{boundary}--\r\n".encode() - ) - headers["Content-Type"] = f"multipart/form-data; boundary={boundary}" - else: - body = urllib.parse.urlencode(fields).encode() - headers["Referer"] = base_url + path - req = urllib.request.Request( - base_url + path, data=body, headers=headers, method=method.upper() + port = ( + subprocess.check_output( + ["docker", "port", app_container, "8000/tcp"], text=True, timeout=10 ) - try: - response = opener.open(req, timeout=config["timeout_seconds"]) - except urllib.error.HTTPError as exc: - response = exc - received = len(response.read()) - status = response.status - ok = status == expected - response.close() - except Exception: - status, ok, received = None, False, 0 - rows.append( - { - "user": i, - "operation": operation, - "transfer": transfer, - "seconds": time.perf_counter() - started, - "status": status, - "ok": ok, - "response_bytes": received, - } + .strip() + .split(":")[-1] ) - return ok + base_url = f"http://127.0.0.1:{port}" + else: + server = subprocess.Popen( + [ + sys.executable, + "-m", + "gunicorn", + "vergabe_teilnahme.wsgi:application", + "--bind", + f"127.0.0.1:{port}", + "--workers", + str(config["workers"]), + "--timeout", + "30", + ], + start_new_session=True, + env=env, + stdout=server_log, + stderr=server_log, + ) + for _ in range(100): + try: + with urllib.request.urlopen(base_url + "/health/", timeout=1) as response: + if response.status == 200: + break + except OSError: + if server is not None and server.poll() is not None: + raise RuntimeError("Gunicorn startup failed; see isolated server log") + time.sleep(0.1) + else: + raise RuntimeError("Gunicorn readiness timeout") + samples = [] + db_samples = [] + phase = {"name": "idle"} - request("login-form", "get", "/accounts/login/") - barrier.wait(timeout=30) - request( - "login", - "post", - "/accounts/login/", - {"username": users[i], "password": password}, - 302, + def sample(): + ticks_per_second = os.sysconf("SC_CLK_TCK") + while not sampling_done.is_set(): + if container_mode: + value = subprocess.check_output( + [ + "docker", + "exec", + app_container, + "sh", + "-c", + ( + "cat /sys/fs/cgroup/cpu.stat; echo MEMORY; " + "cat /sys/fs/cgroup/memory.current" + ), + ], + text=True, + timeout=5, + ) + cpu_text, memory_text = value.split("MEMORY\n") + counters = { + k: int(v) for k, v in (line.split() for line in cpu_text.splitlines()) + } + samples.append( + { + "at": datetime.now(UTC).isoformat(), + "monotonic_seconds": time.monotonic(), + "phase": phase["name"], + "cpu_seconds": counters["usage_usec"] / 1e6, + "rss_bytes": int(memory_text.strip()), + "memory_kind": "cgroup-memory-current", + "cpu_stat": counters, + "pids": [app_container], + } + ) + sampling_done.wait(0.5) + continue + pids = [server.pid] + try: + pids += [ + int(x) + for x in Path(f"/proc/{server.pid}/task/{server.pid}/children") + .read_text() + .split() + ] + cpu, rss = 0, 0 + for pid in pids: + fields = Path(f"/proc/{pid}/stat").read_text().split(") ", 1)[1].split() + cpu += (int(fields[11]) + int(fields[12])) / ticks_per_second + rss += int(fields[21]) * os.sysconf("SC_PAGE_SIZE") + samples.append( + { + "at": datetime.now(UTC).isoformat(), + "monotonic_seconds": time.monotonic(), + "phase": phase["name"], + "cpu_seconds": cpu, + "rss_bytes": rss, + "pids": pids, + } + ) + except (OSError, ValueError): + samples.append({"at": datetime.now(UTC).isoformat(), "sample_error": True}) + sampling_done.wait(0.5) + + def sample_db(): + while not sampling_done.is_set(): + value = subprocess.check_output( + [ + "docker", + "stats", + "--no-stream", + "--format", + "{{json .}}", + os.environ["SYNTHETIC_CONTAINER"], + ], + text=True, + timeout=5, + ) + db_samples.append( + { + "at": datetime.now(UTC).isoformat(), + "phase": phase["name"], + "docker_stats": json.loads(value), + } + ) + sampling_done.wait(1) + + sampler = threading.Thread( + target=guarded_sampler, args=(sample, sampler_errors), daemon=True ) - for n in range(config["iterations"]): - if time.monotonic() >= deadline: - break - request("list", "get", "/ausschreibungen/") - request("search", "get", "/ausschreibungen/?q=Synthetic") - title = f"Workload-{i}-{n}" - created = request( - "tender-create", + db_sampler = threading.Thread( + target=guarded_sampler, args=(sample_db, sampler_errors), daemon=True + ) + sampler_threads.extend([sampler, db_sampler]) + sampler.start() + db_sampler.start() + time.sleep(config["idle_seconds"]) + phase["name"] = "ordinary" + deadline = time.monotonic() + config["duration_seconds"] + completed = [0] * config["users"] + barrier = threading.Barrier(config["users"]) + + def worker(i): + jar = http.cookiejar.CookieJar() + + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + + opener = urllib.request.build_opener( + urllib.request.HTTPCookieProcessor(jar), NoRedirect() + ) + rows = [] + + def request(operation, method, path, data=None, expected=200, transfer=False): + started = time.perf_counter() + try: + headers = {} + body = None + if method == "post": + token = next((c.value for c in jar if c.name == "csrftoken"), "") + fields = dict(data or {}, csrfmiddlewaretoken=token) + if transfer: + boundary = "synthetic-" + uuid.uuid4().hex + body = b"" + for name, value in fields.items(): + body += ( + f"--{boundary}\r\nContent-Disposition: form-data; " + f'name="{name}"\r\n\r\n{value}\r\n' + ).encode() + body += ( + ( + f"--{boundary}\r\nContent-Disposition: form-data; " + 'name="datei"; filename="synthetic.pdf"\r\n' + "Content-Type: application/pdf\r\n\r\n" + ).encode() + + b"%PDF-1.4" + + b"x" * (config["document_bytes"] - 8) + + f"\r\n--{boundary}--\r\n".encode() + ) + headers["Content-Type"] = f"multipart/form-data; boundary={boundary}" + else: + body = urllib.parse.urlencode(fields).encode() + headers["Referer"] = base_url + path + req = urllib.request.Request( + base_url + path, data=body, headers=headers, method=method.upper() + ) + try: + response = opener.open(req, timeout=config["timeout_seconds"]) + except urllib.error.HTTPError as exc: + response = exc + received = len(response.read()) + status = response.status + ok = status == expected + response.close() + except Exception: + status, ok, received = None, False, 0 + rows.append( + { + "user": i, + "operation": operation, + "transfer": transfer, + "seconds": time.perf_counter() - started, + "status": status, + "ok": ok, + "response_bytes": received, + } + ) + return ok + + request("login-form", "get", "/accounts/login/") + barrier.wait(timeout=30) + request( + "login", "post", - "/ausschreibungen/neu/", - {"titel": title, "ausschreiber": "Synthetic"}, + "/accounts/login/", + {"username": users[i], "password": password}, 302, ) - tender = Ausschreibung.objects.filter(titel=title).first() if created else None - if tender: - base = f"/ausschreibungen/{tender.pk}/" - request("detail", "get", base) - request( - "lot-create", + for n in range(config["iterations"]): + if time.monotonic() >= deadline: + break + request("list", "get", "/ausschreibungen/") + request("search", "get", "/ausschreibungen/?q=Synthetic") + title = f"Workload-{i}-{n}" + created = request( + "tender-create", "post", - base + "lose/neu/", - {"losnummer": "1", "lostitel": "Synthetic lot"}, + "/ausschreibungen/neu/", + {"titel": title, "ausschreiber": "Synthetic"}, 302, ) - request( - "task-create", - "post", - base + "aufgaben/neu/", - {"titel": "Synthetic task", "typ": "fachlich", "prioritaet": 2}, - 302, - ) - request( - "document-upload", - "post", - base + "dokumente/hochladen/", - {"kategorie": "intern", "version": "1.0"}, - 302, - True, - ) - document = Dokument.objects.filter(ausschreibung=tender).first() - if document: - request("document-download", "get", document.datei.url, transfer=True) - completed[i] += 1 - time.sleep(config["think_seconds"]) - connections.close_all() - return rows + tender = Ausschreibung.objects.filter(titel=title).first() if created else None + if tender: + base = f"/ausschreibungen/{tender.pk}/" + request("detail", "get", base) + request( + "lot-create", + "post", + base + "lose/neu/", + {"losnummer": "1", "lostitel": "Synthetic lot"}, + 302, + ) + request( + "task-create", + "post", + base + "aufgaben/neu/", + {"titel": "Synthetic task", "typ": "fachlich", "prioritaet": 2}, + 302, + ) + request( + "document-upload", + "post", + base + "dokumente/hochladen/", + {"kategorie": "intern", "version": "1.0"}, + 302, + True, + ) + document = Dokument.objects.filter(ausschreibung=tender).first() + if document: + request("document-download", "get", document.datei.url, transfer=True) + completed[i] += 1 + time.sleep(config["think_seconds"]) + connections.close_all() + return rows - started = datetime.now(UTC).isoformat() - before = time.perf_counter() - with concurrent.futures.ThreadPoolExecutor(max_workers=config["users"]) as pool: - rows = [r for group in pool.map(worker, range(config["users"])) for r in group] - ordinary = summarize([r for r in rows if not r["transfer"]]) - transfers = summarize([r for r in rows if r["transfer"]]) - complete = all(completed) and len(rows) == sum(completed) * 8 + config["users"] * 2 - result = { - "schema_version": 1, - "started_at": started, - "finished_at": datetime.now(UTC).isoformat(), - "duration_seconds": time.perf_counter() - before, - "config": config, - "environment": { - "python": platform.python_version(), - "django": django.get_version(), - "platform": platform.platform(), - "database": "disposable PostgreSQL 16 container", - "transport": "loopback HTTP Gunicorn", - "workers": config["workers"], - }, - "ordinary": ordinary, - "document_transfer": transfers, - "complete": complete, - "provisional_pass": complete - and ordinary["errors"] == 0 - and transfers["errors"] == 0 - and ordinary["p95_seconds"] <= config["ordinary_p95_seconds"], - "limitations": [ - "No browser, ingress, TLS, SSO or application container quota", - "No CPU/memory allocation recommendation or human acceptance", - "No timed startup/restart or distinct burst; configured idle and ordinary phases only", - ], - "operations": rows, - "runtime_samples": samples, - "database_samples": db_samples, - "completed_iterations": completed, - } - sampling_done.set() - sampler.join() - db_sampler.join() - server.terminate() - server.wait(timeout=10) - server_log.close() - connections.close_all() - return result + started = datetime.now(UTC).isoformat() + before = time.perf_counter() + with concurrent.futures.ThreadPoolExecutor(max_workers=config["users"]) as pool: + rows = [r for group in pool.map(worker, range(config["users"])) for r in group] + ordinary = summarize([r for r in rows if not r["transfer"]]) + transfers = summarize([r for r in rows if r["transfer"]]) + complete = all(completed) and len(rows) == sum(completed) * 8 + config["users"] * 2 + result = { + "schema_version": 1, + "started_at": started, + "finished_at": datetime.now(UTC).isoformat(), + "duration_seconds": time.perf_counter() - before, + "config": config, + "environment": { + "python": platform.python_version(), + "django": django.get_version(), + "platform": platform.platform(), + "database": "disposable PostgreSQL 16 container", + "transport": "loopback HTTP Gunicorn", + "workers": config["workers"], + }, + "ordinary": ordinary, + "document_transfer": transfers, + "complete": complete, + "provisional_pass": complete + and ordinary["errors"] == 0 + and transfers["errors"] == 0 + and ordinary["p95_seconds"] <= config["ordinary_p95_seconds"], + "limitations": [ + "No browser, ingress, TLS or SSO", + "No CPU/memory allocation recommendation or human acceptance", + "No timed startup/restart or distinct burst; idle and ordinary phases only", + ], + "operations": rows, + "runtime_samples": samples, + "database_samples": db_samples, + "completed_iterations": completed, + } + sampling_done.set() + for thread in sampler_threads: + thread.join(timeout=7) + gaps = sampling_gaps(samples, db_samples, sampler_errors) + if any(thread.is_alive() for thread in sampler_threads): + gaps.append({"sampler": "cleanup", "reason": "thread did not stop"}) + result["measurement_gaps"] = gaps + result["measurement_complete"] = not gaps + if container_mode: + result["app_container"] = json.loads( + subprocess.check_output( + ["docker", "inspect", "--format", "{{json .}}", app_container], + text=True, + timeout=10, + ) + ) + # Retain only public runtime identifiers/limits, never full container environment. + info = result.pop("app_container") + result["app_runtime"] = { + "image_id": info["Image"], + "nano_cpus": info["HostConfig"]["NanoCpus"], + "memory_bytes": info["HostConfig"]["Memory"], + "memory_kind": "cgroup-memory-current includes page cache", + "oom_killed": info["State"]["OOMKilled"], + "dockerfile_sha256": hashlib.sha256(Path("Dockerfile").read_bytes()).hexdigest(), + } + return result + finally: + sampling_done.set() + for thread in sampler_threads: + if thread.ident is not None: + thread.join(timeout=7) + stop_process(server) + if app_container is not None: + subprocess.run( + ["docker", "rm", "--force", "--volumes", app_container], + timeout=15, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + server_log.close() + connections.close_all() if __name__ == "__main__": @@ -371,29 +562,41 @@ if __name__ == "__main__": config = validate(json.loads(args.config.read_text())) sys.path.insert(0, str(Path(__file__).resolve().parents[1])) name = "vergabe-synthetic-" + uuid.uuid4().hex[:12] - # Cached image only, loopback random host port, no existing volume or database. - subprocess.run( - [ - "docker", - "run", - "--detach", - "--pull=never", - "--name", - name, - "--publish", - "127.0.0.1::5432", - "--env", - "POSTGRES_DB=synthetic", - "--env", - "POSTGRES_USER=synthetic", - "--env", - "POSTGRES_PASSWORD=synthetic-local-only", - "postgres:16-alpine", - ], - check=True, - stdout=subprocess.DEVNULL, - ) + network = name + "-network" + network_args = [] try: + if config.get("runtime") == "container": + subprocess.run( + ["docker", "network", "create", network], + check=True, + timeout=15, + stdout=subprocess.DEVNULL, + ) + os.environ["SYNTHETIC_NETWORK"] = network + network_args = ["--network", network, "--network-alias", "db"] + # Cached image only, loopback random host port, no existing volume or database. + subprocess.run( + [ + "docker", + "run", + "--detach", + "--pull=never", + "--name", + name, + "--publish", + "127.0.0.1::5432", + "--env", + "POSTGRES_DB=synthetic", + "--env", + "POSTGRES_USER=synthetic", + "--env", + "POSTGRES_PASSWORD=synthetic-local-only", + *network_args, + "postgres:16-alpine", + ], + check=True, + stdout=subprocess.DEVNULL, + ) for _ in range(100): ready = subprocess.run( ["docker", "exec", name, "pg_isready", "-U", "synthetic"], @@ -428,9 +631,17 @@ if __name__ == "__main__": finally: subprocess.run( ["docker", "rm", "--force", "--volumes", name], - check=True, + timeout=15, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, ) + if network_args: + subprocess.run( + ["docker", "network", "rm", network], + check=True, + timeout=15, + stdout=subprocess.DEVNULL, + ) result["source_revision"] = subprocess.check_output( ["git", "rev-parse", "HEAD"], text=True ).strip() @@ -459,7 +670,7 @@ if __name__ == "__main__": "cgroup_cpu_max": Path("/sys/fs/cgroup/cpu.max").read_text().strip() if Path("/sys/fs/cgroup/cpu.max").exists() else "unavailable", - "app_cpu_quota": "not imposed by harness", + "app_cpu_quota": config.get("app_cpu_limit", "not imposed by harness"), } result["config_sha256"] = hashlib.sha256(args.config.read_bytes()).hexdigest() args.output.write_text(json.dumps(result, indent=2) + "\n") @@ -476,4 +687,4 @@ if __name__ == "__main__": } ) ) - raise SystemExit(0 if result["provisional_pass"] else 1) + raise SystemExit(0 if result["provisional_pass"] and result["measurement_complete"] else 1) diff --git a/scripts/test_synthetic_pilot.py b/scripts/test_synthetic_pilot.py index 5e546af..535163f 100644 --- a/scripts/test_synthetic_pilot.py +++ b/scripts/test_synthetic_pilot.py @@ -1,4 +1,6 @@ import importlib.util +import subprocess +import sys import unittest from pathlib import Path @@ -52,6 +54,33 @@ class SyntheticContractTests(unittest.TestCase): self.assertEqual(self.config()["ordinary_p95_seconds"], 2) +class LifecycleTests(unittest.TestCase): + def test_process_group_cleanup_stops_real_child(self): + process = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(60)"], start_new_session=True + ) + try: + raise RuntimeError("injected worker failure") + except RuntimeError: + pilot.stop_process(process) + self.assertIsNotNone(process.poll()) + + def test_sampler_exception_is_retained_as_gap(self): + errors = [] + + def broken_sampler(): + raise OSError("injected collection failure") + + pilot.guarded_sampler(broken_sampler, errors) + gaps = pilot.sampling_gaps([], [], errors) + self.assertEqual(gaps[0]["error_type"], "OSError") + self.assertTrue(gaps) + + def test_sparse_samples_are_not_complete(self): + rows = [dict(monotonic_seconds=0), dict(monotonic_seconds=5)] + self.assertTrue(pilot.sampling_gaps(rows, [{}], [])) + + class SizingTests(unittest.TestCase): def test_failed_measurement_does_not_recommend(self): spec = importlib.util.spec_from_file_location( @@ -62,6 +91,15 @@ class SizingTests(unittest.TestCase): with self.assertRaises(ValueError): sizing.recommend({"runtime_samples": [], "complete": False}, {}) + def test_sampler_gap_prevents_recommendation(self): + spec = importlib.util.spec_from_file_location( + "sizing", Path(__file__).with_name("summarize_synthetic_sizing.py") + ) + sizing = importlib.util.module_from_spec(spec) + spec.loader.exec_module(sizing) + with self.assertRaisesRegex(ValueError, "sampling gaps"): + sizing.recommend({"measurement_complete": False}, {}) + def test_cpu_delta_and_rounding(self): spec = importlib.util.spec_from_file_location( "sizing", Path(__file__).with_name("summarize_synthetic_sizing.py")