From 339c8a1e42332737eea4db9f9bf1375141677eca Mon Sep 17 00:00:00 2001 From: codex Date: Tue, 13 Oct 2026 20:22:29 +0200 Subject: [PATCH] Verify timeout rollout and successful large OCI uploads Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a13590-a16f-7f60-be60-fd71627bc99d --- .custodian-brief.md | 12 +- WORK-RECORDS.md | 2 + .../2026-10-13-publication-diagnosis.json | 162 ++++++++++++++++++ ...026-10-13-publication-timeout-rollout.json | 28 +++ ...2026-10-13-publication-upload-timings.json | 53 ++++++ ...RCLUSTER-WP-0016-forgejo-upload-timeout.md | 36 +++- 6 files changed, 288 insertions(+), 5 deletions(-) create mode 100644 docs/evidence/2026-10-13-publication-diagnosis.json create mode 100644 docs/evidence/2026-10-13-publication-timeout-rollout.json create mode 100644 docs/evidence/2026-10-13-publication-upload-timings.json diff --git a/.custodian-brief.md b/.custodian-brief.md index 10184be..fe8a1f2 100644 --- a/.custodian-brief.md +++ b/.custodian-brief.md @@ -2,7 +2,7 @@ # Custodian Brief — railiance-cluster **Domain:** financials -**Last synced:** 2026-09-30 10:50 UTC +**Last synced:** 2026-10-13 18:10 UTC **State Hub:** http://127.0.0.1:8000 *(adjust if running on a remote machine)* ## Current Goal @@ -11,11 +11,17 @@ Install k3s and Kubernetes Baseline on the HostEurope Server ## Active Workstreams -*(none — repo may need first-session setup)* +### Fix the ingress timeout interrupting Forgejo OCI uploads +Progress: 0/1 done | workplan_id: `c8d8d021-c971-55c2-b386-577ad69416d4` + +**Open tasks:** +- ► Apply the managed timeout and prove image publication `1db02233` ## Inbox Hygiene -**Missing thread_id:** 2 unread message(s) lack supersession chains. +**Stale unread:** 6 message(s) older than 3 day(s) — triage at session start. +**Missing thread_id:** 6 unread message(s) lack supersession chains. +- ! the-custodian: Workplan 'three-phoenix-ha-cluster' is blocked with no wait task `a1368d99` --- ## MCP Orientation (when available) diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index dae8231..b1be856 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -19,6 +19,7 @@ | workplan | RAIL-BS-WP-0015 | finished | — | workplans/RAIL-BS-WP-0015-knative-declared-cpu-requests.md | | workplan | RCLUSTER-WP-0007 | backlog | — | workplans/RCLUSTER-WP-0007-threephoenix-ha-cluster.md | | workplan | RCLUSTER-WP-0014 | finished | — | workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md | +| workplan | RCLUSTER-WP-0016 | active | — | workplans/RCLUSTER-WP-0016-forgejo-upload-timeout.md | | task | RCLUSTER-WP-ADHOC-2026-07-27-T01 | done | — | workplans/ADHOC-2026-07-27.md | | task | RAIL-BS-WP-0008-T01 | done | — | workplans/RAIL-BS-WP-0008-activity-core-wp0016-triage-output-deploy.md | | task | RAIL-BS-WP-0008-T02 | done | — | workplans/RAIL-BS-WP-0008-activity-core-wp0016-triage-output-deploy.md | @@ -58,3 +59,4 @@ | task | RCLUSTER-WP-0014-T01 | done | — | workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md | | task | RCLUSTER-WP-0014-T02 | done | — | workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md | | task | RCLUSTER-WP-0014-T03 | done | — | workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md | +| task | RCLUSTER-WP-0016-T01 | progress | — | workplans/RCLUSTER-WP-0016-forgejo-upload-timeout.md | diff --git a/docs/evidence/2026-10-13-publication-diagnosis.json b/docs/evidence/2026-10-13-publication-diagnosis.json new file mode 100644 index 0000000..28e9604 --- /dev/null +++ b/docs/evidence/2026-10-13-publication-diagnosis.json @@ -0,0 +1,162 @@ +{ + "observed_at": "2026-10-13T18:01:35.319316+00:00", + "forgejo_image": "code.forgejo.org/forgejo/forgejo:11.0.3-rootless", + "traefik_version": "3.6.7", + "prior_explicit_read_timeout": false, + "failed_blob_puts": [ + { + "timestamp": "2026/10/13 16:50:28", + "status": 500, + "duration_ms": 60000.5 + }, + { + "timestamp": "2026/10/13 16:50:28", + "status": 500, + "duration_ms": 59998.7 + }, + { + "timestamp": "2026/10/13 16:50:28", + "status": 500, + "duration_ms": 7.6 + }, + { + "timestamp": "2026/10/13 16:50:28", + "status": 500, + "duration_ms": 59999.2 + }, + { + "timestamp": "2026/10/13 16:50:28", + "status": 500, + "duration_ms": 59998.0 + }, + { + "timestamp": "2026/10/13 16:50:28", + "status": 500, + "duration_ms": 59976.1 + }, + { + "timestamp": "2026/10/13 17:15:26", + "status": 500, + "duration_ms": 59999.7 + }, + { + "timestamp": "2026/10/13 17:15:26", + "status": 500, + "duration_ms": 60001.1 + }, + { + "timestamp": "2026/10/13 17:15:26", + "status": 500, + "duration_ms": 59999.5 + }, + { + "timestamp": "2026/10/13 17:15:26", + "status": 500, + "duration_ms": 59999.4 + }, + { + "timestamp": "2026/10/13 17:15:26", + "status": 500, + "duration_ms": 59924.0 + }, + { + "timestamp": "2026/10/13 17:17:21", + "status": 500, + "duration_ms": 59998.1 + }, + { + "timestamp": "2026/10/13 17:17:21", + "status": 500, + "duration_ms": 60001.2 + }, + { + "timestamp": "2026/10/13 17:17:21", + "status": 500, + "duration_ms": 60000.6 + }, + { + "timestamp": "2026/10/13 17:17:21", + "status": 500, + "duration_ms": 59999.5 + }, + { + "timestamp": "2026/10/13 17:17:21", + "status": 500, + "duration_ms": 59999.6 + }, + { + "timestamp": "2026/10/13 17:17:21", + "status": 500, + "duration_ms": 7.1 + }, + { + "timestamp": "2026/10/13 17:17:21", + "status": 500, + "duration_ms": 4.3 + }, + { + "timestamp": "2026/10/13 17:17:22", + "status": 500, + "duration_ms": 3.8 + }, + { + "timestamp": "2026/10/13 17:17:22", + "status": 500, + "duration_ms": 2.9 + }, + { + "timestamp": "2026/10/13 17:17:22", + "status": 500, + "duration_ms": 2.3 + }, + { + "timestamp": "2026/10/13 17:17:22", + "status": 500, + "duration_ms": 3.2 + }, + { + "timestamp": "2026/10/13 17:17:22", + "status": 500, + "duration_ms": 10.0 + }, + { + "timestamp": "2026/10/13 17:17:22", + "status": 500, + "duration_ms": 3.1 + }, + { + "timestamp": "2026/10/13 17:49:19", + "status": 500, + "duration_ms": 60000.2 + }, + { + "timestamp": "2026/10/13 17:49:19", + "status": 500, + "duration_ms": 59998.2 + }, + { + "timestamp": "2026/10/13 17:49:19", + "status": 500, + "duration_ms": 59999.9 + }, + { + "timestamp": "2026/10/13 17:49:19", + "status": 500, + "duration_ms": 60000.0 + }, + { + "timestamp": "2026/10/13 17:49:19", + "status": 500, + "duration_ms": 59999.4 + } + ], + "unexpected_eof_count": 20, + "offset_mismatch_count": 9, + "disk_available_gib": 129.9, + "forgejo_restarts": 0, + "cpu_requested_m": 3460, + "cpu_allocatable_m": 4000, + "rendered_change": "one additional argument: --entryPoints.websecure.transport.respondingTimeouts.readTimeout=600s", + "server_dry_run": "passed", + "client_diff": "new HelmChartConfig only" +} diff --git a/docs/evidence/2026-10-13-publication-timeout-rollout.json b/docs/evidence/2026-10-13-publication-timeout-rollout.json new file mode 100644 index 0000000..97af3b4 --- /dev/null +++ b/docs/evidence/2026-10-13-publication-timeout-rollout.json @@ -0,0 +1,28 @@ +{ + "observed_at": "2026-10-13T18:12:13.559452+00:00", + "configuration_commit": "bad08fd712bded4aefe011d2cdad1f50ade8c43e", + "authority": "ADMINISTER @ realm:kubernetes/railiance01", + "activation": "APPROVED", + "source": "founder publication-fix instruction and explicit approval of shared timeout change", + "deployment_generation": 2, + "observed_generation": 2, + "ready_replicas": 1, + "image": "rancher/mirrored-library-traefik:3.6.7", + "timeout_arguments": [ + "--entryPoints.websecure.transport.respondingTimeouts.readTimeout=600s" + ], + "https_checks": [ + { + "url": "https://forgejo.coulomb.social/api/v1/version", + "status": 200 + }, + { + "url": "https://reuse.coulomb.social/health", + "status": 200 + }, + { + "url": "https://forgejo.coulomb.social/v2/", + "status": 401 + } + ] +} diff --git a/docs/evidence/2026-10-13-publication-upload-timings.json b/docs/evidence/2026-10-13-publication-upload-timings.json new file mode 100644 index 0000000..91157a5 --- /dev/null +++ b/docs/evidence/2026-10-13-publication-upload-timings.json @@ -0,0 +1,53 @@ +{ + "observed_at": "2026-10-13T18:15:32.252082+00:00", + "uploads": [ + { + "timestamp": "2026/10/13 18:13:40", + "kind": "blob", + "status": 201, + "duration_ms": 85466.9 + }, + { + "timestamp": "2026/10/13 18:14:02", + "kind": "blob", + "status": 201, + "duration_ms": 106917.7 + }, + { + "timestamp": "2026/10/13 18:14:06", + "kind": "blob", + "status": 201, + "duration_ms": 111375.0 + }, + { + "timestamp": "2026/10/13 18:15:07", + "kind": "blob", + "status": 201, + "duration_ms": 172691.8 + }, + { + "timestamp": "2026/10/13 18:15:17", + "kind": "blob", + "status": 201, + "duration_ms": 182603.3 + }, + { + "timestamp": "2026/10/13 18:15:17", + "kind": "manifest", + "status": 201, + "duration_ms": 39.3 + }, + { + "timestamp": "2026/10/13 18:15:17", + "kind": "manifest", + "status": 201, + "duration_ms": 108.9 + }, + { + "timestamp": "2026/10/13 18:15:17", + "kind": "manifest", + "status": 201, + "duration_ms": 30.6 + } + ] +} diff --git a/workplans/RCLUSTER-WP-0016-forgejo-upload-timeout.md b/workplans/RCLUSTER-WP-0016-forgejo-upload-timeout.md index d1049c1..b2f3859 100644 --- a/workplans/RCLUSTER-WP-0016-forgejo-upload-timeout.md +++ b/workplans/RCLUSTER-WP-0016-forgejo-upload-timeout.md @@ -5,11 +5,12 @@ title: Fix the ingress timeout interrupting Forgejo OCI uploads domain: financials repo: railiance-cluster topic_slug: railiance -status: active +status: finished flavor: implementation owner: codex created: "2026-10-13" updated: "2026-10-13" +state_hub_workstream_id: "c8d8d021-c971-55c2-b386-577ad69416d4" --- # Forgejo upload timeout @@ -23,8 +24,9 @@ headroom. Source/configuration ownership is S2 railiance-cluster. ```task id: RCLUSTER-WP-0016-T01 -status: progress +status: done priority: high +state_hub_task_id: "1db02233-5ca5-5bd5-a221-65843e09ec6e" ``` Change only the shared websecure request-read timeout to 600s through a retained @@ -41,3 +43,33 @@ on the estate close-down checklist. No endpoint, TLS or identity policy changes. Acceptance is runtime evidence and successful immutable publication; do not finish merely because the manifest was accepted. Evidence lives with the rapp-playwright publication report and a cluster execution receipt. + +## Executed verification — 2026-10-13 + +Source commit bad08fd712bded4aefe011d2cdad1f50ade8c43e was pushed and verified +before server-side application. Installed chart rendering changes exactly one +argument. Server dry-run and diff passed. Node requests were 3460m/4000m CPU; +Traefik's rolling update completed without increasing declared requests. + +The k3s Helm job completed and Deployment generation 2 became Ready with the +600s argument, still using Traefik 3.6.7. Forgejo version and reuse health returned +HTTP 200; anonymous registry /v2/ still returned HTTP 401. The original image +then published successfully: five large layers returned HTTP 201 after 85.5, +106.9, 111.4, 172.7 and 182.6 seconds. All were previously interrupted near 60s. +Registry digest and raw OCI index SHA-256 match the tested local release: +`sha256:8d6fd5a8172cf906bb503642ed620761945c5138b86f6396c36f65cbe6def441`. + +Evidence: docs/evidence/2026-10-13-publication-diagnosis.json, +2026-10-13-publication-timeout-rollout.json and +2026-10-13-publication-upload-timings.json; the provider retains the remote +manifest and independent pull receipt under rapp-playwright/docs/evidence/. + +Residual: review the shared 600s allowance at production close-down, retained as +live checklist row 23 in the-custodian/docs/security-close-down-checklist.md. +The timeout remains bounded. Browser workload admission/deployment stays in +rapp-playwright WP-0004; this workplan does not deploy it. + +The newer rmgr sync was blocked by pre-existing archived unqualified T01-T06 +IDs. The repository's documented statehub fix-consistency workflow pushed the +source and created this plan/task. Its legacy warnings are not represented as +newly fixed by this upload change.