Declare planned FI open-weight reserve bucket
Dedicated Scaleway bucket, no 30-day expiry, strategic/ to Glacier. freedom-intelligence is a potential consumer. Create is operator T08. Assistant: grok Assistant-Session: 01a09c6a-1cfc-75b1-a78d-c13eaf22241d
This commit is contained in:
parent
e3bb6ef09a
commit
117612eae2
3 changed files with 159 additions and 0 deletions
|
|
@ -7,6 +7,13 @@ consumers_potential:
|
||||||
evidence:
|
evidence:
|
||||||
- repo: resource-control
|
- repo: resource-control
|
||||||
path: workplans/RESOURCE-WP-0002-procure-postgres-backup-storage.md
|
path: workplans/RESOURCE-WP-0002-procure-postgres-backup-storage.md
|
||||||
|
- consumer_id: freedom-intelligence
|
||||||
|
resource_id: resource:agents:fi-open-weight-reserve
|
||||||
|
purpose: Open-weight model reserve (R One Zone IA, S Glacier)
|
||||||
|
readiness_state: declared
|
||||||
|
evidence:
|
||||||
|
- repo: freedom-intelligence
|
||||||
|
path: workplans/FI-WP-0004-operational-loop-and-scaleway-reserve.md
|
||||||
consumers_actual:
|
consumers_actual:
|
||||||
- consumer_id: rapp-postgres
|
- consumer_id: rapp-postgres
|
||||||
resource_id: resource:platform:audit-storage
|
resource_id: resource:platform:audit-storage
|
||||||
|
|
@ -17,3 +24,5 @@ consumers_actual:
|
||||||
notes:
|
notes:
|
||||||
- Actual consumer is recorded only after WAL flows (RESOURCE-WP-0002-T05).
|
- Actual consumer is recorded only after WAL flows (RESOURCE-WP-0002-T05).
|
||||||
- rapp-postgres consumes a reviewed destination; it does not procure the bucket.
|
- rapp-postgres consumes a reviewed destination; it does not procure the bucket.
|
||||||
|
- freedom-intelligence is a potential consumer until FI-WP-0004-T08 creates the bucket.
|
||||||
|
- Do not put model weights in railiance-platform-pg-backup (30-day expiry).
|
||||||
|
|
|
||||||
30
substrate/object-stores/fi-open-weight-reserve.yaml
Normal file
30
substrate/object-stores/fi-open-weight-reserve.yaml
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
# Planned attributes for FI-WP-0004. No secrets in this file.
|
||||||
|
# Do NOT copy the postgres backup 30-day expiry onto this bucket.
|
||||||
|
schema_version: "0.1"
|
||||||
|
resource_id: resource:agents:fi-open-weight-reserve
|
||||||
|
provider: Scaleway
|
||||||
|
product: onezone-ia-plus-glacier
|
||||||
|
status: planned
|
||||||
|
endpoint: https://s3.nl-ams.scw.cloud
|
||||||
|
region: nl-ams
|
||||||
|
bucket: railiance-fi-open-weight-reserve
|
||||||
|
prefix: ""
|
||||||
|
versioning: true
|
||||||
|
lifecycle: "no object expiry; prefix strategic/ → Glacier after 1 day"
|
||||||
|
provider_project_ref: e1a0dd0e-04b8-4ea9-8b30-d53f8c35c688
|
||||||
|
intended:
|
||||||
|
public_access: disabled
|
||||||
|
versioning: true
|
||||||
|
lifecycle: long retention (open-weight reserve, not WAL)
|
||||||
|
encryption_at_rest: provider-managed
|
||||||
|
identity: bucket-scoped key, distinct from postgres backup
|
||||||
|
storage_classes:
|
||||||
|
models: ONEZONE_IA
|
||||||
|
strategic: GLACIER
|
||||||
|
notes:
|
||||||
|
- "Consumer: freedom-intelligence (catalog in git, blobs here)."
|
||||||
|
- "Decision: freedom-intelligence/docs/decisions/2026-09-13-scaleway-object-reserve.md"
|
||||||
|
- "Soft budget EUR 15/month. Founder Scaleway billing alert is the hard backstop."
|
||||||
|
- "NEVER apply the 30-day expiry used on railiance-platform-pg-backup."
|
||||||
|
- "Bucket create is FI-WP-0004-T08 (operator). This file stays planned until then."
|
||||||
|
- "resource-control may later cite reef:storage/substrate/object-stores/fi-open-weight-reserve.yaml"
|
||||||
120
tools/create-fi-open-weight-reserve-bucket.sh
Executable file
120
tools/create-fi-open-weight-reserve-bucket.sh
Executable file
|
|
@ -0,0 +1,120 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Create the Freedom Intelligence open-weight reserve bucket.
|
||||||
|
# Reads bootstrap creds from OpenBao. Never prints secret values.
|
||||||
|
# Writes only non-secret attributes into
|
||||||
|
# substrate/object-stores/fi-open-weight-reserve.yaml.
|
||||||
|
#
|
||||||
|
# MUST NOT apply the 30-day expiry used on the postgres backup bucket.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
ATTR="$ROOT/substrate/object-stores/fi-open-weight-reserve.yaml"
|
||||||
|
BAO_ADDR="${BAO_ADDR:-${VAULT_ADDR:-https://bao.coulomb.social}}"
|
||||||
|
BOOTSTRAP_PATH="platform/workloads/railiance/scaleway/bootstrap"
|
||||||
|
REGION="nl-ams"
|
||||||
|
BUCKET="${BUCKET:-railiance-fi-open-weight-reserve}"
|
||||||
|
ENDPOINT="https://s3.nl-ams.scw.cloud"
|
||||||
|
|
||||||
|
need() { command -v "$1" >/dev/null || { echo "missing $1" >&2; exit 2; }; }
|
||||||
|
export PATH="${HOME}/.local/bin:${PATH}"
|
||||||
|
need python3
|
||||||
|
need curl
|
||||||
|
need scw
|
||||||
|
|
||||||
|
TOKEN="${OPENBAO_TOKEN:-${VAULT_TOKEN:-}}"
|
||||||
|
if [[ -z "$TOKEN" && -f "$HOME/.vault-token" ]]; then
|
||||||
|
TOKEN="$(cat "$HOME/.vault-token")"
|
||||||
|
fi
|
||||||
|
[[ -n "$TOKEN" ]] || { echo "no OpenBao token" >&2; exit 2; }
|
||||||
|
|
||||||
|
read_kv() {
|
||||||
|
local path="$1"
|
||||||
|
curl -fsS -H "X-Vault-Token: $TOKEN" \
|
||||||
|
"$BAO_ADDR/v1/platform/data/${path#platform/}" \
|
||||||
|
| python3 -c 'import json,sys; d=json.load(sys.stdin); print(json.dumps(d["data"]["data"]))'
|
||||||
|
}
|
||||||
|
|
||||||
|
if ! BOOTSTRAP_JSON="$(read_kv workloads/railiance/scaleway/bootstrap 2>/dev/null)"; then
|
||||||
|
echo "Bootstrap secret missing at $BOOTSTRAP_PATH" >&2
|
||||||
|
echo "See docs/put-scaleway-bootstrap.md" >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
|
||||||
|
eval "$(BOOTSTRAP_JSON="$BOOTSTRAP_JSON" python3 - <<'PY'
|
||||||
|
import json, os, sys
|
||||||
|
data = json.loads(os.environ["BOOTSTRAP_JSON"])
|
||||||
|
needed = ("ACCESS_KEY", "SECRET_KEY", "DEFAULT_ORGANIZATION_ID", "DEFAULT_PROJECT_ID")
|
||||||
|
missing = [k for k in needed if not (data.get(k) or data.get(k.lower()))]
|
||||||
|
if missing:
|
||||||
|
sys.stderr.write("bootstrap fields missing: " + ",".join(missing) + "\n")
|
||||||
|
sys.exit(4)
|
||||||
|
def g(k):
|
||||||
|
return data.get(k) or data.get(k.lower())
|
||||||
|
print("export SCW_ACCESS_KEY=" + json.dumps(g("ACCESS_KEY")))
|
||||||
|
print("export SCW_SECRET_KEY=" + json.dumps(g("SECRET_KEY")))
|
||||||
|
print("export SCW_DEFAULT_ORGANIZATION_ID=" + json.dumps(g("DEFAULT_ORGANIZATION_ID")))
|
||||||
|
print("export SCW_DEFAULT_PROJECT_ID=" + json.dumps(g("DEFAULT_PROJECT_ID")))
|
||||||
|
PY
|
||||||
|
)"
|
||||||
|
|
||||||
|
export SCW_DEFAULT_REGION="$REGION"
|
||||||
|
|
||||||
|
if scw object bucket get "$BUCKET" region="$REGION" -o json >/dev/null 2>&1; then
|
||||||
|
echo "bucket $BUCKET already exists in $REGION; skipping create"
|
||||||
|
else
|
||||||
|
echo "creating private bucket $BUCKET in $REGION (versioning on, NO 30-day expiry)"
|
||||||
|
scw object bucket create "$BUCKET" region="$REGION" acl=private enable-versioning=true -o json
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "applying lifecycle: strategic/ → Glacier after 1 day; no expiry"
|
||||||
|
VENV="${TMPDIR:-/tmp}/reef-storage-boto3"
|
||||||
|
if [[ ! -x "$VENV/bin/python" ]]; then
|
||||||
|
python3 -m venv "$VENV"
|
||||||
|
"$VENV/bin/pip" -q install boto3
|
||||||
|
fi
|
||||||
|
BUCKET="$BUCKET" AWS_ACCESS_KEY_ID="$SCW_ACCESS_KEY" AWS_SECRET_ACCESS_KEY="$SCW_SECRET_KEY" \
|
||||||
|
"$VENV/bin/python" - <<'PY'
|
||||||
|
import os
|
||||||
|
import boto3
|
||||||
|
client = boto3.client(
|
||||||
|
"s3",
|
||||||
|
region_name="nl-ams",
|
||||||
|
endpoint_url="https://s3.nl-ams.scw.cloud",
|
||||||
|
aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
|
||||||
|
aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
|
||||||
|
)
|
||||||
|
client.put_bucket_lifecycle_configuration(
|
||||||
|
Bucket=os.environ["BUCKET"],
|
||||||
|
LifecycleConfiguration={
|
||||||
|
"Rules": [
|
||||||
|
{
|
||||||
|
"ID": "strategic-to-glacier",
|
||||||
|
"Status": "Enabled",
|
||||||
|
"Filter": {"Prefix": "strategic/"},
|
||||||
|
"Transitions": [{"Days": 1, "StorageClass": "GLACIER"}],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ID": "staging-abort-incomplete",
|
||||||
|
"Status": "Enabled",
|
||||||
|
"Filter": {"Prefix": "staging/"},
|
||||||
|
"AbortIncompleteMultipartUpload": {"DaysAfterInitiation": 7},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
print("lifecycle applied (no object expiry)")
|
||||||
|
PY
|
||||||
|
|
||||||
|
echo "writing non-secret attributes (no keys)"
|
||||||
|
python3 - <<PY
|
||||||
|
from pathlib import Path
|
||||||
|
p = Path("$ATTR")
|
||||||
|
text = p.read_text()
|
||||||
|
text = text.replace("status: planned", "status: active", 1)
|
||||||
|
p.write_text(text)
|
||||||
|
print(f"updated {p} status=active")
|
||||||
|
print(f"endpoint={ENDPOINT} bucket={BUCKET} region={REGION}")
|
||||||
|
PY
|
||||||
|
|
||||||
|
echo "done. Commit the YAML; do not commit any key."
|
||||||
|
echo "Scoped FI key should be a distinct OpenBao path from postgres backup."
|
||||||
Loading…
Add table
Add a link
Reference in a new issue