docs: preserve reviewed NetKingdom session artifacts
All checks were successful
CI Smoke / host-smoke (push) Successful in 1s
CI Smoke / container-smoke (push) Successful in 3s
Reject generated-only commits / generated-commit-guard (push) Successful in 9s

This commit is contained in:
codex 2026-11-01 22:54:51 +01:00
parent d3fb789505
commit 70fcd0a6f0
50 changed files with 8055 additions and 0 deletions

View file

@ -0,0 +1,45 @@
"""Simple file-backed idempotency for dual-run commands."""
from __future__ import annotations
import hashlib
import json
import os
from pathlib import Path
from typing import Any
def _store_path() -> Path:
raw = os.environ.get("RM_IDEMPOTENCY_PATH", "~/.repo-manager/idempotency.json")
return Path(raw).expanduser()
def _load() -> dict[str, Any]:
path = _store_path()
if not path.is_file():
return {}
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return {}
def _save(data: dict[str, Any]) -> None:
path = _store_path()
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
def payload_hash(payload: dict[str, Any]) -> str:
blob = json.dumps(payload, sort_keys=True, default=str)
return hashlib.sha256(blob.encode()).hexdigest()
def get(key: str) -> dict[str, Any] | None:
return _load().get(key)
def put(key: str, payload_hash_value: str, result: dict[str, Any]) -> None:
data = _load()
data[key] = {"payload_hash": payload_hash_value, "result": result}
_save(data)

View file

@ -0,0 +1,109 @@
"""Local JSON projection store for the vertical-slice proof (pre-Postgres)."""
from __future__ import annotations
import json
import os
import tempfile
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any
from repo_manager.time import utc_now_text
def _now() -> str:
return utc_now_text()
@dataclass
class WorkRecordEntry:
kind: str
id: str | None
status: str | None
title: str | None
source_path: str
uuid: str | None = None
parent_id: str | None = None
extra: dict[str, Any] = field(default_factory=dict)
@dataclass
class RepoIndex:
slug: str
repo_root: str
head_sha: str | None
observed_at: str
source_fingerprint: str | None = None
source_files: list[str] = field(default_factory=list)
work_records: list[WorkRecordEntry] = field(default_factory=list)
events: list[dict[str, Any]] = field(default_factory=list)
schema: str = "repo_manager.index.v1"
def to_dict(self) -> dict[str, Any]:
return {
"schema": self.schema,
"slug": self.slug,
"repo_root": self.repo_root,
"head_sha": self.head_sha,
"observed_at": self.observed_at,
"source_fingerprint": self.source_fingerprint,
"source_files": self.source_files,
"work_records": [asdict(r) for r in self.work_records],
"events": self.events,
}
@classmethod
def from_dict(cls, data: dict[str, Any]) -> RepoIndex:
records = [
WorkRecordEntry(**{k: v for k, v in r.items() if k in WorkRecordEntry.__dataclass_fields__})
for r in data.get("work_records", [])
]
return cls(
slug=data["slug"],
repo_root=data["repo_root"],
head_sha=data.get("head_sha"),
observed_at=data.get("observed_at") or _now(),
source_fingerprint=data.get("source_fingerprint"),
source_files=list(data.get("source_files") or []),
work_records=records,
events=list(data.get("events") or []),
schema=data.get("schema") or "repo_manager.index.v0",
)
def default_index_path(repo_root: Path) -> Path:
return repo_root / ".repo-manager" / "index.json"
def save_index(index: RepoIndex, path: Path | None = None) -> Path:
path = path or default_index_path(Path(index.repo_root))
path.parent.mkdir(parents=True, exist_ok=True)
temporary: Path | None = None
try:
with tempfile.NamedTemporaryFile(
mode="w",
encoding="utf-8",
dir=path.parent,
prefix=f".{path.name}.",
delete=False,
) as handle:
temporary = Path(handle.name)
handle.write(json.dumps(index.to_dict(), indent=2) + "\n")
handle.flush()
os.fsync(handle.fileno())
os.replace(temporary, path)
temporary = None
finally:
if temporary is not None:
temporary.unlink(missing_ok=True)
return path
def load_index(path: Path) -> RepoIndex:
return RepoIndex.from_dict(json.loads(path.read_text(encoding="utf-8")))
def append_event(index: RepoIndex, event: dict[str, Any]) -> None:
event = {**event, "emitted_at": event.get("emitted_at") or _now()}
index.events.append(event)

View file

@ -0,0 +1,210 @@
"""Build repository observation + work-record index from files."""
from __future__ import annotations
import re
from pathlib import Path
import yaml
from repo_manager.cache import source_fingerprint
from repo_manager.classification import require_valid_classification
from repo_manager.gitops import head_sha, is_git_repo
from repo_manager.index_store import RepoIndex, WorkRecordEntry, _now
from repo_manager.parse.record import iter_record_files, parse_record_file
from repo_manager.parse.register import iter_register_files, parse_register_file
from repo_manager.parse.workplan import iter_workplan_files, parse_workplan_file
from repo_manager.record_identity import scan_record_identities
class RecordIdentityCollisionError(ValueError):
"""Raised when one canonical record id points at conflicting UUIDs."""
def _slug_from_path(repo_root: Path) -> str:
return re.sub(r"[^a-z0-9]+", "-", repo_root.name.lower()).strip("-") or "repo"
def load_classification(repo_root: Path) -> dict | None:
path = repo_root / ".repo-classification.yaml"
if not path.is_file():
return None
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
if isinstance(data, dict) and "repo_classification" in data:
classification = data["repo_classification"]
else:
classification = data if isinstance(data, dict) else None
if not isinstance(classification, dict):
return None
return require_valid_classification(classification)
def observe_repository(repo_root: Path, *, slug: str | None = None) -> tuple[dict, RepoIndex]:
"""Return (RepositorySnapshot-like dict, rebuilt RepoIndex)."""
repo_root = repo_root.resolve()
slug = slug or _slug_from_path(repo_root)
classification = load_classification(repo_root)
intent_path = None
description = None
for name in ("INTENT.md", "GOAL.md"):
p = repo_root / name
if p.is_file():
intent_path = name
# first non-empty line of body as weak description
text = p.read_text(encoding="utf-8")
for line in text.splitlines():
line = line.strip()
if line and not line.startswith("#") and not line.startswith("---") and not line.startswith(">"):
description = line[:200]
break
break
records: list[WorkRecordEntry] = []
for path in iter_workplan_files(repo_root):
wp = parse_workplan_file(path, repo_root=repo_root)
records.append(
WorkRecordEntry(
kind="workplan",
id=wp.id,
status=wp.status,
title=wp.title,
source_path=wp.path,
uuid=wp.state_hub_workstream_id,
extra={
key: wp.frontmatter[key]
for key in ("depends_on", "related", "needs_human", "intervention_note")
if key in wp.frontmatter
},
)
)
for task_number, task in enumerate(wp.tasks, start=1):
records.append(
WorkRecordEntry(
kind="task",
id=task.id,
status=task.status,
title=task.title,
source_path=wp.path,
uuid=task.state_hub_task_id,
parent_id=wp.id,
extra={
"source_occurrence": f"{wp.path}#task-block-{task_number}",
**{
key: task.raw[key]
for key in (
"depends_on",
"needs_human",
"intervention_note",
"blocking_reason",
)
if key in task.raw
},
},
)
)
for path in iter_record_files(repo_root):
for record_number, record in enumerate(parse_record_file(path, repo_root=repo_root), start=1):
records.append(
WorkRecordEntry(
kind=record.kind,
id=record.id,
status=record.status,
title=record.title,
source_path=record.source_path,
uuid=record.uuid,
extra={
"record": record.raw,
"source_occurrence": f"{record.source_path}#record-block-{record_number}",
},
)
)
for path in iter_register_files(repo_root):
for entry in parse_register_file(path, repo_root=repo_root):
records.append(
WorkRecordEntry(
kind=f"register:{entry.kind}",
id=entry.id,
status=entry.status,
title=entry.title,
source_path=entry.source_path,
extra={"register_kind": entry.kind, "entry": entry.raw},
)
)
identity = scan_record_identities(repo_root)
if identity["identity_collisions"]:
raise RecordIdentityCollisionError(
f"work-record identity collision: {identity['identity_collisions']}"
)
diagnostic_events: list[dict] = []
for duplicate in identity["duplicate_source_occurrences"]:
matches = [
record
for record in records
if record.kind == duplicate["kind"] and record.id == duplicate["id"]
]
if len(matches) < 2:
continue
keeper = matches[0]
keeper.extra.pop("source_occurrence", None)
keeper.extra["source_occurrences"] = duplicate["sources"]
records = [record for record in records if record is keeper or record not in matches]
diagnostic_events.append(
{
"type": "repo.work_record.duplicate_source_occurrence",
"severity": "warning",
"record_kind": duplicate["kind"],
"record_id": duplicate["id"],
"uuid": duplicate["uuid"],
"source_occurrences": duplicate["sources"],
"cleanup_required": True,
"canon": identity["canon"],
}
)
for record in records:
record.extra.pop("source_occurrence", None)
sha = head_sha(repo_root) if is_git_repo(repo_root) else None
fingerprint, source_files = source_fingerprint(repo_root)
index = RepoIndex(
slug=slug,
repo_root=str(repo_root),
head_sha=sha,
observed_at=_now(),
source_fingerprint=fingerprint,
source_files=source_files,
work_records=records,
events=diagnostic_events,
)
snapshot = {
"api_version": "0.1",
"slug": slug,
"lifecycle": "active",
"domain": (classification or {}).get("domain"),
"classification": classification,
"purpose": {
"description": description,
"intent_path": intent_path,
},
"locations": {
"repo_root": str(repo_root),
},
"revision": {
"head_sha": sha,
"observed_at": index.observed_at,
},
"index": {
"workplan_count": sum(1 for r in records if r.kind == "workplan"),
"task_count": sum(1 for r in records if r.kind == "task"),
"register_entry_count": sum(1 for r in records if r.kind.startswith("register:")),
"intake_count": sum(1 for r in records if r.kind == "intake"),
"decision_count": sum(1 for r in records if r.kind == "decision"),
"record_count": len(records),
},
}
return snapshot, index

View file

@ -0,0 +1,177 @@
"""Validation and rendering for owner-consumable task interfaces."""
from __future__ import annotations
from pathlib import Path
from typing import Any
import yaml
API_VERSION = "helixforge.repo-manager/v1"
KIND = "OwnerTaskInterface"
DISPOSITIONS = ("approved", "amended", "rejected")
PRIORITIES = frozenset({"high", "medium", "low"})
def _error(code: str, path: Path, message: str) -> dict[str, str]:
return {"code": code, "path": str(path), "message": message}
def _text(value: Any) -> str | None:
if not isinstance(value, str):
return None
value = value.strip()
return value or None
def _string_list(value: Any) -> list[str] | None:
if not isinstance(value, list) or not value:
return None
values = [_text(item) for item in value]
if any(item is None for item in values):
return None
return [item for item in values if item is not None]
def _interface_paths(path: Path) -> list[Path]:
path = path.expanduser().resolve()
if path.is_file():
return [path]
return sorted(path.glob("*.yaml"))
def validate_owner_interfaces(path: Path, *, owner: str | None = None) -> dict[str, Any]:
"""Validate task interfaces and return their directly consumable content."""
path = path.expanduser().resolve()
errors: list[dict[str, str]] = []
interfaces: list[dict[str, Any]] = []
seen_ids: set[str] = set()
paths = _interface_paths(path)
if not paths:
errors.append(_error("interface_not_found", path, "no interface YAML files found"))
for interface_path in paths:
try:
raw = yaml.safe_load(interface_path.read_text(encoding="utf-8")) or {}
except (OSError, yaml.YAMLError) as exc:
errors.append(_error("interface_invalid", interface_path, str(exc)))
continue
if not isinstance(raw, dict):
errors.append(_error("interface_invalid", interface_path, "root must be a mapping"))
continue
if raw.get("apiVersion") != API_VERSION or raw.get("kind") != KIND:
errors.append(
_error(
"interface_kind_invalid",
interface_path,
f"expected apiVersion {API_VERSION!r} and kind {KIND!r}",
)
)
continue
metadata = raw.get("metadata")
source = raw.get("source")
target = raw.get("target")
approval = raw.get("approval")
task = raw.get("task")
if not all(isinstance(item, dict) for item in (metadata, source, target, approval, task)):
errors.append(
_error(
"interface_shape_invalid",
interface_path,
"metadata, source, target, approval, and task must be mappings",
)
)
continue
interface_id = _text(metadata.get("id"))
title = _text(metadata.get("title"))
source_repo = _text(source.get("repo"))
source_task = _text(source.get("workplan_task"))
target_repo = _text(target.get("repo"))
owner_agent = _text(target.get("owner_agent"))
task_title = _text(task.get("title"))
objective = _text(task.get("objective"))
priority = _text(task.get("priority"))
deliverables = _string_list(task.get("deliverables"))
acceptance = _string_list(task.get("acceptance"))
dispositions = approval.get("dispositions")
required = {
"metadata.id": interface_id,
"metadata.title": title,
"source.repo": source_repo,
"source.workplan_task": source_task,
"target.repo": target_repo,
"target.owner_agent": owner_agent,
"task.title": task_title,
"task.objective": objective,
}
missing = [name for name, value in required.items() if value is None]
if missing:
errors.append(
_error(
"interface_required_field_missing",
interface_path,
"missing non-empty fields: " + ", ".join(missing),
)
)
continue
if priority not in PRIORITIES:
errors.append(
_error(
"interface_priority_invalid",
interface_path,
"task.priority must be high, medium, or low",
)
)
continue
if deliverables is None or acceptance is None:
errors.append(
_error(
"interface_task_invalid",
interface_path,
"task.deliverables and task.acceptance must be non-empty string lists",
)
)
continue
if dispositions != list(DISPOSITIONS):
errors.append(
_error(
"interface_approval_invalid",
interface_path,
"approval.dispositions must be [approved, amended, rejected]",
)
)
continue
if "status" in raw or "status" in metadata:
errors.append(
_error(
"interface_lifecycle_forbidden",
interface_path,
"an interface is not a work record and must not carry lifecycle status",
)
)
continue
if interface_id in seen_ids:
errors.append(
_error("interface_id_duplicate", interface_path, f"duplicate id {interface_id!r}")
)
continue
seen_ids.add(interface_id)
if owner is None or owner in {target_repo, owner_agent}:
document = dict(raw)
document["path"] = str(interface_path)
interfaces.append(document)
return {
"ok": not errors,
"apiVersion": API_VERSION,
"kind": KIND,
"root": str(path),
"interface_count": len(interfaces),
"interfaces": interfaces,
"errors": errors,
}

View file

@ -0,0 +1,3 @@
from repo_manager.parse.workplan import ParsedTask, ParsedWorkplan, parse_workplan_file
__all__ = ["ParsedTask", "ParsedWorkplan", "parse_workplan_file"]

View file

@ -0,0 +1,80 @@
"""Parse repository-owned intake and decision records from Markdown YAML fences."""
from __future__ import annotations
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import yaml
_YAML_FENCE_RE = re.compile(r"```ya?ml\s*\n(.*?)\n```", re.DOTALL | re.IGNORECASE)
SUPPORTED_RECORD_KINDS = frozenset({"intake", "decision"})
@dataclass
class ParsedRecord:
kind: str
id: str
title: str | None
status: str | None
uuid: str | None
source_path: str
raw: dict[str, Any] = field(default_factory=dict)
block_start: int = 0
block_end: int = 0
def parse_record_text(text: str, *, relative_path: str) -> list[ParsedRecord]:
"""Return intake/decision records embedded in YAML fenced blocks."""
records: list[ParsedRecord] = []
for match in _YAML_FENCE_RE.finditer(text):
try:
data = yaml.safe_load(match.group(1)) or {}
except yaml.YAMLError:
continue
if not isinstance(data, dict) or data.get("kind") not in SUPPORTED_RECORD_KINDS:
continue
record_id = data.get("id")
if record_id is None:
continue
kind = str(data["kind"])
uuid_field = "state_hub_intake_id" if kind == "intake" else "state_hub_decision_id"
records.append(
ParsedRecord(
kind=kind,
id=str(record_id),
title=str(data["title"]) if data.get("title") is not None else None,
status=str(data["status"]) if data.get("status") is not None else None,
uuid=str(data[uuid_field]) if data.get(uuid_field) is not None else None,
source_path=relative_path,
raw=data,
block_start=match.start(),
block_end=match.end(),
)
)
return records
def parse_record_file(path: Path, *, repo_root: Path) -> list[ParsedRecord]:
return parse_record_text(
path.read_text(encoding="utf-8"),
relative_path=str(path.relative_to(repo_root)),
)
def iter_record_files(repo_root: Path) -> list[Path]:
"""Find only the governed record locations; do not interpret arbitrary docs."""
files: set[Path] = set()
for relative in ("intakes", "decisions", "docs/intakes", "docs/decisions", "workplans"):
root = repo_root / relative
if root.is_dir():
files.update(path for path in root.rglob("*.md") if not path.name.startswith("."))
# Fleet repositories use both title-case conventions. Keep these explicit:
# arbitrary top-level Markdown remains outside the governed record surface.
for name in ("INTAKES.md", "DECISIONS.md", "intakes.md", "decisions.md"):
path = repo_root / name
if path.is_file():
files.add(path)
return sorted(files)

View file

@ -0,0 +1,73 @@
"""Parser for repository-owned register files."""
from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import yaml
REGISTER_SCHEMA = "repo-manager.register.v0"
SUPPORTED_REGISTER_KINDS = frozenset(
{
"sbom-inventory",
"repo-goals",
"upstream-contributions",
"technical-debt",
"extension-points",
"register-entries",
}
)
@dataclass
class ParsedRegisterEntry:
kind: str
id: str
title: str | None
status: str | None
source_path: str
raw: dict[str, Any] = field(default_factory=dict)
def register_path(repo_root: Path, kind: str) -> Path:
return repo_root / "registers" / f"{kind}.yaml"
def parse_register_file(path: Path, *, repo_root: Path) -> list[ParsedRegisterEntry]:
try:
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
except (OSError, yaml.YAMLError):
return []
if not isinstance(data, dict):
return []
kind = str(data.get("kind") or path.stem)
if kind not in SUPPORTED_REGISTER_KINDS:
return []
entries = data.get("entries") or []
if not isinstance(entries, list):
return []
source_path = str(path.relative_to(repo_root))
parsed: list[ParsedRegisterEntry] = []
for item in entries:
if not isinstance(item, dict) or not item.get("id"):
continue
parsed.append(
ParsedRegisterEntry(
kind=kind,
id=str(item["id"]),
title=str(item["title"]) if item.get("title") is not None else None,
status=str(item["status"]) if item.get("status") is not None else None,
source_path=source_path,
raw=item,
)
)
return parsed
def iter_register_files(repo_root: Path) -> list[Path]:
directory = repo_root / "registers"
if not directory.is_dir():
return []
return sorted(path for path in directory.glob("*.yaml") if not path.name.startswith("."))

View file

@ -0,0 +1,114 @@
"""Workplan file parser (P0 extract — simplified from State Hub consistency_check)."""
from __future__ import annotations
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import yaml
_TASK_BLOCK_RE = re.compile(r"```task\s*\n(.*?)\n```", re.DOTALL)
_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+)$", re.MULTILINE)
def _parse_yaml_block(raw: str) -> dict[str, Any]:
try:
data = yaml.safe_load(raw) or {}
except yaml.YAMLError:
return {}
return data if isinstance(data, dict) else {}
def parse_frontmatter(text: str) -> tuple[dict[str, Any], str]:
if not text.startswith("---"):
return {}, text
parts = text.split("---", 2)
if len(parts) < 3:
return {}, text
return _parse_yaml_block(parts[1].strip()), parts[2]
@dataclass
class ParsedTask:
id: str | None
title: str | None
status: str | None
state_hub_task_id: str | None
raw: dict[str, Any] = field(default_factory=dict)
@dataclass
class ParsedWorkplan:
path: str
id: str | None
title: str | None
status: str | None
state_hub_workstream_id: str | None
frontmatter: dict[str, Any]
tasks: list[ParsedTask]
def parse_task_blocks(body: str) -> list[ParsedTask]:
headings = [
(m.start(), len(m.group(1)), m.group(2).strip())
for m in _HEADING_RE.finditer(body)
]
results: list[ParsedTask] = []
for m in _TASK_BLOCK_RE.finditer(body):
meta = _parse_yaml_block(m.group(1).strip())
prev = [(pos, level, text) for pos, level, text in headings if pos < m.start()]
title = meta.get("title")
if not title and prev:
title = prev[-1][2]
results.append(
ParsedTask(
id=str(meta["id"]) if meta.get("id") is not None else None,
title=str(title) if title else None,
status=str(meta["status"]) if meta.get("status") is not None else None,
state_hub_task_id=(
str(meta["state_hub_task_id"]).strip().strip('"')
if meta.get("state_hub_task_id") is not None
else None
),
raw=meta,
)
)
return results
def parse_workplan_text(text: str, *, relative_path: str) -> ParsedWorkplan:
fm, body = parse_frontmatter(text)
return ParsedWorkplan(
path=relative_path,
id=str(fm["id"]) if fm.get("id") is not None else None,
title=str(fm["title"]) if fm.get("title") is not None else None,
status=str(fm["status"]) if fm.get("status") is not None else None,
state_hub_workstream_id=(
str(fm["state_hub_workstream_id"]).strip().strip('"')
if fm.get("state_hub_workstream_id") is not None
else None
),
frontmatter=fm,
tasks=parse_task_blocks(body),
)
def parse_workplan_file(path: Path, *, repo_root: Path) -> ParsedWorkplan:
text = path.read_text(encoding="utf-8")
rel = str(path.relative_to(repo_root))
return parse_workplan_text(text, relative_path=rel)
def iter_workplan_files(repo_root: Path) -> list[Path]:
wp_dir = repo_root / "workplans"
if not wp_dir.is_dir():
return []
files: list[Path] = []
for p in sorted(wp_dir.rglob("*.md")):
if p.name.startswith("."):
continue
# skip archived copies optionally still include them
files.append(p)
return files

View file

@ -0,0 +1,112 @@
"""Fleet workplan-prefix ownership and uniqueness (RMGR-WP-0004-T08)."""
from __future__ import annotations
from collections import defaultdict
from pathlib import Path
from typing import Any
import yaml
from repo_manager.standards import FLAVOR_MARKER_PREFIX, workplan_ids
DEFAULT_REGISTRY = Path(__file__).resolve().parents[2] / "config" / "workplan-prefix-registry.yaml"
def load_registry(path: Path | None = None) -> dict[str, Any]:
target = path or DEFAULT_REGISTRY
data = yaml.safe_load(target.read_text(encoding="utf-8")) or {}
if not isinstance(data, dict):
return {}
return data
def _skip_dir(name: str) -> bool:
return name.startswith(".") or name in {
"node_modules",
"__pycache__",
".venv",
"venv",
"archive",
}
def iter_repo_roots(root: Path) -> list[Path]:
root = root.resolve()
if (root / "workplans").is_dir() and (root / ".git").exists():
return [root]
found: list[Path] = []
if not root.is_dir():
return found
for child in sorted(root.iterdir()):
if not child.is_dir() or _skip_dir(child.name):
continue
if (child / "workplans").is_dir():
found.append(child)
return found
def scan_prefixes(root: Path, *, registry_path: Path | None = None) -> dict[str, Any]:
"""Detect prefix sharing, identifier reuse, flavor-derived prefixes, number reuse.
Detection only. Does not rewrite files or remediations.
"""
registry = load_registry(registry_path)
owners = {str(k): str(v) for k, v in (registry.get("owners") or {}).items()}
retired = {str(item.get("prefix")) for item in registry.get("retired") or [] if item.get("prefix")}
by_prefix: dict[str, set[str]] = defaultdict(set)
by_id: dict[str, list[tuple[str, str]]] = defaultdict(list)
numbers: dict[tuple[str, str], list[str]] = defaultdict(list)
repos_scanned = []
for repo in iter_repo_roots(root):
repos_scanned.append(repo.name)
for rel, prefix, number in workplan_ids(repo):
by_prefix[prefix].add(repo.name)
ident = f"{prefix}-{number}"
by_id[ident].append((repo.name, rel))
numbers[(repo.name, prefix)].append(number)
shared = {
prefix: sorted(repos)
for prefix, repos in sorted(by_prefix.items())
if len(repos) > 1
}
reused = {
ident: [{"repo": repo, "path": path} for repo, path in entries]
for ident, entries in sorted(by_id.items())
if len({repo for repo, _path in entries}) > 1 or len(entries) > 1
}
flavor_derived = sorted(repo for repo in by_prefix.get(FLAVOR_MARKER_PREFIX, []))
retired_in_use = {
prefix: sorted(repos)
for prefix, repos in by_prefix.items()
if prefix in retired
}
owner_mismatch = {
prefix: {"declared_owner": owners[prefix], "seen_in": sorted(repos)}
for prefix, repos in by_prefix.items()
if prefix in owners and any(repo != owners[prefix] for repo in repos)
}
reused_numbers = []
for (repo, prefix), nums in numbers.items():
seen: set[str] = set()
for num in nums:
if num in seen:
reused_numbers.append({"repo": repo, "prefix": prefix, "number": num})
seen.add(num)
return {
"ok": not shared and not flavor_derived and not reused_numbers and not reused,
"root": str(root.resolve()),
"repos_scanned": repos_scanned,
"shared_prefixes": shared,
"reused_identifiers": reused,
"flavor_derived_prefix": flavor_derived,
"retired_in_use": retired_in_use,
"owner_mismatch": owner_mismatch,
"reused_numbers_in_repo": reused_numbers,
"detection_only": True,
}

View file

@ -0,0 +1,267 @@
"""Fast, guarded trigger for State Hub's forge-derived repository projection."""
from __future__ import annotations
import hashlib
import json
import subprocess
import tempfile
import time
from pathlib import Path
from typing import Any
import httpx
from repo_manager.gitops import GitError, commit_paths, head_sha, push_ff
from repo_manager.identifiers import ensure_missing_work_record_identifiers
def _git(repo: Path, *args: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(["git", *args], cwd=repo, capture_output=True, text=True, check=False)
def _ahead_behind(repo: Path) -> dict[str, Any]:
upstream = _git(repo, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}")
if upstream.returncode != 0:
raise ValueError("current branch has no upstream; central can only derive pushed work")
counts = _git(repo, "rev-list", "--left-right", "--count", "@{u}...HEAD")
if counts.returncode != 0:
raise ValueError(counts.stderr.strip() or "could not compare HEAD with upstream")
try:
behind, ahead = (int(value) for value in counts.stdout.split())
except ValueError as exc:
raise ValueError("could not parse Git ahead/behind state") from exc
return {"upstream": upstream.stdout.strip(), "behind": behind, "ahead": ahead}
def _dirty_authoritative_paths(repo: Path) -> list[str]:
result = _git(repo, "status", "--porcelain", "--", "workplans")
if result.returncode != 0:
raise ValueError(result.stderr.strip() or "could not inspect workplan files")
return [line[3:] for line in result.stdout.splitlines() if len(line) > 3]
def _pending_path(repo: Path) -> Path:
resolved = _git(
repo, "rev-parse", "--git-path", "repo-manager/pending-work-record-projection.json"
)
if resolved.returncode != 0 or not resolved.stdout.strip():
raise ValueError("could not resolve the repository-local Git state directory")
path = Path(resolved.stdout.strip())
return path if path.is_absolute() else repo / path
def _write_pending(repo: Path, payload: dict[str, Any]) -> Path:
path = _pending_path(repo)
path.parent.mkdir(parents=True, exist_ok=True)
temporary: Path | None = None
try:
with tempfile.NamedTemporaryFile(
mode="w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", delete=False
) as handle:
temporary = Path(handle.name)
handle.write(json.dumps(payload, indent=2) + "\n")
temporary.replace(path)
temporary = None
finally:
if temporary is not None:
temporary.unlink(missing_ok=True)
return path
def sync_repository_projection(
repo_root: Path,
*,
api_base: str,
repo_slug: str | None = None,
expected_instance_label: str | None = "railiance01",
commit_identifiers: bool = True,
push: bool = False,
acknowledge_retirements: bool = False,
queue_on_unavailable: bool = True,
transport: httpx.BaseTransport | None = None,
) -> dict[str, Any]:
"""Assign missing IDs, prove the commit is pushed, then reconcile once."""
started = time.perf_counter()
repo_root = repo_root.resolve()
slug = repo_slug or repo_root.name
dirty = _dirty_authoritative_paths(repo_root)
if dirty:
return {
"ok": False,
"status": "pending_commit",
"error": "authoritative workplan files have uncommitted changes",
"dirty_paths": dirty,
}
assignments = ensure_missing_work_record_identifiers(repo_root)
if assignments["files_changed"] and commit_identifiers:
try:
commit_paths(
repo_root,
assignments["files_changed"],
message=(
"repo.work.assign_missing_identifiers\n\n"
"source: repo-manager\nreason: deterministic projection registration\n"
),
)
except GitError as exc:
return {"ok": False, "status": "failed", "error": str(exc), "identifiers": assignments}
dirty = _dirty_authoritative_paths(repo_root)
if dirty:
return {
"ok": False,
"status": "pending_commit",
"error": "authoritative workplan files have uncommitted changes",
"dirty_paths": dirty,
"identifiers": assignments,
}
try:
git = _ahead_behind(repo_root)
except ValueError as exc:
return {"ok": False, "status": "failed", "error": str(exc), "identifiers": assignments}
if git["behind"]:
return {
"ok": False,
"status": "behind_upstream",
"error": "branch is behind upstream; reconcile Git before projection",
"git": git,
}
if git["ahead"] and push:
pushed, message = push_ff(repo_root)
if not pushed:
return {
"ok": False,
"status": "push_failed",
"error": message,
"git": git,
}
git = _ahead_behind(repo_root)
if git["ahead"]:
return {
"ok": True,
"status": "pending_push",
"message": "commit is not yet visible to the authoritative forge",
"git": git,
"identifiers": assignments,
}
commit = head_sha(repo_root)
if not commit:
return {"ok": False, "status": "failed", "error": "repository HEAD is unavailable"}
request_path = f"/repos/{slug}/work-record-projection/reconcile"
request_body = {
"expected_commit": commit,
"acknowledge_retirements": acknowledge_retirements,
}
# The review acknowledgement changes request semantics at the same commit.
# Bind the key to the complete POST, retaining identical keys for retries.
request_digest = hashlib.sha256(json.dumps(
{"method": "POST", "path": request_path, "body": request_body},
sort_keys=True, separators=(",", ":"),
).encode()).hexdigest()
pending = {
"schema": "repo-manager.pending-work-record-projection.v1",
"repo_slug": slug,
"expected_commit": commit,
"api_base": api_base.rstrip("/"),
"acknowledge_retirements": acknowledge_retirements,
"expected_instance_label": expected_instance_label,
}
headers = {
"Idempotency-Key": f"rmgr-projection:v2:{request_digest}",
"X-StateHub-Source-Agent": "repo-manager",
}
try:
with httpx.Client(
base_url=api_base.rstrip("/"),
timeout=httpx.Timeout(120.0, connect=5.0),
follow_redirects=True,
transport=transport,
) as client:
health = client.get("/state/health")
health.raise_for_status()
identity = health.json()
identity_ok = identity.get("instance_role") == "primary" and (
expected_instance_label is None
or identity.get("instance_label") == expected_instance_label
)
if not identity_ok:
if not queue_on_unavailable:
return {
"ok": False,
"status": "wrong_instance",
"instance": identity,
}
path = _write_pending(repo_root, {**pending, "reason": "wrong_instance"})
return {
"ok": True,
"status": "queued",
"reason": "wrong_instance",
"instance": identity,
"pending_path": str(path),
}
response = client.post(
request_path,
json=request_body,
headers=headers,
)
response.raise_for_status()
receipt = response.json()
except (httpx.ConnectError, httpx.TimeoutException, httpx.NetworkError) as exc:
if not queue_on_unavailable:
return {"ok": False, "status": "unavailable", "error": str(exc)}
path = _write_pending(repo_root, {**pending, "reason": "hub_unavailable"})
return {
"ok": True,
"status": "queued",
"reason": "hub_unavailable",
"pending_path": str(path),
}
except httpx.HTTPStatusError as exc:
detail: Any
try:
detail = exc.response.json()
except ValueError:
detail = exc.response.text[:500]
return {
"ok": False,
"status": "rejected",
"http_status": exc.response.status_code,
"error": detail,
}
# A successful HTTP response alone is not an exact-commit primary receipt.
outcome = receipt.get("outcome") if isinstance(receipt, dict) else None
if not (
isinstance(receipt, dict)
and receipt.get("schema") == "state-hub.repository-projection-reconcile.v1"
and receipt.get("instance_role") == "primary"
and receipt.get("instance_label") == identity.get("instance_label")
and receipt.get("expected_commit") == commit
and receipt.get("derived_commit") == commit
and isinstance(outcome, dict)
and outcome.get("repo_slug") == slug
and outcome.get("commit") == commit
and outcome.get("status") in {"applied", "noop", "refused"}
):
return {"ok": False, "status": "invalid_receipt", "repo_slug": slug,
"commit": commit, "error": "response does not bind the requested primary/repository/commit"}
outcome_status = outcome["status"]
if outcome_status in {"applied", "noop"}:
_pending_path(repo_root).unlink(missing_ok=True)
return {
"ok": outcome_status in {"applied", "noop"},
"status": outcome_status,
"repo_slug": slug,
"commit": commit,
"requests": 2,
"elapsed_ms": round((time.perf_counter() - started) * 1000, 1),
"git": git,
"identifiers": assignments,
"receipt": receipt,
}

View file

@ -0,0 +1,135 @@
"""Install and report on coding-assistant commit provenance."""
from __future__ import annotations
import re
import subprocess
from collections import defaultdict
from pathlib import Path
from typing import Any
import yaml
_TRAILER_RE = re.compile(r"^(Assistant(?:-Model|-Process|-Session)?):\s*(.+)$", re.MULTILINE)
def install_hook(hooks_path: Path) -> dict[str, Any]:
"""Configure the user's Git installation to use the governed hook directory."""
hooks_path = hooks_path.expanduser().resolve()
hook = hooks_path / "prepare-commit-msg"
if not hook.is_file():
return {"ok": False, "error": f"missing hook: {hook}"}
if not hook.stat().st_mode & 0o111:
return {"ok": False, "error": f"hook is not executable: {hook}"}
completed = subprocess.run(
["git", "config", "--global", "core.hooksPath", str(hooks_path)],
check=False,
capture_output=True,
text=True,
)
if completed.returncode:
return {"ok": False, "error": completed.stderr.strip() or "git config failed"}
return {"ok": True, "hooks_path": str(hooks_path), "hook": str(hook)}
def _cutover(repo_root: Path) -> str | None:
path = repo_root / "config" / "assistant-provenance.yaml"
if not path.is_file():
return None
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
value = data.get("cutover_commit") if isinstance(data, dict) else None
return str(value) if value else None
def _commits(repo_root: Path, rev: str, max_count: int | None) -> list[dict[str, Any]]:
command = ["git", "log", "--reverse", "--format=%H%x1f%aI%x1f%an%x1f%ae%x1f%B%x1e"]
if max_count:
command.append(f"--max-count={max_count}")
command.append(rev)
completed = subprocess.run(command, cwd=repo_root, check=True, capture_output=True, text=True)
commits: list[dict[str, Any]] = []
for raw in completed.stdout.split("\x1e"):
raw = raw.strip("\n")
if not raw:
continue
parts = raw.split("\x1f", 4)
if len(parts) != 5:
continue
sha, authored_at, author, email, body = parts
trailers = {key: value.strip() for key, value in _TRAILER_RE.findall(body)}
commits.append(
{
"sha": sha,
"authored_at": authored_at,
"author": author,
"email": email,
"assistant": trailers.get("Assistant"),
"model": trailers.get("Assistant-Model"),
"process": trailers.get("Assistant-Process"),
"session": trailers.get("Assistant-Session"),
}
)
return commits
def assistant_report(repo_root: Path, *, rev: str = "HEAD", max_count: int | None = None) -> dict[str, Any]:
"""Derive assistant activity and interleaved-session signals from Git alone."""
repo_root = repo_root.resolve()
commits = _commits(repo_root, rev, max_count)
cutover = _cutover(repo_root)
cutover_seen = False
assistants: dict[str, dict[str, Any]] = defaultdict(
lambda: {"commit_count": 0, "models": set(), "sessions": set(), "processes": set()}
)
unattributed = {"before_cutover": 0, "after_cutover": 0, "known_automation": 0}
session_sequence: list[str] = []
for commit in commits:
if cutover and commit["sha"].startswith(cutover):
cutover_seen = True
assistant = commit["assistant"]
if assistant:
entry = assistants[assistant]
entry["commit_count"] += 1
for field, bucket in (("model", "models"), ("session", "sessions"), ("process", "processes")):
if commit[field]:
entry[bucket].add(commit[field])
if commit["session"]:
session_sequence.append(commit["session"])
elif commit["author"] == "custodian-sync" or commit["email"] == "custodian-sync@railiance.local":
unattributed["known_automation"] += 1
elif cutover_seen:
unattributed["after_cutover"] += 1
else:
unattributed["before_cutover"] += 1
interleaved: set[tuple[str, str]] = set()
positions: dict[str, list[int]] = defaultdict(list)
for position, session in enumerate(session_sequence):
positions[session].append(position)
sessions = sorted(positions)
for index, first in enumerate(sessions):
for second in sessions[index + 1 :]:
merged = [session_sequence[pos] for pos in sorted(positions[first] + positions[second])]
compressed = [value for pos, value in enumerate(merged) if pos == 0 or value != merged[pos - 1]]
if len(compressed) >= 3:
interleaved.add((first, second))
return {
"ok": True,
"repo": str(repo_root),
"revision": rev,
"cutover_commit": cutover,
"commit_count": len(commits),
"assistants": {
name: {
"commit_count": item["commit_count"],
"models": sorted(item["models"]),
"sessions": sorted(item["sessions"]),
"processes": sorted(item["processes"]),
}
for name, item in sorted(assistants.items())
},
"interleaved_sessions": [list(pair) for pair in sorted(interleaved)],
"unattributed": unattributed,
}

View file

@ -0,0 +1,139 @@
"""Work-record identity validation and duplicate-source reconciliation."""
from __future__ import annotations
import re
from collections import defaultdict
from pathlib import Path
from typing import Any
import yaml
from repo_manager.parse.record import iter_record_files, parse_record_file
from repo_manager.parse.workplan import iter_workplan_files, parse_workplan_file
DEFAULT_KIND_REGISTRY = Path(__file__).resolve().parents[2] / "config" / "work-record-types.yaml"
CANON_WORK_RECORD_TYPES = "the-custodian/canon/standards/work-record-types.yaml"
def load_kind_registry(path: Path | None = None) -> dict[str, Any]:
data = yaml.safe_load((path or DEFAULT_KIND_REGISTRY).read_text(encoding="utf-8")) or {}
if not isinstance(data, dict) or not isinstance(data.get("kinds"), list):
raise TypeError("invalid work-record kind registry")
return data
def classify_record_id(
kind: str,
identifier: str,
*,
parent_id: str | None = None,
registry_path: Path | None = None,
) -> str | None:
"""Return canonical/grandfathered when the canon registry accepts an id."""
registry = load_kind_registry(registry_path)
entry = next((item for item in registry["kinds"] if item.get("kind") == kind), None)
if not isinstance(entry, dict):
return None
if any(re.fullmatch(pattern, identifier) for pattern in entry.get("id_patterns") or []):
return "canonical"
for legacy in entry.get("legacy_patterns") or []:
if legacy.get("grandfathered") and re.fullmatch(
str(legacy.get("pattern") or ""), identifier
):
return "grandfathered"
# Historical task blocks may use a local TNN label. Accept these only
# under an explicitly enumerated parent workplan; the parent scope is
# part of identity and no source id/UUID is rewritten.
if parent_id:
for legacy in entry.get("scoped_legacy_patterns") or []:
if (
legacy.get("grandfathered")
and re.fullmatch(str(legacy.get("pattern") or ""), identifier)
and re.fullmatch(str(legacy.get("parent_pattern") or ""), parent_id)
):
return "grandfathered"
return None
def scan_record_identities(repo_root: Path, *, registry_path: Path | None = None) -> dict[str, Any]:
"""Describe invalid ids, benign duplicate sources, and hard collisions."""
occurrences: dict[tuple[str, str, str | None], list[dict[str, Any]]] = defaultdict(list)
invalid: list[dict[str, str]] = []
def add(
kind: str,
identifier: str | None,
record_uuid: str | None,
source: str,
*,
parent_id: str | None = None,
) -> None:
if not identifier:
return
classification = classify_record_id(
kind, identifier, parent_id=parent_id, registry_path=registry_path
)
item = {
"kind": kind,
"id": identifier,
"uuid": record_uuid,
"source": source,
**({"parent_id": parent_id} if parent_id else {}),
}
# Only the explicitly scoped local aliases use parent scope. Qualified
# canonical IDs remain globally collision checked.
local_alias = (
kind == "task"
and parent_id is not None
and re.fullmatch(r"T[0-9]{2}", identifier) is not None
and classification == "grandfathered"
)
occurrences[(kind, identifier, parent_id if local_alias else None)].append(item)
if classification is None or (local_alias and not record_uuid):
invalid.append({"kind": kind, "id": identifier, "source": source})
for path in iter_workplan_files(repo_root):
workplan = parse_workplan_file(path, repo_root=repo_root)
add("workplan", workplan.id, workplan.state_hub_workstream_id, workplan.path)
for number, task in enumerate(workplan.tasks, start=1):
add(
"task",
task.id,
task.state_hub_task_id,
f"{workplan.path}#task-block-{number}",
parent_id=workplan.id,
)
for path in iter_record_files(repo_root):
for number, record in enumerate(parse_record_file(path, repo_root=repo_root), start=1):
add(record.kind, record.id, record.uuid, f"{record.source_path}#record-block-{number}")
duplicates: list[dict[str, Any]] = []
collisions: list[dict[str, Any]] = []
for (kind, identifier, _parent_id), items in sorted(
occurrences.items(), key=lambda pair: (pair[0][0], pair[0][1], pair[0][2] or "")
):
if len(items) < 2:
continue
uuids = {item["uuid"] for item in items}
result = {
"kind": kind,
"id": identifier,
"uuid": items[0]["uuid"] if len(uuids) == 1 else None,
"sources": sorted(item["source"] for item in items),
}
if items[0].get("parent_id"):
result["parent_id"] = items[0]["parent_id"]
if len(uuids) == 1 and None not in uuids:
duplicates.append(result)
else:
result["uuids"] = sorted(str(value) for value in uuids)
collisions.append(result)
return {
"ok": not invalid and not collisions,
"invalid_identifiers": invalid,
"duplicate_source_occurrences": duplicates,
"identity_collisions": collisions,
"canon": CANON_WORK_RECORD_TYPES,
}

View file

@ -0,0 +1,30 @@
"""Interim single-writer for hub identifiers (ADR-007 decision 2).
Until deterministic derivation (RMGR-WP-0005-T03) ships, exactly one
instance may mint ``state_hub_workstream_id`` / ``state_hub_task_id``
into repository files. Other instances may read and project; they must
not write new hub primary keys into git-tracked files.
The registrar is the automated production instance (hostname prefix
``railiance``). Override with ``STATEHUB_REGISTRAR=1`` or ``=0``.
"""
from __future__ import annotations
import os
import socket
_TRUE = frozenset({"1", "true", "yes", "on"})
_FALSE = frozenset({"0", "false", "no", "off"})
REGISTRAR_HOST_PREFIX = "railiance"
def is_identifier_registrar(*, hostname: str | None = None) -> bool:
"""Return True if this process may mint hub IDs into repository files."""
raw = os.environ.get("STATEHUB_REGISTRAR", "").strip().lower()
if raw in _TRUE:
return True
if raw in _FALSE:
return False
host = (hostname if hostname is not None else socket.gethostname()).strip().lower()
return host.startswith(REGISTRAR_HOST_PREFIX)

View file

@ -0,0 +1,723 @@
"""Publish repository classifications through the frozen ``port.repo`` contract."""
from __future__ import annotations
import base64
import hashlib
import hmac
import json
import re
import subprocess
import threading
from collections import OrderedDict
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Any
from urllib.parse import quote, urlparse
from uuid import UUID
import httpx
import yaml
from repo_manager import __version__
from repo_manager.classification import ClassificationError, require_valid_classification
from repo_manager.observe import load_classification
from repo_manager.time import format_utc, utc_now
CONTRACT_ID = "helixforge.repository-classification-projection"
CONTRACT_VERSION = "1.0.0"
CLASSIFICATION_CONTRACT_ID = "RMGR-CONTRACT-CLASSIFICATION-0001"
CLASSIFICATION_CONTRACT_VERSION = "1.0"
REGISTRY_SCHEMA = "repo-manager.repository-registry.v1"
LIFECYCLES = frozenset({"active", "archived", "retired"})
_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$")
class RegistryError(ValueError):
"""The operator-owned repository registry cannot be used safely."""
class ProjectionCursorError(ValueError):
"""A projection page cursor is malformed, stale, or has been tampered with."""
@dataclass(frozen=True, slots=True)
class RepositoryRegistration:
repository_id: str
slug: str
lifecycle: str
path: Path | None = None
forgejo_repository: str | None = None
@dataclass(frozen=True, slots=True)
class ClassificationSnapshot:
snapshot_id: str
generated_at: str
source_revision: str
repositories: tuple[dict[str, Any], ...]
diagnostics: tuple[dict[str, Any], ...]
def load_repository_registry(path: Path) -> tuple[RepositoryRegistration, ...]:
"""Load the private operator registry used to join UUIDs to local checkouts."""
try:
raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
except (OSError, yaml.YAMLError) as exc:
raise RegistryError(f"cannot read repository registry: {exc}") from exc
if not isinstance(raw, dict) or raw.get("schema") != REGISTRY_SCHEMA:
raise RegistryError(f"repository registry must declare schema {REGISTRY_SCHEMA!r}")
rows = raw.get("repositories")
if not isinstance(rows, list):
raise RegistryError("repository registry `repositories` must be a list")
registrations: list[RepositoryRegistration] = []
ids: set[str] = set()
slugs: set[str] = set()
for number, row in enumerate(rows, start=1):
if not isinstance(row, dict):
raise RegistryError(f"repository registry row {number} must be an object")
try:
repository_id = str(UUID(str(row["repository_id"])))
slug = str(row["slug"])
lifecycle = str(row["lifecycle"])
raw_path = row.get("path")
raw_forgejo = row.get("forgejo_repository")
except (KeyError, TypeError, ValueError) as exc:
raise RegistryError(f"invalid repository registry row {number}: {exc}") from exc
if repository_id in ids:
raise RegistryError(f"duplicate repository_id {repository_id}")
if slug in slugs:
raise RegistryError(f"duplicate repository slug {slug!r}")
if not _SLUG_RE.fullmatch(slug) or len(slug) > 120:
raise RegistryError(f"invalid repository slug {slug!r}")
if lifecycle not in LIFECYCLES:
raise RegistryError(
f"repository {slug!r} lifecycle must be one of {', '.join(sorted(LIFECYCLES))}"
)
if (raw_path is None) == (raw_forgejo is None):
raise RegistryError(
f"repository {slug!r} must declare exactly one of path or forgejo_repository"
)
repo_path = Path(str(raw_path)) if raw_path is not None else None
forgejo_repository = str(raw_forgejo) if raw_forgejo is not None else None
if repo_path is not None and not repo_path.is_absolute():
raise RegistryError(f"repository {slug!r} path must be absolute")
if forgejo_repository is not None and forgejo_repository != f"coulomb/{slug}":
raise RegistryError(
f"repository {slug!r} Forgejo identity must be exactly 'coulomb/{slug}'"
)
ids.add(repository_id)
slugs.add(slug)
registrations.append(
RepositoryRegistration(
repository_id=repository_id,
slug=slug,
lifecycle=lifecycle,
path=repo_path,
forgejo_repository=forgejo_repository,
)
)
return tuple(sorted(registrations, key=lambda item: item.repository_id))
def import_state_hub_registry(
fleet_root: Path,
*,
api_base: str,
timeout_seconds: float = 10.0,
workers: int = 4,
retries: int = 2,
source: str = "local",
transport: httpx.BaseTransport | None = None,
) -> dict[str, Any]:
"""Bootstrap Repo Manager identities from the retiring authoritative registrar.
The bulk State Hub repository route is intentionally not used: direct
identity reads keep the migration bounded and let independent repositories
complete concurrently. Host paths come only from the operator-selected
local fleet root; remote State Hub path values are never trusted.
"""
parsed = urlparse(api_base)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise ValueError("State Hub API base must be an absolute HTTP(S) origin")
if timeout_seconds <= 0 or workers < 1 or retries < 0:
raise ValueError("timeout_seconds/workers must be positive and retries non-negative")
if source not in {"local", "forgejo"}:
raise ValueError("registry import source must be 'local' or 'forgejo'")
paths = _discover_classified_repositories(fleet_root)
if not paths:
raise ValueError("fleet root contains no classified Git repositories")
with httpx.Client(
base_url=api_base.rstrip("/"),
timeout=httpx.Timeout(timeout_seconds),
follow_redirects=True,
transport=transport,
) as client:
try:
health = client.get("/state/health")
health.raise_for_status()
identity = health.json()
except (httpx.HTTPError, ValueError) as exc:
raise ValueError(f"State Hub registry source is unavailable: {exc}") from exc
if not isinstance(identity, dict):
raise RegistryError("State Hub health response is not a JSON object")
if identity.get("instance_role") != "primary":
raise ValueError("State Hub registry import requires an instance_role=primary source")
def fetch(path: Path) -> tuple[Path, dict[str, Any]]:
for attempt in range(retries + 1):
try:
response = client.get(f"/repos/{path.name}")
response.raise_for_status()
payload = response.json()
if not isinstance(payload, dict):
raise RegistryError("repository response is not a JSON object")
return path, payload
except (httpx.TimeoutException, httpx.NetworkError):
if attempt == retries:
raise
raise AssertionError("unreachable retry loop")
rows: list[dict[str, Any]] = []
diagnostics: list[dict[str, str]] = []
with ThreadPoolExecutor(max_workers=workers) as executor:
pending = {executor.submit(fetch, path): path for path in paths}
for future in as_completed(pending):
path = pending[future]
try:
local_path, payload = future.result()
repository_id = str(UUID(str(payload["id"])))
slug = str(payload["slug"])
lifecycle = str(payload.get("status") or "active")
if lifecycle not in LIFECYCLES:
raise ValueError(f"unsupported lifecycle {lifecycle!r}")
if not _SLUG_RE.fullmatch(slug):
raise ValueError(f"invalid canonical slug {slug!r}")
except httpx.HTTPStatusError as exc:
missing = exc.response.status_code == 404
diagnostics.append(
{
"severity": "warning" if missing else "error",
"code": (
"registry_import.unregistered_checkout"
if missing
else "registry_import.repository_failed"
),
"repository": path.name,
"message": str(exc)[:500],
}
)
except (httpx.HTTPError, KeyError, TypeError, ValueError) as exc:
diagnostics.append(
{
"severity": "error",
"code": "registry_import.repository_failed",
"repository": path.name,
"message": str(exc)[:500],
}
)
else:
rows.append(
{
"repository_id": repository_id,
"slug": slug,
"lifecycle": lifecycle,
**(
{"path": str(local_path.resolve())}
if source == "local"
else {"forgejo_repository": f"coulomb/{slug}"}
),
}
)
rows.sort(key=lambda row: row["repository_id"])
diagnostics.sort(key=lambda row: row["repository"])
ids = [row["repository_id"] for row in rows]
slugs = [row["slug"] for row in rows]
if len(ids) != len(set(ids)) or len(slugs) != len(set(slugs)):
diagnostics.append(
{
"severity": "error",
"code": "registry_import.identity_collision",
"repository": "fleet",
"message": "State Hub returned duplicate repository UUIDs or canonical slugs",
}
)
return {
"ok": not any(item["severity"] == "error" for item in diagnostics),
"schema": REGISTRY_SCHEMA,
"source": {
"system": "state-hub",
"instance_role": identity.get("instance_role"),
"instance_label": identity.get("instance_label"),
},
"generated_at": format_utc(utc_now()),
"repositories": rows,
"diagnostics": diagnostics,
}
class ForgejoClassificationClient:
"""Read exact default-branch classifications from the Forgejo API."""
def __init__(
self,
base_url: str,
*,
token: str | None = None,
timeout_seconds: float = 10.0,
retries: int = 2,
transport: httpx.BaseTransport | None = None,
) -> None:
parsed = urlparse(base_url)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise ValueError("Forgejo base URL must be an absolute HTTP(S) origin")
if parsed.username or parsed.password or parsed.query or parsed.fragment:
raise ValueError("Forgejo base URL must not contain credentials, query, or fragment")
if timeout_seconds <= 0 or retries < 0:
raise ValueError("Forgejo timeout must be positive and retries non-negative")
headers = {"Authorization": f"token {token}"} if token else None
self._client = httpx.Client(
base_url=base_url.rstrip("/"),
headers=headers,
timeout=httpx.Timeout(timeout_seconds),
follow_redirects=True,
transport=transport,
)
self.retries = retries
def observe(
self, registration: RepositoryRegistration, *, observed_at: str
) -> dict[str, Any]:
repository = registration.forgejo_repository
if repository is None:
raise ValueError("registration has no Forgejo repository source")
owner, slug = repository.split("/", 1)
owner_path = quote(owner, safe="")
slug_path = quote(slug, safe="")
prefix = f"/api/v1/repos/{owner_path}/{slug_path}"
metadata = self._get_object(prefix)
if metadata.get("full_name") != repository:
raise ValueError("Forgejo repository identity does not match the registry")
branch = metadata.get("default_branch")
if not isinstance(branch, str) or not branch or branch.startswith("/"):
raise ValueError("Forgejo default branch is unavailable")
branch_payload = self._get_object(f"{prefix}/branches/{quote(branch, safe='')}")
commit = branch_payload.get("commit")
head_sha = commit.get("id") if isinstance(commit, dict) else None
if not isinstance(head_sha, str) or not re.fullmatch(r"[a-f0-9]{40,64}", head_sha):
raise ValueError("Forgejo default-branch revision is invalid")
content = self._get_object(
f"{prefix}/contents/.repo-classification.yaml",
params={"ref": head_sha},
)
if content.get("type", "file") != "file" or content.get("encoding") != "base64":
raise ValueError("Forgejo classification content has an unsupported representation")
encoded = content.get("content")
if not isinstance(encoded, str):
raise TypeError("Forgejo classification content is missing")
try:
raw = base64.b64decode(encoded, validate=True)
except (ValueError, TypeError) as exc:
raise ValueError("Forgejo classification content is not valid base64") from exc
if not raw or len(raw) > 131072:
raise ValueError("Forgejo classification content size is outside the safe bound")
classification = _classification_from_bytes(raw)
normalized = _normalize_classification(classification)
return {
"operation": "upsert",
"repository_id": registration.repository_id,
"slug": registration.slug,
"lifecycle": registration.lifecycle,
"classification": normalized,
"revision": {
"head_sha": head_sha,
"source_fingerprint": hashlib.sha256(raw).hexdigest(),
"observed_at": observed_at,
},
}
def close(self) -> None:
self._client.close()
def _get_object(
self, path: str, *, params: dict[str, str] | None = None
) -> dict[str, Any]:
for attempt in range(self.retries + 1):
try:
response = self._client.get(path, params=params)
response.raise_for_status()
payload = response.json()
if not isinstance(payload, dict):
raise TypeError("Forgejo response is not a JSON object")
return payload
except (httpx.TimeoutException, httpx.NetworkError):
if attempt == self.retries:
raise ValueError(f"Forgejo request failed after retries: {path}") from None
except httpx.HTTPStatusError as exc:
raise ValueError(
f"Forgejo request returned HTTP {exc.response.status_code}: {path}"
) from exc
except ValueError:
raise
raise AssertionError("unreachable retry loop")
def build_classification_snapshot(
registrations: tuple[RepositoryRegistration, ...],
*,
observed_at: datetime | None = None,
forgejo_client: ForgejoClassificationClient | None = None,
remote_workers: int = 8,
) -> ClassificationSnapshot:
"""Observe one atomic, full fleet generation without publishing host-local paths."""
instant = observed_at or utc_now()
generated_at = format_utc(instant)
repositories: list[dict[str, Any]] = []
diagnostics: list[dict[str, Any]] = []
def observe(registration: RepositoryRegistration) -> dict[str, Any]:
if registration.path is not None:
return _observe_registration(registration, observed_at=generated_at)
if forgejo_client is None:
raise ValueError("Forgejo registry source requires a configured Forgejo client")
return forgejo_client.observe(registration, observed_at=generated_at)
ordered = sorted(registrations, key=lambda item: item.repository_id)
local = [registration for registration in ordered if registration.path is not None]
remote = [registration for registration in ordered if registration.forgejo_repository is not None]
observations: dict[str, dict[str, Any]] = {}
errors: dict[str, Exception] = {}
for registration in local:
try:
observations[registration.repository_id] = observe(registration)
except (ClassificationError, OSError, ValueError) as exc:
errors[registration.repository_id] = exc
if remote:
if remote_workers < 1:
raise ValueError("remote_workers must be positive")
with ThreadPoolExecutor(max_workers=remote_workers) as executor:
pending = {executor.submit(observe, registration): registration for registration in remote}
for future in as_completed(pending):
registration = pending[future]
try:
observations[registration.repository_id] = future.result()
except (ClassificationError, OSError, TypeError, ValueError) as exc:
errors[registration.repository_id] = exc
for registration in ordered:
if registration.repository_id in errors:
exc = errors[registration.repository_id]
diagnostics.append(
{
"severity": "error",
"code": "repo_projection.source_invalid",
"message": f"{registration.slug}: {str(exc)[:900]}",
"repository_id": registration.repository_id,
}
)
elif registration.repository_id in observations:
repositories.append(observations[registration.repository_id])
source_revision = _sha256(
{
"registry_schema": REGISTRY_SCHEMA,
"repositories": repositories,
"diagnostics": diagnostics,
}
)
snapshot_id = _sha256(
{
"contract_id": CONTRACT_ID,
"contract_version": CONTRACT_VERSION,
"generated_at": generated_at,
"source_revision": source_revision,
}
)
return ClassificationSnapshot(
snapshot_id=snapshot_id,
generated_at=generated_at,
source_revision=source_revision,
repositories=tuple(repositories),
diagnostics=tuple(diagnostics),
)
def projection_page(
snapshot: ClassificationSnapshot,
*,
offset: int,
page_size: int,
cursor_secret: bytes,
page_cursor: str | None,
) -> dict[str, Any]:
if not 1 <= page_size <= 500:
raise ValueError("page_size must be between 1 and 500")
if offset < 0 or offset > len(snapshot.repositories):
raise ProjectionCursorError("projection cursor offset exceeds the snapshot")
page_rows = snapshot.repositories[offset : offset + page_size]
next_offset = offset + len(page_rows)
final_page = next_offset >= len(snapshot.repositories)
next_cursor = None
if not final_page:
next_cursor = encode_projection_cursor(
snapshot_id=snapshot.snapshot_id,
offset=next_offset,
page_size=page_size,
secret=cursor_secret,
)
return {
"contract_id": CONTRACT_ID,
"contract_version": CONTRACT_VERSION,
"source": {
"system": "repo-manager",
"classification_contract_id": CLASSIFICATION_CONTRACT_ID,
"classification_contract_version": CLASSIFICATION_CONTRACT_VERSION,
"producer_version": __version__,
},
"snapshot": {
"snapshot_id": snapshot.snapshot_id,
"mode": "full",
"generated_at": snapshot.generated_at,
"source_revision": snapshot.source_revision,
"page_cursor": page_cursor,
"next_cursor": next_cursor,
"final_page": final_page,
"total_repository_count": len(snapshot.repositories),
},
"repositories": list(page_rows),
"diagnostics": list(snapshot.diagnostics) if offset == 0 else [],
}
class ClassificationPublisher:
"""Bounded in-memory holder for in-flight, immutable snapshot transfers."""
def __init__(
self,
registry_path: Path,
*,
cursor_secret: str | bytes,
page_size: int = 100,
max_snapshots: int = 4,
forgejo_base_url: str | None = None,
forgejo_token: str | None = None,
forgejo_timeout_seconds: float = 10.0,
forgejo_retries: int = 2,
remote_workers: int = 8,
forgejo_transport: httpx.BaseTransport | None = None,
now: Callable[[], datetime] = utc_now,
) -> None:
secret = cursor_secret.encode() if isinstance(cursor_secret, str) else cursor_secret
if len(secret) < 32:
raise ValueError("cursor_secret must contain at least 32 bytes")
if not 1 <= page_size <= 500:
raise ValueError("page_size must be between 1 and 500")
if max_snapshots < 1:
raise ValueError("max_snapshots must be positive")
if remote_workers < 1:
raise ValueError("remote_workers must be positive")
self.registry_path = registry_path
self.cursor_secret = secret
self.page_size = page_size
self.max_snapshots = max_snapshots
self.remote_workers = remote_workers
self.now = now
self._forgejo_client = (
ForgejoClassificationClient(
forgejo_base_url,
token=forgejo_token,
timeout_seconds=forgejo_timeout_seconds,
retries=forgejo_retries,
transport=forgejo_transport,
)
if forgejo_base_url is not None
else None
)
self._snapshots: OrderedDict[str, ClassificationSnapshot] = OrderedDict()
self._lock = threading.Lock()
def fetch_page(self, cursor: str | None = None) -> dict[str, Any]:
with self._lock:
return self._fetch_page(cursor)
def _fetch_page(self, cursor: str | None = None) -> dict[str, Any]:
if cursor is None:
snapshot = build_classification_snapshot(
load_repository_registry(self.registry_path),
observed_at=self.now(),
forgejo_client=self._forgejo_client,
remote_workers=self.remote_workers,
)
self._snapshots[snapshot.snapshot_id] = snapshot
self._snapshots.move_to_end(snapshot.snapshot_id)
while len(self._snapshots) > self.max_snapshots:
self._snapshots.popitem(last=False)
return projection_page(
snapshot,
offset=0,
page_size=self.page_size,
cursor_secret=self.cursor_secret,
page_cursor=None,
)
cursor_data = decode_projection_cursor(cursor, secret=self.cursor_secret)
snapshot = self._snapshots.get(cursor_data["snapshot_id"])
if snapshot is None:
raise ProjectionCursorError("projection cursor snapshot has expired")
self._snapshots.move_to_end(snapshot.snapshot_id)
return projection_page(
snapshot,
offset=cursor_data["offset"],
page_size=cursor_data["page_size"],
cursor_secret=self.cursor_secret,
page_cursor=cursor,
)
def close(self) -> None:
if self._forgejo_client is not None:
self._forgejo_client.close()
def encode_projection_cursor(
*, snapshot_id: str, offset: int, page_size: int, secret: bytes
) -> str:
payload = json.dumps(
{"snapshot_id": snapshot_id, "offset": offset, "page_size": page_size},
separators=(",", ":"),
sort_keys=True,
).encode()
signature = hmac.new(secret, payload, hashlib.sha256).digest()
return f"{_b64encode(payload)}.{_b64encode(signature)}"
def decode_projection_cursor(cursor: str, *, secret: bytes) -> dict[str, Any]:
try:
encoded_payload, encoded_signature = cursor.split(".", 1)
payload = _b64decode(encoded_payload)
signature = _b64decode(encoded_signature)
expected = hmac.new(secret, payload, hashlib.sha256).digest()
if not hmac.compare_digest(signature, expected):
raise ProjectionCursorError("projection cursor signature is invalid")
data = json.loads(payload)
snapshot_id = str(data["snapshot_id"])
offset = int(data["offset"])
page_size = int(data["page_size"])
except ProjectionCursorError:
raise
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
raise ProjectionCursorError("projection cursor is malformed") from exc
if not re.fullmatch(r"[a-f0-9]{64}", snapshot_id):
raise ProjectionCursorError("projection cursor snapshot id is invalid")
if offset < 1 or not 1 <= page_size <= 500:
raise ProjectionCursorError("projection cursor bounds are invalid")
return {"snapshot_id": snapshot_id, "offset": offset, "page_size": page_size}
def _observe_registration(
registration: RepositoryRegistration, *, observed_at: str
) -> dict[str, Any]:
repo_root = registration.path
if repo_root is None:
raise ValueError("registration has no local checkout source")
if not repo_root.is_dir() or not (repo_root / ".git").exists():
raise ValueError("registered checkout is missing or is not a Git working copy")
classification_path = repo_root / ".repo-classification.yaml"
if not classification_path.is_file():
raise ValueError("authoritative .repo-classification.yaml is missing")
classification_status = subprocess.run(
["git", "status", "--porcelain", "--", ".repo-classification.yaml"],
cwd=repo_root,
capture_output=True,
text=True,
check=False,
)
if classification_status.returncode != 0 or classification_status.stdout.strip():
raise ValueError("authoritative classification has uncommitted changes")
classification = load_classification(repo_root)
if classification is None:
raise ValueError("authoritative classification is empty")
head_sha = _git_head(repo_root)
if head_sha is None:
raise ValueError("repository HEAD is unavailable")
normalized = _normalize_classification(classification)
return {
"operation": "upsert",
"repository_id": registration.repository_id,
"slug": registration.slug,
"lifecycle": registration.lifecycle,
"classification": normalized,
"revision": {
"head_sha": head_sha,
"source_fingerprint": hashlib.sha256(classification_path.read_bytes()).hexdigest(),
"observed_at": observed_at,
},
}
def _git_head(repo_root: Path) -> str | None:
result = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=repo_root,
capture_output=True,
text=True,
check=False,
)
value = result.stdout.strip().lower()
return value if result.returncode == 0 and re.fullmatch(r"[a-f0-9]{40,64}", value) else None
def _classification_from_bytes(raw: bytes) -> dict[str, Any]:
try:
document = yaml.safe_load(raw.decode("utf-8")) or {}
except (UnicodeDecodeError, yaml.YAMLError) as exc:
raise ValueError(f"classification YAML is invalid: {exc}") from exc
if not isinstance(document, dict):
raise TypeError("classification document must be an object")
classification = document.get("repo_classification")
if not isinstance(classification, dict):
raise TypeError("classification document has no repo_classification object")
return require_valid_classification(classification)
def _normalize_classification(classification: dict[str, Any]) -> dict[str, Any]:
return {
"category": classification["category"],
"domain": classification["domain"],
"secondary_domains": sorted(classification.get("secondary_domains") or []),
"capability_tags": sorted(classification.get("capability_tags") or []),
"business_stake": sorted(classification.get("business_stake") or []),
"business_mechanics": sorted(classification.get("business_mechanics") or []),
}
def _discover_classified_repositories(root: Path) -> tuple[Path, ...]:
resolved = root.expanduser().resolve()
if (resolved / ".git").exists() and (resolved / ".repo-classification.yaml").is_file():
return (resolved,)
if not resolved.is_dir():
raise ValueError(f"fleet root does not exist: {resolved}")
return tuple(
child
for child in sorted(resolved.iterdir())
if child.is_dir()
and (child / ".git").exists()
and (child / ".repo-classification.yaml").is_file()
)
def _sha256(value: Any) -> str:
encoded = json.dumps(value, separators=(",", ":"), sort_keys=True).encode()
return hashlib.sha256(encoded).hexdigest()
def _b64encode(value: bytes) -> str:
return base64.urlsafe_b64encode(value).rstrip(b"=").decode()
def _b64decode(value: str) -> bytes:
return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))

View file

@ -0,0 +1,81 @@
"""Read-only check of an Activity Core retirement disposition (RMGR-WP-0019-T03).
The 2026-10-04 Activity receiving run refused exactly three empty duplicate
projections. Before STATE-WP-0095-T04 may apply them, the owner's
``expected_retirement_uuids`` must match what Repo Manager observed: the same
three projections, none bound to tasks, and never the source-bound parent.
This module only compares data; it performs no hub or source writes.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
# Projections the receiving run refused as "would be retired" (evidence file
# docs/evidence/2026-10-04-historical-registrar-verification.json) with the
# hub ids observed read-only on 2026-10-06.
REFUSED_DUPLICATE_PROJECTIONS: dict[str, dict[str, str]] = {
"683724d9-4249-5e0e-88e0-7baff4330d58": {
"slug": "custodian-wp-0003",
"backing_relative_path": "workplans/archived/260707-custodian-WP-0003-event-bridge.md",
},
"644017c7-c5f2-5a64-81c3-d8ee806f8251": {
"slug": "custodian-wp-0003a",
"backing_relative_path": "workplans/custodian-WP-0003a-event-bridge-model-rules-registry.md",
},
"a5806df9-5b9a-50da-81c1-7e606076bed9": {
"slug": "custodian-wp-0003b",
"backing_relative_path": "workplans/custodian-WP-0003b-event-bridge-parser-workflow-triggers.md",
},
}
# Source-bound parent every custodian-WP-0003* file names; it must survive.
PRESERVED_PARENT_UUIDS = frozenset({"b4eb45a9-69e3-4ab0-b00c-67a53c3117c5"})
def check_disposition(
expected_retirement_uuids: list[str],
task_counts: dict[str, int] | None = None,
) -> list[str]:
"""Return problems; an empty list means the disposition is consistent."""
problems: list[str] = []
given = [u.lower() for u in expected_retirement_uuids]
if len(given) != len(set(given)):
problems.append("duplicate UUIDs in expected_retirement_uuids")
given_set = set(given)
known = set(REFUSED_DUPLICATE_PROJECTIONS)
for uuid in sorted(given_set & PRESERVED_PARENT_UUIDS):
problems.append(f"{uuid} is the preserved source-bound parent")
for uuid in sorted(given_set - known - PRESERVED_PARENT_UUIDS):
problems.append(f"{uuid} was not refused by the receiving run")
for uuid in sorted(known - given_set):
slug = REFUSED_DUPLICATE_PROJECTIONS[uuid]["slug"]
problems.append(f"{uuid} ({slug}) refused but missing from disposition")
for uuid in sorted(given_set & known):
count = (task_counts or {}).get(uuid)
if count is None:
problems.append(f"{uuid} task count not verified")
elif count != 0:
problems.append(f"{uuid} holds {count} tasks; not an empty projection")
return problems
def main(argv: list[str]) -> int:
"""Usage: python -m repo_manager.retirement_disposition DISPOSITION.json
The JSON needs ``expected_retirement_uuids`` and may carry ``task_counts``.
"""
if len(argv) != 2:
print(main.__doc__, file=sys.stderr)
return 2
data = json.loads(Path(argv[1]).read_text())
problems = check_disposition(
data.get("expected_retirement_uuids", []), data.get("task_counts")
)
print(json.dumps({"ok": not problems, "problems": problems}, indent=2))
return 0 if not problems else 1
if __name__ == "__main__":
raise SystemExit(main(sys.argv))

View file

@ -0,0 +1,155 @@
"""Read-only HTTP runtime for Repo Manager projection publication."""
from __future__ import annotations
import hmac
import os
import secrets
from contextlib import asynccontextmanager
from dataclasses import dataclass
from pathlib import Path
from fastapi import FastAPI, Header, HTTPException, Query, Response, status
from repo_manager import __version__
from repo_manager.repository_publisher import (
ClassificationPublisher,
ProjectionCursorError,
RegistryError,
load_repository_registry,
)
@dataclass(frozen=True, slots=True)
class PublisherSettings:
registry_path: Path | None = None
cursor_secret: str | None = None
api_token: str | None = None
page_size: int = 100
max_snapshots: int = 4
forgejo_base_url: str | None = None
forgejo_token: str | None = None
forgejo_timeout_seconds: float = 10.0
forgejo_retries: int = 2
remote_workers: int = 8
@classmethod
def from_env(cls) -> PublisherSettings:
registry = os.getenv("REPO_MANAGER_REGISTRY_PATH")
token = os.getenv("REPO_MANAGER_FORGEJO_TOKEN")
token_file = os.getenv("REPO_MANAGER_FORGEJO_TOKEN_FILE")
if token and token_file:
raise ValueError(
"set only one of REPO_MANAGER_FORGEJO_TOKEN and REPO_MANAGER_FORGEJO_TOKEN_FILE"
)
if token_file:
try:
token = Path(token_file).read_text(encoding="utf-8").strip()
except OSError as exc:
raise ValueError(f"cannot read Forgejo token file: {exc}") from exc
if not token:
raise ValueError("Forgejo token file is empty")
return cls(
registry_path=Path(registry) if registry else None,
cursor_secret=os.getenv("REPO_MANAGER_CURSOR_SECRET") or secrets.token_urlsafe(32),
api_token=os.getenv("REPO_MANAGER_API_TOKEN"),
page_size=int(os.getenv("REPO_MANAGER_PROJECTION_PAGE_SIZE", "100")),
max_snapshots=int(os.getenv("REPO_MANAGER_PROJECTION_MAX_SNAPSHOTS", "4")),
forgejo_base_url=os.getenv(
"REPO_MANAGER_FORGEJO_BASE_URL", "https://forgejo.coulomb.social"
),
forgejo_token=token,
forgejo_timeout_seconds=float(
os.getenv("REPO_MANAGER_FORGEJO_TIMEOUT_SECONDS", "10")
),
forgejo_retries=int(os.getenv("REPO_MANAGER_FORGEJO_RETRIES", "2")),
remote_workers=int(os.getenv("REPO_MANAGER_FORGEJO_WORKERS", "8")),
)
def create_app(*, settings: PublisherSettings | None = None) -> FastAPI:
resolved = settings or PublisherSettings.from_env()
publisher: ClassificationPublisher | None = None
configuration_error: str | None = None
try:
if resolved.registry_path is None:
raise ValueError("REPO_MANAGER_REGISTRY_PATH is required")
cursor_secret = resolved.cursor_secret or secrets.token_urlsafe(32)
publisher = ClassificationPublisher(
resolved.registry_path,
cursor_secret=cursor_secret,
page_size=resolved.page_size,
max_snapshots=resolved.max_snapshots,
forgejo_base_url=resolved.forgejo_base_url,
forgejo_token=resolved.forgejo_token,
forgejo_timeout_seconds=resolved.forgejo_timeout_seconds,
forgejo_retries=resolved.forgejo_retries,
remote_workers=resolved.remote_workers,
)
except ValueError as exc:
configuration_error = str(exc)
@asynccontextmanager
async def lifespan(_app: FastAPI):
yield
if publisher is not None:
publisher.close()
app = FastAPI(
title="Repo Manager projection publisher",
version=__version__,
description="Read-only port.repo repository projection publication.",
lifespan=lifespan,
)
app.state.publisher = publisher
app.state.configuration_error = configuration_error
@app.get("/healthz", tags=["system"])
async def healthz() -> dict[str, str]:
return {"status": "ok", "service": "repo-manager", "version": __version__}
@app.get("/readyz", tags=["system"])
async def readyz(response: Response) -> dict[str, str | int]:
error = app.state.configuration_error
if error is None:
try:
registrations = load_repository_registry(resolved.registry_path) # type: ignore[arg-type]
except RegistryError as exc:
error = str(exc)
if error is not None:
response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
return {"status": "degraded", "error": error}
return {"status": "ok", "repository_count": len(registrations)}
@app.get(
"/ports/repositories/classifications",
tags=["port.repo"],
openapi_extra={"x-port-id": "port.repo", "x-direction": "out"},
)
def repository_classifications(
cursor: str | None = Query(default=None, max_length=1000),
authorization: str | None = Header(default=None),
) -> dict:
if resolved.api_token is not None and not hmac.compare_digest(
authorization or "", f"Bearer {resolved.api_token}"
):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="unauthorized")
active = app.state.publisher
if active is None:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=app.state.configuration_error,
)
try:
return active.fetch_page(cursor)
except ProjectionCursorError as exc:
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc
except RegistryError as exc:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(exc)
) from exc
return app
app = create_app()

View file

@ -0,0 +1,379 @@
"""Deprecated Repo Manager CLI adapter for the SBOM Nexus product owner."""
from __future__ import annotations
import json
import os
import shutil
import subprocess
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from urllib.parse import quote
import httpx
SNAPSHOT_SCHEMA = "sbom-nexus.snapshot.v1"
SNAPSHOT_REQUIRED_FIELDS = frozenset(
{
"schema",
"ok",
"repo_slug",
"source_revision",
"generated_at",
"entry_count",
"entries",
"sources",
"licence_report",
"errors",
}
)
PREVIEW_CONTEXT = {
"mode": "local-preview",
"authoritative": False,
"persisted": False,
"advances_last_attempt_at": False,
"advances_last_success_at": False,
"creates_snapshot_history": False,
}
SERVICE_CONTEXT = {
"mode": "authoritative-service",
"authoritative": True,
"product_owner": "sbom-nexus",
}
class SBOMContractError(ValueError):
"""Raised when a Nexus response is outside the pinned consumer contract."""
class SBOMServiceError(RuntimeError):
"""A sanitized, deterministic failure from the authoritative service client."""
def __init__(
self,
code: str,
method: str,
path: str,
*,
status_code: int | None = None,
mutation_may_have_committed: bool = False,
) -> None:
self.code = code
self.method = method
self.path = path
self.status_code = status_code
self.mutation_may_have_committed = mutation_may_have_committed
status = f" HTTP {status_code}" if status_code is not None else ""
super().__init__(f"SBOM Nexus {method} {path} failed ({code}{status})")
def to_dict(self) -> dict[str, Any]:
return {
"code": self.code,
"method": self.method,
"path": self.path,
"status_code": self.status_code,
"mutation_may_have_committed": self.mutation_may_have_committed,
}
@dataclass(frozen=True)
class SBOMNexusConfig:
"""Explicit service configuration; credentials are deliberately repr-hidden."""
base_url: str
timeout_seconds: float = 30.0
bearer_token: str | None = field(default=None, repr=False)
def __post_init__(self) -> None:
url = httpx.URL(self.base_url)
if url.scheme not in {"http", "https"} or not url.host:
raise ValueError("SBOM Nexus base_url must be an absolute HTTP(S) URL")
if url.username or url.password or url.query or url.fragment:
raise ValueError("SBOM Nexus base_url must not contain credentials, query, or fragment")
if not 0.1 <= self.timeout_seconds <= 300.0:
raise ValueError("SBOM Nexus timeout_seconds must be between 0.1 and 300")
object.__setattr__(self, "base_url", self.base_url.rstrip("/"))
@classmethod
def from_environment(cls) -> SBOMNexusConfig:
base_url = os.getenv("SBOM_NEXUS_URL")
if not base_url:
raise ValueError("SBOM_NEXUS_URL is required for authoritative service mode")
try:
timeout = float(os.getenv("SBOM_NEXUS_TIMEOUT_SECONDS", "30"))
except ValueError as exc:
raise ValueError("SBOM_NEXUS_TIMEOUT_SECONDS must be numeric") from exc
return cls(
base_url=base_url,
timeout_seconds=timeout,
bearer_token=os.getenv("SBOM_NEXUS_TOKEN") or None,
)
class SBOMNexusClient:
"""Bounded client for Nexus-owned authoritative repository SBOM state."""
def __init__(
self,
config: SBOMNexusConfig,
*,
transport: httpx.BaseTransport | None = None,
) -> None:
self.config = config
self._transport = transport
def upsert_repository(
self,
repo_slug: str,
*,
nexus_checkout_path: str | None,
active: bool = True,
source_ref: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Project identity/path; the path must resolve inside Nexus's controlled source plane."""
body: dict[str, Any] = {"checkout_path": nexus_checkout_path, "active": active}
if source_ref is not None:
body["source_ref"] = source_ref
result = self._request_object(
"PUT",
f"/repositories/{quote(repo_slug, safe='')}",
body=body,
operation="repository-projection",
writes_state=True,
required_fields=frozenset(
{"slug", "active"} | ({"source_ref"} if source_ref is not None else set())
),
)
if source_ref is not None and result.get("source_ref") != source_ref:
raise SBOMServiceError(
"source_ref_projection_mismatch",
"PUT",
f"/repositories/{quote(repo_slug, safe='')}",
mutation_may_have_committed=True,
)
return result
def ingest_repository(
self,
repo_slug: str,
*,
expected_source_revision: str | None = None,
operation_id: str | None = None,
) -> dict[str, Any]:
"""Persist one Nexus scan outcome without automatic mutation retries."""
headers = {"Idempotency-Key": operation_id} if operation_id else None
result = self._request_object(
"POST",
f"/sbom/{quote(repo_slug, safe='')}/ingest",
headers=headers,
operation="ingest",
writes_state=True,
required_fields=frozenset(
{"repo_slug", "snapshot_id", "status", "entry_count", "snapshot_at"}
),
)
if expected_source_revision and result.get("status") == "ingested":
actual = result.get("source_revision")
if actual != expected_source_revision:
raise SBOMServiceError(
"source_revision_mismatch",
"POST",
f"/sbom/{quote(repo_slug, safe='')}/ingest",
mutation_may_have_committed=True,
)
return result
def latest_snapshot(self, repo_slug: str) -> dict[str, Any]:
return self._request_object(
"GET",
f"/sbom/{quote(repo_slug, safe='')}",
operation="latest-snapshot",
writes_state=False,
required_fields=frozenset(
{
"repo_slug",
"last_attempt_at",
"last_success_at",
"last_status",
"entry_count",
"snapshot_id",
"entries",
}
),
)
def licence_report(self) -> dict[str, Any]:
return self._request_object(
"GET",
"/sbom/report/licences/",
operation="licence-report",
writes_state=False,
required_fields=frozenset({"groups", "copyleft_direct_prod", "copyleft_direct_count"}),
)
def _request_object(
self,
method: str,
path: str,
*,
body: dict[str, Any] | None = None,
headers: dict[str, str] | None = None,
operation: str,
writes_state: bool,
required_fields: frozenset[str],
) -> dict[str, Any]:
request_headers = dict(headers or {})
if self.config.bearer_token:
request_headers["Authorization"] = f"Bearer {self.config.bearer_token}"
try:
with httpx.Client(
base_url=self.config.base_url,
timeout=self.config.timeout_seconds,
headers=request_headers,
transport=self._transport,
) as client:
response = client.request(method, path, json=body)
except httpx.TimeoutException as exc:
raise SBOMServiceError("timeout", method, path) from exc
except httpx.RequestError as exc:
raise SBOMServiceError("transport_error", method, path) from exc
if response.status_code >= 400:
raise SBOMServiceError(
"http_error",
method,
path,
status_code=response.status_code,
mutation_may_have_committed=writes_state and response.status_code >= 500,
)
try:
payload = response.json()
except ValueError as exc:
raise SBOMServiceError(
"invalid_json",
method,
path,
mutation_may_have_committed=writes_state,
) from exc
if not isinstance(payload, dict):
raise SBOMServiceError(
"invalid_response_type",
method,
path,
mutation_may_have_committed=writes_state,
)
if required_fields - payload.keys():
raise SBOMServiceError(
"contract_error",
method,
path,
mutation_may_have_committed=writes_state,
)
payload["repo_manager_context"] = {
**SERVICE_CONTEXT,
"operation": operation,
"writes_state": writes_state,
}
return payload
def validate_snapshot_contract(payload: dict[str, Any]) -> None:
"""Accept additive fields but reject unknown schemas or missing required fields."""
if payload.get("schema") != SNAPSHOT_SCHEMA:
raise SBOMContractError(
f"unsupported SBOM Nexus schema {payload.get('schema')!r}; expected {SNAPSHOT_SCHEMA!r}"
)
missing = sorted(SNAPSHOT_REQUIRED_FIELDS - payload.keys())
if missing:
raise SBOMContractError(f"SBOM Nexus snapshot is missing required fields: {missing}")
def _mark_preview(payload: dict[str, Any]) -> dict[str, Any]:
payload["repo_manager_context"] = dict(PREVIEW_CONTEXT)
payload["delegated_by"] = "repo-manager"
payload["product_owner"] = "sbom-nexus"
return payload
def scan_repository_via_nexus(repo_root: Path, *, slug: str | None = None) -> dict[str, Any]:
"""Delegate a non-authoritative local preview to the Nexus-owned CLI."""
executable = os.getenv("SBOM_NEXUS_CLI") or shutil.which("sbom-nexus")
if not executable:
return _error(
"sbom-nexus CLI not found; install the sbom-nexus package or set "
"SBOM_NEXUS_CLI to its executable path"
)
command = [executable, "scan", str(repo_root)]
if slug:
command.extend(["--slug", slug])
try:
completed = subprocess.run(
command,
capture_output=True,
text=True,
check=False,
)
except OSError as exc:
return _error(f"could not execute SBOM Nexus CLI: {exc}")
try:
payload = json.loads(completed.stdout)
except json.JSONDecodeError:
detail = completed.stderr.strip() or completed.stdout.strip() or "no output"
return _error(
f"SBOM Nexus CLI returned invalid JSON (exit {completed.returncode}): {detail}"
)
if not isinstance(payload, dict):
return _error("SBOM Nexus CLI returned a non-object JSON response")
try:
validate_snapshot_contract(payload)
except SBOMContractError as exc:
return _error(str(exc), reason="sbom-nexus-contract")
_mark_preview(payload)
if completed.returncode and payload.get("ok", True):
payload["ok"] = False
payload.setdefault("errors", []).append(
{
"reason": "sbom-nexus-exit",
"detail": completed.stderr.strip() or f"exit {completed.returncode}",
}
)
return payload
def licence_report_from_snapshot(snapshot: dict[str, Any]) -> dict[str, Any]:
"""Preserve the old rmgr report-shaped view over a Nexus snapshot."""
return {
"ok": bool(snapshot.get("ok")),
"repo_slug": snapshot.get("repo_slug"),
"source_revision": snapshot.get("source_revision"),
"generated_at": snapshot.get("generated_at"),
"entry_count": int(snapshot.get("entry_count") or 0),
"licence_report": snapshot.get("licence_report")
or {
"groups": [],
"copyleft_direct_prod": [],
"copyleft_direct_count": 0,
},
"errors": snapshot.get("errors") or [],
"repo_manager_context": dict(PREVIEW_CONTEXT),
"delegated_by": "repo-manager",
"product_owner": "sbom-nexus",
}
def _error(detail: str, *, reason: str = "sbom-nexus-delegation") -> dict[str, Any]:
return _mark_preview(
{
"schema": SNAPSHOT_SCHEMA,
"ok": False,
"entry_count": 0,
"entries": [],
"sources": [],
"errors": [{"reason": reason, "detail": detail}],
}
)

View file

@ -0,0 +1,428 @@
"""Content-silent adapter for staged-blob secret detection with pinned Gitleaks.
This is a repository control adapter, not credential custody or an authorization
boundary. It does not install hooks, alter indexes, send content off-host, or
claim history/push/CI coverage. Its caller must treat unverified as refusal.
"""
from __future__ import annotations
import argparse
import contextlib
import ctypes
import fcntl
import functools
import hashlib
import json
import os
import re
import shutil
import signal
import resource
import stat
import sys
import subprocess
import tempfile
import time
from pathlib import Path
# Resolve libc and prctl before fork; child_limits performs no library loading.
_LIBC = ctypes.CDLL(None, use_errno=True)
_PRCTL = _LIBC.prctl
_PRCTL.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong]
_PRCTL.restype = ctypes.c_int
VERSION = '8.30.1'
BINARY_SHA256 = '88f91962aa2f93ac6ab281d553b9e125f5197bbbce38f9f2437f7299c32e5509'
MAX_BLOB_BYTES = 8 * 1024 * 1024
MAX_TOTAL_BYTES = 64 * 1024 * 1024
MAX_BLOBS = 500
MAX_BATCH_BLOBS = 64
MAX_BATCH_BYTES = 8 * 1024 * 1024
# PIPE_BUF bytes fit the original stdin writer's single atomic write. Larger
# blobs retain the stdin path, avoiding a change to fragment boundaries.
DIRECTORY_BLOB_BYTES = 4096
MAX_REPORT_BYTES = 1024 * 1024
SCAN_SECONDS = 120
CONFIG = '[extend]\nuseDefault = true\n'
class Unverified(Exception):
"""A fixed reason code, never a tool exception or repository content."""
def git(repo: Path, *args: str) -> bytes:
env = {**os.environ, 'GIT_NO_REPLACE_OBJECTS': '1'}
result = subprocess.run(['git', '-C', str(repo), *args], stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL, env=env, timeout=20, check=False,
preexec_fn=functools.partial(child_limits, MAX_TOTAL_BYTES, os.getpid()))
if result.returncode:
raise Unverified('git-read-failed')
return result.stdout
def staged_objects(repo: Path) -> tuple[bytes, list[str]]:
snapshot = git(repo, 'ls-files', '--stage', '-z')
changed = set(git(repo, 'diff', '--cached', '--name-only', '-z', '--no-renames',
'--no-ext-diff', '--no-textconv', '--diff-filter=ACMRTUXB').split(b'\0')) - {b''}
objects: set[str] = set()
seen: set[bytes] = set()
for record in snapshot.split(b'\0'):
if not record:
continue
header, path = record.split(b'\t', 1)
if path not in changed:
continue
mode, oid, stage = header.split()
if stage != b'0' or mode not in (b'100644', b'100755', b'120000'):
raise Unverified('unsupported-index-entry')
if not re.fullmatch(rb'[a-f0-9]{40}|[a-f0-9]{64}', oid):
raise Unverified('invalid-object-id')
objects.add(oid.decode('ascii'))
seen.add(path)
if changed != seen:
raise Unverified('incomplete-index-enumeration')
if len(objects) > MAX_BLOBS:
raise Unverified('staged-blob-limit')
return snapshot, sorted(objects)
# Source bytes must never be copied onto persistent storage. The watchdog owns
# no source data; its pipe EOF detects parent death without a service or hook.
_CLEANUP = """
import os, select, shutil, signal, stat, sys
path, dev, ino, uid, parent = sys.argv[1:]
root = os.open('/dev/shm', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
if os.getppid() != int(parent):
sys.exit(4)
pid = os.pidfd_open(int(parent))
if os.getppid() != int(parent):
sys.exit(4)
os.write(1, b'R')
ready, _, _ = select.select([0, pid], [], [], 180)
# Exceptional abort targets only the exact parent, never a process group.
# If kernel quiescence stalls, retain this observer/duty until death is proven.
if not ready:
try:
signal.pidfd_send_signal(pid, signal.SIGTERM)
except ProcessLookupError:
pass
dead, _, _ = select.select([pid], [], [], 5)
if not dead:
try:
signal.pidfd_send_signal(pid, signal.SIGKILL)
except ProcessLookupError:
pass
select.select([pid], [], [])
name = os.path.basename(path)
try:
info = os.stat(name, dir_fd=root, follow_symlinks=False)
except FileNotFoundError:
sys.exit(0)
if (info.st_dev, info.st_ino, info.st_uid) != (int(dev), int(ino), int(uid)):
sys.exit(3)
if not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o700:
sys.exit(3)
shutil.rmtree(name, dir_fd=root)
"""
def is_tmpfs(directory: Path) -> bool:
descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
result = ctypes.create_string_buffer(256)
if _LIBC.fstatfs(descriptor, ctypes.byref(result)):
raise OSError(ctypes.get_errno(), 'statfs failed')
return ctypes.c_long.from_buffer(result).value == 0x01021994
finally:
os.close(descriptor)
def remove_storage(directory: Path, identity: os.stat_result) -> None:
root = os.open('/dev/shm', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
try:
current = os.stat(directory.name, dir_fd=root, follow_symlinks=False)
except FileNotFoundError:
return
if ((current.st_dev, current.st_ino, current.st_uid) !=
(identity.st_dev, identity.st_ino, identity.st_uid)
or not stat.S_ISDIR(current.st_mode)
or stat.S_IMODE(current.st_mode) != 0o700):
raise Unverified('volatile-storage-identity-changed')
shutil.rmtree(directory.name, dir_fd=root)
finally:
os.close(root)
@contextlib.contextmanager
def private_storage():
if not is_tmpfs(Path('/dev/shm')):
raise Unverified('volatile-storage-unavailable')
temporary = tempfile.mkdtemp(prefix='rmgr-secret-scan-', dir='/dev/shm')
directory = Path(temporary)
identity = directory.lstat()
watcher = None
try:
if not is_tmpfs(directory):
raise Unverified('volatile-storage-unavailable')
watcher = subprocess.Popen([sys.executable, '-I', '-c', _CLEANUP, temporary,
str(identity.st_dev), str(identity.st_ino), str(identity.st_uid), str(os.getpid())],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
env={'PATH': '/usr/bin:/bin', 'LANG': 'C'}, close_fds=True)
import select
ready, _, _ = select.select([watcher.stdout], [], [], 5)
if not ready or watcher.stdout.read(1) != b'R' or watcher.poll() is not None:
raise Unverified('cleanup-watchdog-unavailable')
watcher.stdout.close()
yield directory, watcher
if watcher.poll() is not None:
raise Unverified('cleanup-watchdog-unavailable')
finally:
pending_error = sys.exc_info()[0] is not None
# All synchronous Git/scanner children have exited before yield unwinds.
try:
remove_storage(directory, identity)
finally:
if watcher is not None:
watcher.stdin.close()
watcher.stdout.close()
try:
code = watcher.wait(timeout=5)
except subprocess.TimeoutExpired:
watcher.kill()
code = watcher.wait(timeout=5)
if code and not pending_error:
raise Unverified('cleanup-watchdog-failed')
def child_limits(file_bytes: int, parent_pid: int) -> None:
# Parent loss must quiesce every source-reading/writing child. Install the
# signal before any exec and close the fork/prctl race using the exact PID.
if _PRCTL(1, signal.SIGKILL, 0, 0, 0) or os.getppid() != parent_pid:
os._exit(127)
# Linux adapter already requires /proc/self/fd. Limits apply only to children.
resource.setrlimit(resource.RLIMIT_FSIZE, (file_bytes, file_bytes))
resource.setrlimit(resource.RLIMIT_CPU, (20, 20))
resource.setrlimit(resource.RLIMIT_AS, (8 * 1024**3, 8 * 1024**3))
def object_batch(repo: Path, objects: list[str], directory: Path) -> list[tuple[str, bytes]]:
"""Bound Git's output before capture, then validate the complete framing."""
request = ''.join(oid + '\n' for oid in objects).encode('ascii')
env = {**os.environ, 'GIT_NO_REPLACE_OBJECTS': '1'}
with tempfile.TemporaryFile(dir=directory) as output:
result = subprocess.run(['git', '-C', str(repo), 'cat-file', '--batch'],
input=request, stdout=output, stderr=subprocess.DEVNULL,
env=env, timeout=20, check=False,
preexec_fn=functools.partial(child_limits,
MAX_BATCH_BYTES + len(objects) * 128, os.getpid()))
if result.returncode:
raise Unverified('git-read-failed')
output.seek(0)
blobs = []
for oid in objects:
header = output.readline(128)
match = re.fullmatch(rb'([a-f0-9]{40}|[a-f0-9]{64}) blob ([0-9]+)\n', header)
if not match or match[1].decode() != oid:
raise Unverified('invalid-object-batch')
size = int(match[2])
if size > MAX_BLOB_BYTES or size > MAX_BATCH_BYTES:
raise Unverified('staged-byte-limit')
content = output.read(size)
if len(content) != size or output.read(1) != b'\n':
raise Unverified('blob-size-mismatch')
algorithm = 'sha1' if len(oid) == 40 else 'sha256'
digest = hashlib.new(algorithm, b'blob ' + str(size).encode() + b'\0' + content)
if digest.hexdigest() != oid:
raise Unverified('blob-digest-mismatch')
blobs.append((oid, content))
if output.read(1):
raise Unverified('invalid-object-batch')
return blobs
def scan_staged(repo: Path, binary: Path) -> dict:
"""Return only fixed statuses/counts and Git blob IDs, never finding contents."""
scanned = 0
descriptor = None
executable = None
deadline = time.monotonic() + SCAN_SECONDS
try:
descriptor = os.open(binary, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
info = os.fstat(descriptor)
if (not stat.S_ISREG(info.st_mode) or info.st_mode & 0o022
or not info.st_mode & 0o111 or info.st_uid not in (0, os.getuid())
or info.st_size > 64 * 1024 * 1024):
raise Unverified('scanner-file-refused')
executable = os.memfd_create('rmgr-verified-scanner', os.MFD_ALLOW_SEALING)
digest = hashlib.sha256()
copied = 0
while chunk := os.read(descriptor, 1024 * 1024):
copied += len(chunk)
if copied > 64 * 1024 * 1024:
raise Unverified('scanner-file-refused')
digest.update(chunk)
with memoryview(chunk) as view:
while view:
written = os.write(executable, view)
if written <= 0:
raise Unverified('scanner-file-refused')
view = view[written:]
if digest.hexdigest() != BINARY_SHA256:
raise Unverified('scanner-digest-mismatch')
fcntl.fcntl(executable, fcntl.F_ADD_SEALS, fcntl.F_SEAL_WRITE | fcntl.F_SEAL_GROW
| fcntl.F_SEAL_SHRINK | fcntl.F_SEAL_SEAL)
snapshot, objects = staged_objects(repo)
blocked = []
# Neither source names nor repository config/ignore files enter this
# private directory. Reports contain ONLY neutral blob paths.
with private_storage() as (directory, watcher):
temporary = str(directory)
config = directory / 'scanner.toml'; config.write_text(CONFIG)
ignore = directory / '.gitleaksignore'; ignore.write_text('')
template = directory / 'report.tmpl'
template.write_text('{{range .}}{{.File}}\n{{end}}')
env = {'PATH': '/usr/bin:/bin', 'HOME': temporary, 'LANG': 'C',
'TMPDIR': temporary, 'GOMAXPROCS': '2'}
# One bounded metadata query replaces a size subprocess per object.
request = ''.join(oid + '\n' for oid in objects).encode('ascii')
with tempfile.TemporaryFile(dir=directory) as output:
result = subprocess.run(['git', '-C', str(repo), 'cat-file', '--batch-check'],
input=request, stdout=output, stderr=subprocess.DEVNULL,
env={**os.environ, 'GIT_NO_REPLACE_OBJECTS': '1'}, timeout=20,
check=False, preexec_fn=functools.partial(child_limits, MAX_BLOBS * 128, os.getpid()))
if result.returncode:
raise Unverified('git-read-failed')
output.seek(0)
metadata = output.read(MAX_BLOBS * 128 + 1).splitlines()
if len(metadata) != len(objects):
raise Unverified('invalid-object-batch')
sizes = []
for oid, line in zip(objects, metadata):
match = re.fullmatch(rb'([a-f0-9]{40}|[a-f0-9]{64}) blob ([0-9]+)', line)
if not match or match[1].decode() != oid:
raise Unverified('invalid-object-batch')
sizes.append(int(match[2]))
if any(size > MAX_BLOB_BYTES for size in sizes) or sum(sizes) > MAX_TOTAL_BYTES:
raise Unverified('staged-byte-limit')
def invoke(mode: str, content: bytes | None = None) -> list[str]:
if watcher.poll() is not None:
raise Unverified('cleanup-watchdog-unavailable')
remaining = deadline - time.monotonic()
if remaining <= 0:
raise Unverified('scan-deadline-exceeded')
report = directory / 'report.paths'
report.unlink(missing_ok=True)
command = [str(binary), mode, '--config', str(config),
'--gitleaks-ignore-path', str(ignore), '--ignore-gitleaks-allow',
'--redact=100', '--no-banner', '--no-color', '--log-level', 'warn',
'--exit-code', '20', '--timeout', '15']
if mode == 'dir':
command += ['--report-format', 'template', '--report-template', str(template), '--report-path', str(report),
'--max-target-megabytes', '0', 'blobs']
with tempfile.TemporaryFile(dir=directory) as diagnostics:
result = subprocess.run(command, executable=f'/proc/self/fd/{executable}',
pass_fds=(executable,), input=content, cwd=directory, env=env,
stdout=subprocess.DEVNULL, stderr=diagnostics,
timeout=min(20, remaining), check=False,
preexec_fn=functools.partial(child_limits, MAX_REPORT_BYTES, os.getpid()))
# The scanner can warn and skip a file without an error exit.
# Refuse ALL warnings/errors, including partial traversal.
diagnostics.seek(0)
messages = diagnostics.read(MAX_REPORT_BYTES + 1)
# v8.30.1 emits exactly this summary for a successful blocked
# scan. Every other warning/error is a refusal, never output.
summary = re.fullmatch(
rb'[0-9]{1,2}:[0-9]{2}(?:AM|PM) WRN leaks found: [1-9][0-9]*\n',
messages)
if (result.returncode not in (0, 20)
or (messages and not (result.returncode == 20 and summary))):
raise Unverified('scanner-failed')
if mode == 'stdin':
return ['stdin'] if result.returncode == 20 else []
if not report.exists() or report.stat().st_size > MAX_REPORT_BYTES:
raise Unverified('scanner-report-refused')
paths = report.read_bytes().splitlines()
known = {('blobs/' + oid).encode(): oid for oid in small}
if any(path not in known for path in paths):
raise Unverified('scanner-report-refused')
findings = sorted({known[path] for path in paths})
if bool(findings) != (result.returncode == 20):
raise Unverified('scanner-report-refused')
return findings
offset = 0
while offset < len(objects):
batch = []
batch_bytes = 0
while offset < len(objects) and len(batch) < MAX_BATCH_BLOBS:
size = sizes[offset]
if batch and batch_bytes + size > MAX_BATCH_BYTES:
break
batch.append(objects[offset]); batch_bytes += size; offset += 1
if watcher.poll() is not None:
raise Unverified('cleanup-watchdog-unavailable')
blobs = object_batch(repo, batch, directory)
# Check metadata again against exact bytes, including same-size
# substitution via corrupt object storage. Hashes are checked above.
expected = dict(zip(objects, sizes))
if any(len(content) != expected[oid] for oid, content in blobs):
raise Unverified('blob-size-mismatch')
payload = directory / 'blobs'; payload.mkdir(mode=0o700)
small = []
try:
for oid, content in blobs:
if len(content) > DIRECTORY_BLOB_BYTES:
if invoke('stdin', content):
blocked.append(oid)
scanned += 1
else:
path = payload / oid
with path.open('xb') as target:
target.write(content)
path.chmod(0o600)
small.append(oid)
if small:
blocked.extend(invoke('dir'))
scanned += len(small)
finally:
for path in payload.iterdir():
path.unlink()
payload.rmdir()
if git(repo, 'ls-files', '--stage', '-z') != snapshot:
raise Unverified('index-changed-during-scan')
if time.monotonic() > deadline:
raise Unverified('scan-deadline-exceeded')
return {'status': 'blocked' if blocked else 'clean', 'scope': 'changed-staged-blobs',
'scanner_version': VERSION, 'scanner_sha256': BINARY_SHA256,
'scanned_blobs': scanned, 'blocked_blob_ids': sorted(blocked)}
except Unverified as error:
reason = error.args[0]
except (OSError, ValueError, subprocess.SubprocessError):
reason = 'scan-unavailable'
finally:
if descriptor is not None:
os.close(descriptor)
if executable is not None:
os.close(executable)
return {'status': 'unverified', 'scope': 'changed-staged-blobs',
'reason': reason, 'scanned_blobs': scanned}
def exit_code(result: dict) -> int:
return {'clean': 0, 'blocked': 1, 'unverified': 2}[result['status']]
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--path', type=Path, default=Path('.'))
parser.add_argument('--binary', type=Path, required=True)
args = parser.parse_args(argv)
result = scan_staged(args.path, args.binary)
print(json.dumps(result, sort_keys=True))
return exit_code(result)
if __name__ == '__main__':
raise SystemExit(main())

View file

@ -0,0 +1,174 @@
"""Controlled Forgejo source references for authoritative SBOM ingestion."""
from __future__ import annotations
import re
import subprocess
from collections.abc import Callable
from pathlib import Path
from typing import Any
from urllib.parse import quote, urlparse
import httpx
from repo_manager.time import utc_now_text
DEFAULT_FORGEJO_BASE_URL = "https://forgejo.coulomb.social"
SOURCE_REF_KIND = "forgejo-archive-v1"
_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$")
_SHA_RE = re.compile(r"^[0-9a-f]{40}$")
_SCP_REMOTE_RE = re.compile(r"^(?:[^@]+@)?(?P<host>[^:]+):(?P<path>.+)$")
_ALLOWED_REMOTE_HOSTS = frozenset({"forgejo.coulomb.social", "forgejo-remote"})
def canonical_coulomb_repository(remote_url: str, repo_slug: str) -> str | None:
"""Normalize accepted Forgejo remotes to the strict public v1 identity."""
if not _SLUG_RE.fullmatch(repo_slug):
return None
remote_url = remote_url.strip()
parsed = urlparse(remote_url)
if "://" in remote_url:
if parsed.scheme not in {"http", "https", "ssh"}:
return None
if (
parsed.username not in {None, "git"}
or parsed.password
or parsed.query
or parsed.fragment
):
return None
host = parsed.hostname
path = parsed.path
else:
match = _SCP_REMOTE_RE.fullmatch(remote_url)
if not match:
return None
host = match.group("host")
path = match.group("path")
if host not in _ALLOWED_REMOTE_HOSTS:
return None
parts = path.strip("/").removesuffix(".git").split("/")
if parts != ["coulomb", repo_slug]:
return None
return f"coulomb/{repo_slug}"
class ForgejoSourceResolver:
"""Observe a public default-branch commit without using workstation Git state."""
def __init__(
self,
*,
base_url: str = DEFAULT_FORGEJO_BASE_URL,
timeout_seconds: float = 10.0,
transport: httpx.BaseTransport | None = None,
now: Callable[[], str] = utc_now_text,
) -> None:
parsed = httpx.URL(base_url)
if parsed.scheme not in {"http", "https"} or not parsed.host:
raise ValueError("Forgejo base_url must be an absolute HTTP(S) URL")
if parsed.username or parsed.password or parsed.query or parsed.fragment:
raise ValueError("Forgejo base_url must not contain credentials, query, or fragment")
if not 0.1 <= timeout_seconds <= 60.0:
raise ValueError("Forgejo timeout_seconds must be between 0.1 and 60")
self.base_url = base_url.rstrip("/")
self.timeout_seconds = timeout_seconds
self._transport = transport
self._now = now
def resolve(
self,
repo_root: Path,
*,
repo_slug: str | None = None,
remote_name: str = "origin",
) -> dict[str, Any]:
repo_root = repo_root.expanduser().resolve()
slug = repo_slug or repo_root.name
if not _SLUG_RE.fullmatch(slug):
return self._unsupported(slug, "invalid-repo-slug")
remote = subprocess.run(
["git", "remote", "get-url", remote_name],
cwd=repo_root,
capture_output=True,
text=True,
check=False,
)
if remote.returncode != 0:
return self._unsupported(slug, "missing-canonical-remote")
repository = canonical_coulomb_repository(remote.stdout, slug)
if repository is None:
return self._unsupported(slug, "non-coulomb-or-identity-mismatch")
repo_path = f"/api/v1/repos/coulomb/{quote(slug, safe='')}"
metadata = self._get_object(repo_path)
if metadata is None:
return self._unsupported(slug, "source-unresolvable")
if metadata.get("private") is True:
return self._unsupported(slug, "private-repository")
if metadata.get("full_name") != repository:
return self._unsupported(slug, "forgejo-identity-mismatch")
branch = metadata.get("default_branch")
if not isinstance(branch, str) or not branch or branch.startswith("/"):
return self._unsupported(slug, "default-branch-unresolvable")
observed_ref = f"refs/heads/{branch}"
ref_path = f"{repo_path}/git/refs/heads/{quote(branch, safe='')}"
refs = self._get_json(ref_path)
if isinstance(refs, dict):
refs = [refs]
if not isinstance(refs, list):
return self._unsupported(slug, "default-branch-unresolvable")
matching = next(
(item for item in refs if isinstance(item, dict) and item.get("ref") == observed_ref),
None,
)
obj = matching.get("object") if matching else None
revision = obj.get("sha") if isinstance(obj, dict) else None
if not isinstance(revision, str) or not _SHA_RE.fullmatch(revision):
return self._unsupported(slug, "default-branch-unresolvable")
return {
"ok": True,
"supported": True,
"repo_slug": slug,
"source_ref": {
"kind": SOURCE_REF_KIND,
"repository": repository,
"revision": revision,
"observed_ref": observed_ref,
"observed_at": self._now(),
},
}
def _get_object(self, path: str) -> dict[str, Any] | None:
payload = self._get_json(path)
return payload if isinstance(payload, dict) else None
def _get_json(self, path: str) -> Any | None:
try:
with httpx.Client(
base_url=self.base_url,
timeout=self.timeout_seconds,
transport=self._transport,
) as client:
response = client.get(path)
except httpx.RequestError:
return None
if response.status_code != 200:
return None
try:
return response.json()
except ValueError:
return None
@staticmethod
def _unsupported(repo_slug: str, reason: str) -> dict[str, Any]:
return {
"ok": True,
"supported": False,
"repo_slug": repo_slug,
"reason": reason,
"source_ref": None,
}

View file

@ -0,0 +1,315 @@
"""Repository standards as Repo Manager implements Custodian canon.
Canon remains authoritative. This module does not fork the rules; it names
the signals, required files, and anti-patterns the canon already states.
Sources:
- the-custodian/canon/standards/project-repository-flavor_v0.1.md
- the-custodian/canon/standards/repo-classification-standard_v1.0.md
- ADR-007 decision 1 (prefix from repo/project identity, never flavor)
"""
from __future__ import annotations
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Literal
from repo_manager.observe import load_classification
from repo_manager.parse.workplan import parse_frontmatter
from repo_manager.record_identity import scan_record_identities
CANON_FLAVOR = "the-custodian/canon/standards/project-repository-flavor_v0.1.md"
CANON_CLASSIFICATION = "the-custodian/canon/standards/repo-classification-standard_v1.0.md"
DURABLE_FLAVORS = ("experimental", "research", "tooling", "product", "business", "publication")
CATEGORIES = (*DURABLE_FLAVORS, "project")
FLAVOR_MARKER_PREFIX = "PRJ-WP"
WP_ID_RE = re.compile(r"^([A-Z][A-Z0-9]*)-WP-(\d{4})(?:-T\d{2})?$")
GOAL_SECTIONS = ("outcome", "invariants", "success gates", "project retirement")
Severity = Literal["missing", "contradictory", "warning"]
@dataclass
class Finding:
code: str
severity: Severity
path: str
message: str
canon: str
@dataclass
class FlavorResolution:
flavor: str
prj_layout: bool
signals: dict[str, str | None]
disagreements: list[str]
@dataclass
class ConformanceReport:
slug: str
flavor: FlavorResolution
findings: list[Finding] = field(default_factory=list)
@property
def ok(self) -> bool:
return not any(f.severity in {"missing", "contradictory"} for f in self.findings)
def to_dict(self) -> dict[str, Any]:
return {
"ok": self.ok,
"slug": self.slug,
"flavor": self.flavor.flavor,
"prj_layout": self.flavor.prj_layout,
"signals": self.flavor.signals,
"disagreements": self.flavor.disagreements,
"findings": [f.__dict__ for f in self.findings],
"canon": [CANON_FLAVOR, CANON_CLASSIFICATION],
}
def _slug(repo_root: Path) -> str:
return re.sub(r"[^a-z0-9]+", "-", repo_root.name.lower()).strip("-") or "repo"
def _goal_flavor(repo_root: Path) -> str | None:
path = repo_root / "GOAL.md"
if not path.is_file():
return None
fm, _ = parse_frontmatter(path.read_text(encoding="utf-8"))
raw = fm.get("repo_flavor")
return str(raw).strip().lower() if raw else None
def resolve_flavor(repo_root: Path, *, slug: str | None = None) -> FlavorResolution:
"""Precedence: classification category, GOAL.md repo_flavor, slug prefix."""
slug = slug or _slug(repo_root)
classification = load_classification(repo_root) or {}
category = classification.get("category")
category = str(category).strip().lower() if category else None
goal_flavor = _goal_flavor(repo_root)
slug_prj = slug.startswith("prj-")
signals = {
"classification.category": category,
"GOAL.md.repo_flavor": goal_flavor,
"slug_prefix": "prj-" if slug_prj else None,
}
if category in CATEGORIES:
flavor = category
elif goal_flavor in CATEGORIES:
flavor = goal_flavor
elif slug_prj:
flavor = "project"
else:
flavor = "tooling"
disagreements: list[str] = []
if category and goal_flavor and category != goal_flavor:
disagreements.append(
f"classification.category={category!r} disagrees with GOAL.md repo_flavor={goal_flavor!r}"
)
if slug_prj and flavor != "project":
disagreements.append(f"slug is prj-* but resolved flavor is {flavor!r}")
if category == "project" and not slug_prj and goal_flavor != "project":
disagreements.append(
"category is project but slug is not prj-*; not every category:project is a prj- repo"
)
prj_layout = slug_prj or goal_flavor == "project"
return FlavorResolution(
flavor=flavor,
prj_layout=prj_layout,
signals=signals,
disagreements=disagreements,
)
def required_files(flavor: FlavorResolution) -> tuple[str, ...]:
common = ("README.md", "SCOPE.md", "AGENTS.md", ".repo-classification.yaml", "workplans")
if flavor.prj_layout:
return ("GOAL.md", *common)
return ("INTENT.md", *common)
def purpose_document(flavor: FlavorResolution) -> str:
return "GOAL.md" if flavor.prj_layout else "INTENT.md"
def expected_workplan_prefix(repo_root: Path, *, slug: str | None = None) -> str | None:
"""Prefix must come from repo/project identity, never the flavor marker."""
slug = slug or _slug(repo_root)
if slug.startswith("prj-"):
stem = slug.removeprefix("prj-")
letters = re.sub(r"[^a-z0-9]", "", stem)
if letters:
return letters[:8].upper() + "-WP"
return None
compact = re.sub(r"[^a-z0-9]", "", slug)
if not compact:
return None
return compact[:8].upper() + "-WP"
def workplan_ids(repo_root: Path) -> list[tuple[str, str, str]]:
"""Return (path, prefix, number) for each PREFIX-WP-NNNN workplan id."""
found: list[tuple[str, str, str]] = []
wp_dir = repo_root / "workplans"
if not wp_dir.is_dir():
return found
for path in sorted(wp_dir.rglob("*.md")):
if path.name.startswith("ADHOC-"):
continue
fm, _ = parse_frontmatter(path.read_text(encoding="utf-8"))
ident = fm.get("id")
if ident is None:
continue
match = WP_ID_RE.match(str(ident).strip())
if not match:
continue
found.append((str(path.relative_to(repo_root)), match.group(1) + "-WP", match.group(2)))
return found
def check_repository(repo_root: Path, *, slug: str | None = None) -> ConformanceReport:
repo_root = repo_root.resolve()
slug = slug or _slug(repo_root)
flavor = resolve_flavor(repo_root, slug=slug)
report = ConformanceReport(slug=slug, flavor=flavor)
for note in flavor.disagreements:
report.findings.append(
Finding(
code="flavor-signal-disagreement",
severity="warning",
path=".repo-classification.yaml",
message=note,
canon=CANON_CLASSIFICATION,
)
)
for rel in required_files(flavor):
target = repo_root / rel
present = target.is_dir() if rel.endswith("/") or rel == "workplans" else target.is_file()
if not present:
report.findings.append(
Finding(
code="required-file-missing",
severity="missing",
path=rel,
message=f"{rel} is required for flavor {flavor.flavor}",
canon=CANON_FLAVOR if flavor.prj_layout else CANON_CLASSIFICATION,
)
)
has_intent = (repo_root / "INTENT.md").is_file()
has_goal = (repo_root / "GOAL.md").is_file()
if flavor.prj_layout and has_intent and has_goal:
report.findings.append(
Finding(
code="intent-and-goal",
severity="contradictory",
path="INTENT.md",
message="prj- flavor forbids shipping both INTENT.md and GOAL.md",
canon=CANON_FLAVOR,
)
)
if flavor.prj_layout and has_intent and not has_goal:
report.findings.append(
Finding(
code="intent-instead-of-goal",
severity="contradictory",
path="INTENT.md",
message="prj- flavor requires GOAL.md and must not use INTENT.md as the purpose document",
canon=CANON_FLAVOR,
)
)
if not flavor.prj_layout and has_intent and has_goal:
report.findings.append(
Finding(
code="intent-and-goal",
severity="warning",
path="GOAL.md",
message="durable flavors use INTENT.md; GOAL.md is the project-flavor purpose document",
canon=CANON_FLAVOR,
)
)
if flavor.prj_layout and has_goal:
body = (repo_root / "GOAL.md").read_text(encoding="utf-8").lower()
missing = [name for name in GOAL_SECTIONS if name not in body]
if missing:
report.findings.append(
Finding(
code="goal-sections-missing",
severity="missing",
path="GOAL.md",
message="GOAL.md must include " + ", ".join(GOAL_SECTIONS) + f"; missing {missing}",
canon=CANON_FLAVOR,
)
)
ids = workplan_ids(repo_root)
prefixes = {prefix for _path, prefix, _num in ids}
if FLAVOR_MARKER_PREFIX in prefixes:
report.findings.append(
Finding(
code="flavor-derived-prefix",
severity="contradictory",
path="workplans/",
message="PRJ-WP- is derived from the flavor marker; prefixes must come from project identity",
canon=CANON_FLAVOR,
)
)
if len(prefixes) > 1:
report.findings.append(
Finding(
code="multiple-prefixes",
severity="warning",
path="workplans/",
message=f"repository uses more than one workplan prefix: {sorted(prefixes)}",
canon="ADR-007",
)
)
identity = scan_record_identities(repo_root)
for item in identity["invalid_identifiers"]:
report.findings.append(
Finding(
code="work-record-id-invalid",
severity="contradictory",
path=item["source"],
message=f"{item['kind']} id {item['id']!r} is not accepted by the canon kind registry",
canon=identity["canon"],
)
)
for item in identity["duplicate_source_occurrences"]:
report.findings.append(
Finding(
code="work-record-duplicate-source-occurrence",
severity="warning",
path=item["sources"][0],
message=(
f"{item['kind']} {item['id']!r} repeats with UUID {item['uuid']}; "
f"index once and clean up sources {item['sources']}"
),
canon=identity["canon"],
)
)
for item in identity["identity_collisions"]:
report.findings.append(
Finding(
code="work-record-identity-collision",
severity="contradictory",
path=item["sources"][0],
message=(
f"{item['kind']} {item['id']!r} has conflicting UUID assignments "
f"at {item['sources']}"
),
canon=identity["canon"],
)
)
return report

View file

@ -0,0 +1,26 @@
"""Canonical UTC time helpers (RMGR-ADR-002)."""
from __future__ import annotations
from datetime import UTC, date, datetime
def utc_now() -> datetime:
"""Return the current timezone-aware UTC instant."""
return datetime.now(UTC)
def format_utc(value: datetime) -> str:
"""Serialize an aware instant as canonical RFC 3339 UTC with ``Z``."""
if value.tzinfo is None or value.utcoffset() is None:
raise ValueError("timestamp must be timezone-aware")
return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
def utc_now_text() -> str:
return format_utc(utc_now())
def utc_today() -> date:
"""Return the calendar date derived from the current UTC instant."""
return utc_now().date()

View file

@ -0,0 +1,238 @@
"""Read-only index and resolver for authoritative rapp workload declarations."""
from __future__ import annotations
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any
import yaml
REFERENCE_CONTRACT = "helixforge.workload-reference/v1"
@dataclass(frozen=True)
class WorkloadRecord:
rapp_id: str
name: str
declaration_repo: str
declaration_path: str
ownership_repo: str | None
readiness_state: str | None
data_classification: str | None
criticality: str | None
deployables: tuple[str, ...]
def to_dict(self) -> dict[str, Any]:
data = asdict(self)
data["deployables"] = list(self.deployables)
data["reference"] = {"rapp_id": self.rapp_id, "name": self.name}
return data
def _error(code: str, path: Path, message: str) -> dict[str, str]:
return {"code": code, "path": str(path), "message": message}
def _package_dirs(root: Path) -> list[Path]:
root = root.expanduser().resolve()
if root.name.startswith("rapp-"):
return [root]
return sorted(path for path in root.glob("rapp-*") if path.is_dir())
def _text(value: Any) -> str | None:
if not isinstance(value, str):
return None
stripped = value.strip()
return stripped or None
def index_workloads(root: Path) -> dict[str, Any]:
"""Index authoritative declarations without inventing missing identity."""
root = root.expanduser().resolve()
records: list[WorkloadRecord] = []
errors: list[dict[str, str]] = []
for package_dir in _package_dirs(root):
declaration = package_dir / "declarations" / "rapp.yaml"
if not declaration.is_file():
errors.append(
_error(
"declaration_missing",
declaration,
"rapp repository has no authoritative declarations/rapp.yaml",
)
)
continue
try:
raw = yaml.safe_load(declaration.read_text(encoding="utf-8")) or {}
except (OSError, yaml.YAMLError) as exc:
errors.append(_error("declaration_invalid", declaration, str(exc)))
continue
if not isinstance(raw, dict):
errors.append(
_error("declaration_invalid", declaration, "declaration root must be a mapping")
)
continue
if raw.get("kind") != "managed-workload-package":
errors.append(
_error(
"declaration_kind_invalid",
declaration,
"kind must be managed-workload-package",
)
)
continue
rapp_id = _text(raw.get("rapp_id"))
identity = raw.get("workload_identity")
name = _text(identity.get("name")) if isinstance(identity, dict) else None
if not rapp_id or not name:
errors.append(
_error(
"identity_missing",
declaration,
"rapp_id and workload_identity.name are required",
)
)
continue
if rapp_id != package_dir.name:
errors.append(
_error(
"rapp_repo_mismatch",
declaration,
f"rapp_id {rapp_id!r} does not match declaration repo {package_dir.name!r}",
)
)
continue
deployables: list[str] = []
composition = raw.get("composition")
members = composition.get("member_repos", []) if isinstance(composition, dict) else []
if not isinstance(members, list):
errors.append(
_error(
"composition_invalid",
declaration,
"composition.member_repos must be a list",
)
)
continue
for member in members:
if not isinstance(member, dict) or not isinstance(member.get("deployables"), list):
errors.append(
_error(
"composition_invalid",
declaration,
"every composition member must declare a deployables list",
)
)
continue
deployables.extend(
value for value in (_text(item) for item in member["deployables"]) if value
)
if not deployables:
errors.append(
_error(
"deployables_missing",
declaration,
"the workload declaration has no running deployables",
)
)
continue
records.append(
WorkloadRecord(
rapp_id=rapp_id,
name=name,
declaration_repo=package_dir.name,
declaration_path=str(declaration.relative_to(root)),
ownership_repo=_text(raw.get("ownership_repo")),
readiness_state=_text(raw.get("readiness_state")),
data_classification=_text(raw.get("data_classification")),
criticality=_text(raw.get("criticality")),
deployables=tuple(sorted(set(deployables))),
)
)
references: dict[tuple[str, str], list[WorkloadRecord]] = {}
deployable_owners: dict[str, list[WorkloadRecord]] = {}
for record in records:
references.setdefault((record.rapp_id, record.name), []).append(record)
for deployable in record.deployables:
deployable_owners.setdefault(deployable, []).append(record)
for reference, owners in sorted(references.items()):
if len(owners) > 1:
errors.append(
_error(
"reference_duplicate",
root,
f"workload reference {reference!r} resolves to {len(owners)} declarations",
)
)
for deployable, owners in sorted(deployable_owners.items()):
if len(owners) > 1:
errors.append(
_error(
"deployable_duplicate",
root,
f"deployable {deployable!r} belongs to more than one rapp: "
+ ", ".join(sorted(owner.rapp_id for owner in owners)),
)
)
return {
"ok": not errors,
"contract": REFERENCE_CONTRACT,
"root": str(root),
"declaration_count": len(records),
"workloads": [record.to_dict() for record in sorted(records, key=lambda r: r.rapp_id)],
"errors": errors,
}
def resolve_workload(
root: Path,
*,
rapp_id: str,
name: str,
deployable: str | None = None,
) -> dict[str, Any]:
"""Resolve an explicit reference, returning unknown instead of guessing."""
index = index_workloads(root)
matches = [
record
for record in index["workloads"]
if record["rapp_id"] == rapp_id and record["name"] == name
]
reference: dict[str, str] = {"rapp_id": rapp_id, "name": name}
if deployable:
reference["deployable"] = deployable
if len(matches) != 1:
return {
"ok": True,
"contract": REFERENCE_CONTRACT,
"status": "unknown",
"reference": reference,
"reason": "not_found" if not matches else "ambiguous",
"index_errors": index["errors"],
}
workload = matches[0]
if deployable and deployable not in workload["deployables"]:
return {
"ok": True,
"contract": REFERENCE_CONTRACT,
"status": "unknown",
"reference": reference,
"reason": "deployable_not_declared",
"index_errors": index["errors"],
}
return {
"ok": True,
"contract": REFERENCE_CONTRACT,
"status": "resolved",
"reference": reference,
"workload": workload,
"index_errors": index["errors"],
}

View file

@ -0,0 +1,177 @@
---
id: RAIL-BS-WP-0002
type: workplan
title: "k3s and Kubernetes Platform Baseline"
domain: financials
repo: railiance-cluster
status: finished
owner: railiance
topic_slug: railiance
repo_goal_id: "70ab2379-fb9d-4fec-a09d-b2a717e4ace8"
state_hub_workstream_id: "4c63dfc6-9eac-4e79-9f77-8f644ad7147d"
created: "2026-03-09"
updated: "2026-03-10"
completed: "2026-03-10"
---
> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
# k3s and Kubernetes Platform Baseline
## Goal
Install k3s, Helm, and the baseline Kubernetes services on the converged
HostEurope node. This workplan picks up exactly where `railiance-hosts`
leaves off: a hardened, verified OS node that is ready for Kubernetes.
## Pre-condition
`railiance-infra` converge + Goss verify must pass before any task here
is executed:
```bash
# In railiance-infra/
make converge
make verify # must exit 0
```
## Boundary
This repo owns everything from k3s upward. It must not re-configure items
defined in `railiance-infra/spec/server-baseline.yaml`. See ADR-003:
`railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`.
**Out of scope here:** platform services (PostgreSQL, storage, identity)
→ `railiance-platform`. Application deployments (Gitea, coulomb services)
→ `railiance-apps`.
---
## Tasks
### T01 — Ansible playbook: install k3s (server mode)
```task
id: T01
status: done
priority: high
state_hub_task_id: "3f042630-eab0-4c6a-9167-e2b28ff20e40"
completed: "2026-03-10"
```
Harden `ansible/bootstrap.yml` to a production-ready k3s install:
- Use the official k3s install script pinned to a specific version
(`INSTALL_K3S_VERSION=v1.35.1+k3s1`)
- `INSTALL_K3S_EXEC="server --cluster-init --write-kubeconfig-mode=644"`
(cluster-init enables embedded etcd for future HA expansion)
- Wait for node `Ready` before proceeding:
```bash
k3s kubectl wait node --all --for=condition=Ready --timeout=120s
```
- Fetch kubeconfig to the control node as `~/.kube/config-hosteurope`
**Done when:** `k3s kubectl get nodes` returns `Ready` from both the server
and the control node (via kubeconfig).
---
### T02 — Helm installation
```task
id: T02
status: done
priority: high
state_hub_task_id: "e8510646-46ed-4697-a345-f3d3009eea78"
completed: "2026-03-10"
```
Add a task (or a role `roles/helm/`) that:
1. Downloads the Helm binary (pinned version) to `/usr/local/bin/helm`
2. Verifies the checksum
3. Confirms `helm version` succeeds
**Done when:** `helm version` succeeds on the HostEurope node.
---
### T03 — Smoke test: k3s + Helm
```task
id: T03
status: done
priority: high
state_hub_task_id: "dab2c07f-8aa0-4635-8df6-857e87e93fc5"
completed: "2026-03-10"
```
Extend `tests/smoke_kube.sh` to assert:
- `k3s kubectl get nodes` → node in Ready state
- `helm version` exits 0
- CoreDNS pod running in `kube-system`
- Traefik ingress controller pod running (default in k3s)
Run via:
```bash
ansible-playbook -i ansible/hosts.ini ansible/smoke.yml
```
or directly over SSH if the kubeconfig is available locally.
**Done when:** all assertions pass and the script exits 0.
---
### T04 — Commit kubeconfig management notes
```task
id: T04
status: done
priority: medium
state_hub_task_id: "5c3d40e4-239b-488e-9519-6f7a38d2325f"
completed: "2026-03-10"
```
Document in `docs/kubeconfig.md`:
- Where the kubeconfig is fetched to (`~/.kube/config-hosteurope`)
- How to merge it into `~/.kube/config`
- How to switch context: `kubectl config use-context default`
- Security note: kubeconfig is gitignored (contains cluster CA + client cert)
**Done when:** doc written and committed.
---
### T05 — Add `make k3s-install` and `make smoke` targets
```task
id: T05
status: done
priority: medium
state_hub_task_id: "7f9e0e58-a130-467a-a2d0-b3f2564e496f"
completed: "2026-03-10"
```
Add to Makefile (create one if none exists):
```makefile
k3s-install: ## Install k3s and Helm on all inventory hosts
ansible-playbook -i ansible/hosts.ini ansible/bootstrap.yml
smoke: ## Run Kubernetes smoke tests
bash tests/smoke_kube.sh
```
**Done when:** both targets work and are listed in `make help`.
---
## References
- Repo goal: `70ab2379-fb9d-4fec-a09d-b2a717e4ace8` (Install k3s and Kubernetes Baseline)
- Domain goal: `6f96c712-60e6-4ea9-ab06-168878eafbce` (Three-Phoenix Secure Kubernetes Infrastructure)
- Pre-condition: railiance-infra WP-0001 (Secure Single-Server Bootstrap) — completed 2026-03-09
- Boundary ADR: `railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`
- k3s releases: https://github.com/k3s-io/k3s/releases

View file

@ -0,0 +1,196 @@
---
id: RAIL-BS-WP-0003
type: workplan
title: "pgpool CrashLoopBackOff on PostgreSQL HA failover — missing secret key"
domain: financials
repo: railiance-cluster
status: finished
owner: tegwick
created: "2026-03-10"
updated: "2026-03-10"
state_hub_workstream_id: "7ee9ee22-1fae-4567-9194-8d70a9e0f45b"
---
> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
# Bug Report: pgpool CrashLoopBackOff on PostgreSQL HA failover
## Summary
On 2026-03-10 a PostgreSQL HA failover caused all three postgresql pods to
restart. pgpool — the connection pooler between Gitea and PostgreSQL — then
entered CrashLoopBackOff and produced no logs. As a result Gitea's login
and all write operations hung indefinitely. The root page was still served
(from Valkey cache) which masked the failure.
The fix was to patch a missing key in a Kubernetes secret. The root cause is
that the `gitea-12.2.0` Helm chart (postgresql-ha subchart v16.2.2) does not
populate the `pgpool-password` key in the `gitea-postgresql-ha-postgresql`
secret, even though the pgpool pod requires it at startup.
---
## Timeline
| Time (UTC) | Event |
|---|---|
| ~09:45 | postgresql-0, postgresql-2 pods restarted (repmgr failover) |
| ~09:45 | pgpool pod restarted and entered CrashLoopBackOff |
| ~11:00 | User noticed Gitea login hanging; home page still loading |
| ~13:00 | Root cause identified: missing `pgpool-password` secret key |
| ~13:10 | Secret patched; pgpool pod deleted and restarted cleanly |
| ~13:15 | Gitea fully operational |
---
## Root Cause
The Bitnami `pgpool` container startup script reads the file
`/opt/bitnami/pgpool/secrets/pgpool-password`, which is mounted from the
`gitea-postgresql-ha-postgresql` Kubernetes Secret via a `subPath` volume
mount. That secret key was never created by the Helm chart, so the file did
not exist. The container exited immediately with no logs.
The pod had been running for 20 days without a restart, so this gap was
never discovered during initial deployment.
---
## Evidence
```bash
# Secret was missing the pgpool-password key
sudo k3s kubectl get secret -n default gitea-postgresql-ha-postgresql -o yaml
# data: keys were password, postgres-password, repmgr-password only
# pgpool-password was absent
# pgpool pod describe showed 824 back-off restarts over 173 minutes
# No logs in either current or --previous output
sudo k3s kubectl logs -n default <pgpool-pod> --previous
# (empty)
# Gitea process had zero TCP connections to PostgreSQL port 5432
# but many connections to Valkey port 6379
cat /proc/<gitea-pid>/net/tcp | grep 1538 # 1538 = 5432 hex — no results
```
---
## Immediate Fix Applied
```bash
# Add the missing key (value = sr-check-password = changeme4 = base64: Y2hhbmdlbWU0)
sudo k3s kubectl patch secret -n default gitea-postgresql-ha-postgresql \
--type='json' \
-p='[{"op":"add","path":"/data/pgpool-password","value":"Y2hhbmdlbWU0"}]'
# Restart pgpool
sudo k3s kubectl delete pod -n default <pgpool-pod-name>
```
---
## Risk: Fix Will Be Lost on helm upgrade
The patched secret is managed by Helm (annotation:
`meta.helm.sh/release-name: gitea`). A `helm upgrade` will regenerate the
secret from the chart template, which does not include `pgpool-password`,
and the bug will recur.
---
## Tasks
### T01 — Add pgpool-password to Helm values
```task
id: T01
status: done
priority: high
state_hub_task_id: "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc"
```
Create or update `helm/gitea-values.yaml` (or equivalent) to permanently
include the pgpool-password so it survives `helm upgrade`:
```yaml
postgresql-ha:
postgresql:
pgpoolPassword: <value matching sr-check-password>
```
**Done when:** `helm upgrade gitea` completes and pgpool starts cleanly
without manual secret patching.
---
### T02 — Add pgpool health check to smoke test
```task
id: T02
status: done
priority: high
state_hub_task_id: "ab166073-30a7-4702-a037-4091e8706e20"
```
Extend `tests/smoke_kube.sh` to assert:
```bash
# All postgresql-ha pods Running
kubectl get pods -n default | grep gitea-postgresql-ha | grep -v Running && exit 1
# pgpool specifically not in CrashLoopBackOff
kubectl get pod -n default -l app.kubernetes.io/component=pgpool \
-o jsonpath='{.items[0].status.containerStatuses[0].state}' | grep -v crash
```
**Done when:** the smoke test catches a pgpool failure within 5 minutes.
---
### T03 — Add HA failover test
```task
id: T03
status: done
priority: high
state_hub_task_id: "140da396-8e30-4f4d-b88c-c42c0cd46c01"
```
Create `tests/test_ha_failover.sh` that:
1. Records Gitea login response time (baseline)
2. Kills the primary PostgreSQL pod: `kubectl delete pod gitea-postgresql-ha-postgresql-0 -n default`
3. Waits for repmgr to promote a replica (max 60s)
4. Asserts Gitea login POST still succeeds within 10s
5. Asserts pgpool pod is Running (not CrashLoopBackOff)
6. Asserts all postgresql pods return to Running
This test must pass before any PostgreSQL HA deployment is considered done.
**Done when:** script exits 0 against a live cluster.
---
### T04 — Document the incident in docs/
```task
id: T04
status: done
priority: medium
state_hub_task_id: "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d"
```
Add `docs/incidents/2026-03-10-pgpool-missing-secret.md` with the full
timeline, root cause, and fix, so future operators understand what happened
and how to recover.
**Done when:** doc committed and linked from `docs/README.md`.
---
## References
- Bitnami postgresql-ha chart v16.2.2
- Gitea Helm chart v12.2.0
- Related decision: D3 (HA testing policy) in `DECISIONS.md`

View file

@ -0,0 +1,275 @@
---
id: RAIL-BS-WP-0004
type: workplan
title: "Integrated Backup — S2 Kubernetes Runtime Layer"
domain: financials
repo: railiance-cluster
status: finished
owner: tegwick
topic_slug: railiance
state_hub_workstream_id: "7e8b0c20-51eb-40c9-9e3b-85dd380d7625"
created: "2026-02-25"
updated: "2026-03-26"
---
> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
# Integrated Backup — S2 Kubernetes Runtime Layer
## Goal
Implement the Q3 (Operability & Resilience) integrated backup for
railiance-cluster (S2). Backs up what S2 owns — the Kubernetes runtime state —
encrypted with age, written to a local directory on the server. No external
dependencies required.
## Architecture (Decision D4)
Each railiance repo implements its own backup for what it owns. No central
backup service. See `DECISIONS.md` D4 for full rationale.
**Standard interface every railiance repo must provide:**
```bash
make backup # encrypt + write to /opt/backup/railiance/<layer>/
make restore # restore from most recent local backup
```
Encryption: age, same key pair as SOPS secrets (`.sops.yaml` public key).
Output: `/opt/backup/railiance/cluster/` on the server.
## What S2 (railiance-cluster) owns and must back up
| Asset | Why it matters |
|---|---|
| k3s etcd snapshots | Full cluster state — all workloads, configs, secrets |
| Helm release values | Runtime values not in git (any manually applied overrides) |
| kubeconfig | Admin access to the cluster |
**Not S2's responsibility:**
- Custodian State Hub DB → the-custodian owns this
- Operator workstation config (`.claude/`, `.gitconfig`) → operator's own concern
- Application data (Gitea repos, uploads) → S5 (railiance-apps) owns this
- PostgreSQL data volumes → S3 (railiance-platform) owns this
## Encryption
Reuse the age public key from `.sops.yaml`:
```bash
AGE_PUBLIC_KEY=$(grep 'age:' .sops.yaml | awk '{print $2}')
tar -czf - <assets> | age -r "${AGE_PUBLIC_KEY}" -o backup.tar.gz.age
```
Decryption requires the private key at `~/.config/sops/age/keys.txt`
(same key used for `sops -d`). No additional key management needed.
## Extension Point EP-RAIL-005
Once all five OAS layers implement this interface, the custodian can
orchestrate a full-stack backup with:
```bash
for repo in railiance-infra railiance-cluster railiance-platform \
railiance-enablement railiance-apps; do
make -C ~/$repo backup
done
```
No special protocol needed — just the standard interface.
---
## Tasks
### T01 — Define backup directory and encryption wrapper
```task
id: T01
status: done
priority: high
state_hub_task_id: "4526a842-ea31-4874-9231-92ab556cfe7b"
```
Create `tools/cmd/railiance-backup-s2` (replacing the old `railiance-backup`):
- Backup dir: `/opt/backup/railiance/cluster/` (create with `mkdir -p`)
- Encrypt each artifact with age using public key from `.sops.yaml`
- Write timestamp-named files: `etcd-<ts>.snap.age`, `helm-values-<ts>.tar.gz.age`, `kubeconfig-<ts>.yaml.age`
- Keep last 7 of each type
- Write `.last-backup` stamp
- Exit 0 on success, non-zero on any failure
- No network required
Also remove the old `tools/cmd/railiance-backup` (backed up Docker-based
custodian DB — wrong scope, not applicable to this server).
**Done when:** `make backup` runs on COULOMBCORE without error and files
appear in `/opt/backup/railiance/cluster/`.
---
### T02 — Back up k3s state (SQLite hot backup)
```task
id: T02
status: done
priority: high
state_hub_task_id: "a6313e06-1976-46a7-8e31-df4eb2eca880"
```
k3s has built-in etcd snapshot support:
```bash
sudo k3s etcd-snapshot save --name railiance-$(date -u +%Y%m%dT%H%M%SZ)
# Default location: /var/lib/rancher/k3s/server/db/snapshots/
```
Add to the backup script: take a fresh snapshot, encrypt with age,
copy to `/opt/backup/railiance/cluster/`.
> **Note — verify etcd is in use before implementing:**
> `k3s etcd-snapshot` only works if k3s was started with `--cluster-init`.
> Without it, k3s uses SQLite and this command will fail.
> Verify first: `sudo k3s etcd-snapshot ls 2>&1`
> **Note — sudo required:** etcd snapshot requires root. See T06 for how
> this is resolved (backup runs under root's crontab).
**Done when:** backup includes a current etcd snapshot.
---
### T03 — Back up Helm release values
```task
id: T03
status: done
priority: medium
state_hub_task_id: "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3"
```
Capture current runtime Helm values for all releases:
```bash
KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm list -A -o json | \
jq -r '.[].name + " " + .namespace' | \
while read name ns; do
KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm get values "$name" -n "$ns" -o yaml
done
```
Tar and age-encrypt into `helm-values-<ts>.tar.gz.age`.
> **Note — kubeconfig permissions:** `/etc/rancher/k3s/k3s.yaml` is root-readable
> only by default. The backup script must either run as root (see T06) or k3s
> must be configured with `--write-kubeconfig-mode=644`. Running as root
> (via root crontab) is the chosen approach — no config change needed.
**Done when:** backup includes a snapshot of all Helm release values.
---
### T04 — Back up kubeconfig
```task
id: T04
status: done
priority: medium
state_hub_task_id: "08233868-d522-4117-bc4e-6c0f52545665"
```
Age-encrypt `~/.kube/config-hosteurope` (or `/etc/rancher/k3s/k3s.yaml`)
into `kubeconfig-<ts>.yaml.age` in the backup directory.
**Done when:** backup includes the encrypted kubeconfig.
---
### T05 — make restore target
```task
id: T05
status: done
priority: medium
state_hub_task_id: "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8"
```
Add `tools/cmd/railiance-restore-s2` that decrypts and lists available
backups, with guided restore for the etcd snapshot case.
Restore of etcd from snapshot:
```bash
sudo k3s server --cluster-reset \
--cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/<name>
```
**Done when:** `make restore` prints available backups and a restore guide.
---
### T06 — Install cron job and run restore drill
```task
id: T06
status: done
priority: medium
state_hub_task_id: "f8e4a094-c367-40eb-b895-da17bc144b07"
```
#### Solving the sudo problem
The backup script needs root for two reasons:
- `k3s etcd-snapshot save` requires root
- `/etc/rancher/k3s/k3s.yaml` (kubeconfig) is root-readable only
**Solution: run the cron under root's crontab.**
This is the correct pattern for system-level backup jobs. It avoids a
proliferating sudoers whitelist (one entry per command, brittle to maintain)
and matches how tools like `rsnapshot`, `bacula`, and `borgbackup` work in
production. The backup writes to `/opt/backup/` which is root-owned anyway.
Install the cron as root:
```bash
sudo crontab -e
# Add:
0 2 * * * make -C /home/tegwick/railiance-cluster backup >> /opt/backup/railiance/cluster/backup.log 2>&1
```
Note: use the absolute path to the repo — `~` does not expand reliably in
root's crontab unless HOME is set.
Verify it is installed:
```bash
sudo crontab -l | grep railiance
```
#### Restore drill
Once T01–T04 are done, run a decrypt-and-verify drill:
```bash
# Decrypt the etcd snapshot and verify it is a valid snapshot file
sudo age -d -i ~/.config/sops/age/keys.txt \
/opt/backup/railiance/cluster/etcd-$(ls /opt/backup/railiance/cluster/etcd-*.snap.age | sort -r | head -1 | xargs basename | sed 's/etcd-//;s/.snap.age//').snap.age \
| file -
# Record the drill
echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) restore drill OK" \
>> /opt/backup/railiance/cluster/restore-drill.log
```
**Done when:** cron installed under root, drill completes without error,
log entry written.
---
## References
- Decision D4: Integrated backup per capability (`DECISIONS.md`)
- Decision D2: Nextcloud as optional offsite extension (still valid, not a requirement)
- OAS Q3: Operability & Resilience
- Extension point EP-RAIL-005: Custodian full-stack backup orchestration
- k3s etcd snapshots: https://docs.k3s.io/datastore/backup-restore

View file

@ -0,0 +1,52 @@
import shutil
from pathlib import Path
import pytest
from repo_manager.identifiers import ensure_missing_work_record_identifiers as ensure
from repo_manager.record_identity import scan_record_identities
FIXTURE=Path(__file__).parent/'fixtures/railiance-cluster'
@pytest.fixture
def repo(tmp_path):
root=tmp_path/'railiance-cluster';shutil.copytree(FIXTURE,root)
live=root/'workplans/RCLUSTER-WP-0018.md';live.write_text('---\nid: RCLUSTER-WP-0018\nstatus: ready\n---\n\n```task\nid: RCLUSTER-WP-0018-T01\nstatus: todo\n```\n')
return root,live
def test_exact_three_histories_preserve_15_bindings_and_strict_findings(repo):
root,live=repo;archive={p:p.read_bytes() for p in (root/'workplans/archived').glob('*.md')};before=scan_record_identities(root)
assert len(before['invalid_identifiers'])==15 and len(before['identity_collisions'])==5
preview=ensure(root,execute=False);assert len(preview['assignments'])==2
assert preview['historical_invalid_identifiers']==before['invalid_identifiers']
assert preview['historical_identity_collisions']==before['identity_collisions']
result=ensure(root,execute=True);assert len(result['assignments'])==2
assert all(p.read_bytes()==raw for p,raw in archive.items())
assert not scan_record_identities(root)['ok']
assert ensure(root,execute=False)['assignments']==[]
@pytest.mark.parametrize('change',['bytes','missing','symlink','live','nonterminal','uuid'])
def test_unknown_history_change_refuses_before_any_allocation(repo,change):
root,live=repo;p=next((root/'workplans/archived').glob('*.md'));before=live.read_bytes()
raw=p.read_bytes()
if change=='missing':p.unlink()
elif change=='symlink':p.unlink();target=root/'saved';target.write_bytes(raw);p.symlink_to(target)
elif change=='live':p.write_bytes(raw.replace(b'status: finished',b'status: active',1))
elif change=='nonterminal':p.write_bytes(raw.replace(b'status: done',b'status: todo',1))
elif change=='uuid':p.write_bytes(raw.replace(b'state_hub_task_id:',b'old_task_id:',1))
else:p.write_bytes(raw+b'\nunreviewed change\n')
with pytest.raises(ValueError):ensure(root)
assert live.read_bytes()==before
def test_any_unknown_or_live_source_in_collision_still_refuses(repo):
root,live=repo;before=live.read_bytes();other=root/'workplans/unknown.md'
other.write_text('---\nid: RCLUSTER-WP-0099\nstatus: active\n---\n\n```task\nid: T01\nstatus: todo\nstate_hub_task_id: 11111111-1111-4111-8111-111111111111\n```\n')
with pytest.raises(ValueError,match='identities are not safe'):ensure(root)
assert live.read_bytes()==before
def test_other_repo_does_not_inherit_exact_frozen_exception(repo):
root,live=repo;other=root.with_name('other');root.rename(other)
with pytest.raises(ValueError,match='identities are not safe'):ensure(other)
def test_qualified_collision_stays_global(repo):
root,live=repo;before=live.read_bytes()
(root/'workplans/other.md').write_text('---\nid: RCLUSTER-WP-0019\nstatus: active\n---\n\n```task\nid: RCLUSTER-WP-0018-T01\nstatus: done\nstate_hub_task_id: 11111111-1111-4111-8111-111111111111\n```\n')
with pytest.raises(ValueError,match='collisions='):ensure(root)
assert live.read_bytes()==before

View file

@ -0,0 +1,130 @@
from pathlib import Path
from repo_manager.identifiers import ensure_missing_work_record_identifiers
from repo_manager.record_identity import scan_record_identities
def test_custodian_scoped_legacy_ids_are_grandfathered_and_parent_scoped(tmp_path: Path) -> None:
root = tmp_path
for number, workplan_id in enumerate(
("CUST-WP-0000c", "CUST-WP-0010b", "CUST-WP-0017-LEGACY", "CUST-WP-0053-LEGACY"), start=1
):
_write_workplan(
root / "workplans" / f"{number}.md",
workplan_id,
f"{workplan_id}-T01",
f"{number:08d}-1111-4111-8111-111111111111",
)
report = scan_record_identities(root)
assert report["invalid_identifiers"] == []
assert report["identity_collisions"] == []
assert report["duplicate_source_occurrences"] == []
def test_full_legacy_bindings_dry_run_has_no_assignments(tmp_path: Path) -> None:
for number, workplan_id in enumerate(
("CUST-WP-0000c", "CUST-WP-0010b", "CUST-WP-0017-LEGACY", "CUST-WP-0053-LEGACY"), start=1
):
_write_workplan(
tmp_path / "workplans" / f"{number}.md",
workplan_id,
f"{workplan_id}-T01",
f"{number:08d}-1111-4111-8111-111111111111",
)
result = ensure_missing_work_record_identifiers(tmp_path, execute=False)
assert result["assignments"] == []
assert result["files_changed"] == []
def test_unqualified_task_remains_invalid_outside_allowlisted_parents(tmp_path: Path) -> None:
workplan = tmp_path / "workplans" / "CUST-WP-0099.md"
workplan.parent.mkdir()
workplan.write_text(
'---\nid: CUST-WP-0099\nstate_hub_workstream_id: "11111111-1111-4111-8111-111111111111"\n---\n'
'\n```task\nid: T01\nstatus: done\nstate_hub_task_id: "22222222-2222-4222-8222-222222222222"\n```\n',
encoding="utf-8",
)
report = scan_record_identities(tmp_path)
assert report["invalid_identifiers"][0]["id"] == "T01"
def _write_workplan(path: Path, workplan_id: str, task_id: str, task_uuid: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
f'---\nid: {workplan_id}\nstate_hub_workstream_id: "11111111-1111-4111-8111-111111111111"\n---\n'
f'\n```task\nid: {task_id}\nstatus: done\nstate_hub_task_id: "{task_uuid}"\n```\n',
encoding="utf-8",
)
def test_canonical_qualified_task_collision_remains_global(tmp_path: Path) -> None:
_write_workplan(
tmp_path / "workplans" / "a.md",
"CUST-WP-0099",
"CUST-WP-0099-T01",
"22222222-2222-4222-8222-222222222222",
)
_write_workplan(
tmp_path / "workplans" / "b.md",
"CUST-WP-0100",
"CUST-WP-0099-T01",
"33333333-3333-4333-8333-333333333333",
)
report = scan_record_identities(tmp_path)
assert report["identity_collisions"][0]["id"] == "CUST-WP-0099-T01"
def test_same_parent_local_alias_collision_fails_closed(tmp_path: Path) -> None:
path = tmp_path / "workplans" / "CUST-WP-0006.md"
path.parent.mkdir()
path.write_text(
'---\nid: CUST-WP-0006\nstate_hub_workstream_id: "11111111-1111-4111-8111-111111111111"\n---\n'
'\n```task\nid: T01\nstatus: done\nstate_hub_task_id: "22222222-2222-4222-8222-222222222222"\n```\n'
'\n```task\nid: T01\nstatus: done\nstate_hub_task_id: "33333333-3333-4333-8333-333333333333"\n```\n',
encoding="utf-8",
)
report = scan_record_identities(tmp_path)
assert report["identity_collisions"][0]["id"] == "T01"
def test_unknown_alias_and_missing_managed_uuid_are_invalid(tmp_path: Path) -> None:
_write_workplan(
tmp_path / "workplans" / "unknown.md",
"CUST-WP-0006",
"T99",
"22222222-2222-4222-8222-222222222222",
)
_write_workplan(tmp_path / "workplans" / "missing.md", "CUST-WP-0006", "T01", "")
report = scan_record_identities(tmp_path)
assert {item["id"] for item in report["invalid_identifiers"]} == {"T99", "T01"}
def test_all_38_local_aliases_preserve_binding_and_source_bytes(tmp_path):
before = {}
for parent, count in (
("CUST-WP-0006", 5),
("CUST-WP-0007", 17),
("CUST-WP-0030", 9),
("CUST-WP-0031", 7),
):
path = tmp_path / "workplans" / (parent + ".md")
path.parent.mkdir(exist_ok=True)
body = f'---\nid: {parent}\nstate_hub_workstream_id: "11111111-1111-4111-8111-111111111111"\n---\n'
for n in range(1, count + 1):
body += f'\n```task\nid: T{n:02d}\nstatus: done\nstate_hub_task_id: "{int(parent[-4:]):08d}-1111-4111-8111-{n:012d}"\n```\n'
path.write_text(body)
before[path] = path.read_bytes()
assert scan_record_identities(tmp_path)["ok"]
result = ensure_missing_work_record_identifiers(tmp_path, execute=True)
assert result["assignments"] == [] and result["files_changed"] == []
assert all(path.read_bytes() == body for path, body in before.items())
def test_new_suffix_task_is_not_grandfathered(tmp_path):
_write_workplan(
tmp_path / "workplans" / "new.md",
"CUST-WP-0000c",
"CUST-WP-0000c-T99",
"22222222-2222-4222-8222-222222222222",
)
assert not scan_record_identities(tmp_path)["ok"]

View file

@ -0,0 +1,56 @@
"""Exact frozen sources retain bytes and bindings while new work stays canonical."""
import hashlib
from pathlib import Path
import pytest
from repo_manager import historical_sources as hs
from repo_manager.identifiers import ensure_missing_work_record_identifiers
from repo_manager.record_identity import scan_record_identities
def _fixture(tmp_path,monkeypatch,repo='activity-core'):
root=tmp_path/repo
path=root/'workplans/old.md';path.parent.mkdir(parents=True)
rid='custodian-WP-0001' if repo=='activity-core' else 'CUST-WP-ADHOC-2026-08-25-RETIRED'
data=(f'---\nid: {rid}\ntype: workplan\nrepo: {repo}\nstatus: finished\n'
'state_hub_workstream_id: e22516cc-4fdf-4dad-a928-a9e04a9fa7c7\n---\n').encode()
path.write_bytes(data)
monkeypatch.setattr(hs,'BINDINGS',[dict(repo=repo,path='workplans/old.md',sha256=hashlib.sha256(data).hexdigest())])
prefix='ACTIVITY' if repo=='activity-core' else 'CUST'
current=root/'workplans/current.md'
current.write_text(f'---\nid: {prefix}-WP-9999\ntype: workplan\nstatus: active\n---\n')
return root,path,current,data
@pytest.mark.parametrize('repo',['activity-core','the-custodian'])
def test_frozen_source_retains_bytes_and_existing_uuid(tmp_path,monkeypatch,repo):
root,path,current,data=_fixture(tmp_path,monkeypatch,repo)
report=ensure_missing_work_record_identifiers(root)
assert path.read_bytes()==data
assert len(report['historical_invalid_identifiers'])==1
assert len(report['assignments'])==1 and report['assignments'][0]['path']=='workplans/current.md'
assert not scan_record_identities(root)['ok']
assert ensure_missing_work_record_identifiers(root)['assignments']==[]
@pytest.mark.parametrize('change',['id','uuid','status','missing','symlink','extra'])
def test_unreviewed_history_refuses_before_any_allocation(tmp_path,monkeypatch,change):
root,path,current,data=_fixture(tmp_path,monkeypatch)
original=current.read_bytes()
if change=='id':path.write_bytes(data.replace(b'custodian-WP-0001',b'custodian-WP-0002'))
elif change=='uuid':path.write_bytes(data.replace(b'e22516cc',b'f22516cc'))
elif change=='status':path.write_bytes(data.replace(b'finished',b'active'))
elif change=='missing':path.unlink()
elif change=='symlink':
path.unlink();target=root/'other.md';target.write_bytes(data);path.symlink_to(target)
else:(root/'workplans/unreviewed.md').write_bytes(data.replace(b'custodian-WP-0001',b'custodian-WP-0099'))
with pytest.raises(ValueError):ensure_missing_work_record_identifiers(root)
assert current.read_bytes()==original
def test_other_repo_cannot_inherit_frozen_exception(tmp_path,monkeypatch):
root,path,current,data=_fixture(tmp_path,monkeypatch)
other=root.with_name('other-repo');root.rename(other)
with pytest.raises(ValueError,match='identities are not safe'):
ensure_missing_work_record_identifiers(other)

View file

@ -0,0 +1,102 @@
from pathlib import Path
import pytest
from repo_manager.identifiers import ensure_missing_work_record_identifiers
from repo_manager.record_identity import scan_record_identities
def _sources(root: Path, *, status='completed', task_status='done',
task_id='NK-WP-0003-T08a', binding='22222222-2222-4222-8222-222222222222'):
archive = root / 'workplans' / 'archived' / 'NK-WP-0003.md'
archive.parent.mkdir(parents=True)
archive.write_text(
f'---\nid: NK-WP-0003\nstatus: {status}\n'
'state_hub_workstream_id: "11111111-1111-4111-8111-111111111111"\n---\n'
f'\n```task\nid: {task_id}\nstatus: {task_status}\n'
f'state_hub_task_id: "{binding}"\n```\n'
)
live = root / 'workplans' / 'NK-WP-0099.md'
live.write_text('---\nid: NK-WP-0099\nstatus: active\n---\n'
'\n```task\nid: NK-WP-0099-T01\nstatus: todo\n```\n')
return archive, live
@pytest.mark.parametrize('status', ['finished', 'archived', 'completed'])
def test_historical_warning_preserves_source_and_allows_allocation(tmp_path, status):
archive, live = _sources(tmp_path, status=status)
before = {p: p.read_bytes() for p in (archive, live)}
preview = ensure_missing_work_record_identifiers(tmp_path, execute=False)
assert all(p.read_bytes() == content for p, content in before.items())
result = ensure_missing_work_record_identifiers(tmp_path)
assert result['assignments'] == preview['assignments']
assert {x['record_id'] for x in result['assignments']} == {'NK-WP-0099', 'NK-WP-0099-T01'}
assert archive.read_bytes() == before[archive]
assert result['historical_invalid_identifiers'] == scan_record_identities(tmp_path)['invalid_identifiers']
assert result['historical_invalid_identifiers'][0]['id'] == 'NK-WP-0003-T08a'
assert not scan_record_identities(tmp_path)['ok']
assert ensure_missing_work_record_identifiers(tmp_path)['assignments'] == []
@pytest.mark.parametrize('kwargs', [
{'status': 'active'}, {'status': 'unknown'}, {'task_status': 'todo'},
{'binding': ''}, {'binding': 'not-a-uuid'}, {'task_id': 'T08a'},
{'task_id': 'T01'}, {'task_id': 'NK-WP-0099-T08a'},
])
def test_unsafe_history_refuses_before_any_allocation(tmp_path, kwargs):
paths = _sources(tmp_path, **kwargs)
before = {p: p.read_bytes() for p in paths}
with pytest.raises(ValueError, match='identities are not safe'):
ensure_missing_work_record_identifiers(tmp_path)
assert all(p.read_bytes() == content for p, content in before.items())
def test_historical_collision_still_blocks(tmp_path):
archive, live = _sources(tmp_path)
duplicate = archive.with_name('duplicate.md')
duplicate.write_text(archive.read_text().replace(
'22222222-2222-4222-8222-222222222222',
'33333333-3333-4333-8333-333333333333'))
before = live.read_bytes()
with pytest.raises(ValueError, match='collisions='):
ensure_missing_work_record_identifiers(tmp_path)
assert live.read_bytes() == before
def _railiance_item(root: Path, *, parent="RAILIANCE-WP-0004", status="finished",
task_status="done", task_id="RAILIANCE-WP-0004-I01",
binding="44444444-4444-4444-8444-444444444444") -> Path:
path = root / "workplans" / "archived" / "railiance.md"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
f'---\nid: {parent}\nstatus: {status}\n'
'state_hub_workstream_id: "33333333-3333-4333-8333-333333333333"\n---\n'
f'\n```task\nid: {task_id}\nstatus: {task_status}\n'
f'state_hub_task_id: "{binding}"\n```\n',
encoding="utf-8",
)
return path
def test_railiance_historical_item_labels_preserve_uuid(tmp_path: Path) -> None:
path = _railiance_item(tmp_path)
before = path.read_bytes()
result = ensure_missing_work_record_identifiers(tmp_path, execute=False)
assert result["assignments"] == []
assert result["files_changed"] == []
assert path.read_bytes() == before
@pytest.mark.parametrize("kwargs", [
{"status": "active"},
{"task_status": "todo"},
{"parent": "RAILIANCE-WP-0005", "task_id": "RAILIANCE-WP-0005-I01"},
{"binding": ""},
{"binding": "not-a-uuid"},
{"task_id": "RAILIANCE-WP-0004-I08"},
{"task_id": "OTHER-WP-0004-I01"},
])
def test_railiance_item_label_preservation_is_bounded(tmp_path: Path, kwargs) -> None:
_railiance_item(tmp_path, **kwargs)
with pytest.raises(ValueError, match="identities are not safe"):
ensure_missing_work_record_identifiers(tmp_path)

View file

@ -0,0 +1,207 @@
---
id: RMGR-WP-0019
type: workplan
title: "Admit exact historical source views after receiving preservation support"
domain: infotech
repo: repo-manager
status: active
owner: codex
created: "2026-10-04"
updated: "2026-10-17"
flavor: implementation
depends_on: []
state_hub_workstream_id: "efe0ecb2-2f3f-5f49-baee-b90ad1d353ab"
---
## Goal
Unblock normal authoritative registration of the exact closed Activity legacy
source views and Custodian retired ADHOC source after receiving preserves their
existing records. Preserve every legacy source byte, label and UUID; do not
expand canonical syntax or the general grandfathering rules. CUST-WP-0077 owns
cross-owner coordination with STATE-WP-0095.
## Review exact historical receiving contract
```task
id: RMGR-WP-0019-T01
status: done
priority: high
state_hub_task_id: "d78a622e-39f9-5923-b4e1-f70fdc17fb42"
```
Seven Activity sources yield four existing parents/64 closed bound tasks;
three split views repeat the archived parent. One Custodian retired source keeps
its existing parent/task IDs. Earlier /tmp/activity-core-bound-history-write.py
is superseded and must not be applied: registrar bypass alone would admit an
unsupported receiving projection. RMGR-WP-0014/0015 are finished and do not own
this new implementation demand; this workplan is not residual.
## Enable exact frozen registrar exceptions after receiving deployment
```task
id: RMGR-WP-0019-T02
status: done
priority: high
depends_on: [STATE-WP-0095-T03]
state_hub_task_id: "95240c81-1805-5365-9b35-d238621a16bf"
```
Prepared code permits only source paths with reviewed full SHA256 digests in
repo_manager/historical_sources.py. Strict conformance scans retain their legacy
findings. Unknown/missing/altered source refuses before allocation. Source UUIDs
are neither changed nor allocated for these histories. Receiving and registrar
manifests must remain identical. Apply helper requires a deployment receipt.
## Run normal owner sync and verify preserved identities
```task
id: RMGR-WP-0019-T03
status: done
priority: high
depends_on: [RMGR-WP-0019-T02]
state_hub_task_id: "682eea14-a4a8-5f33-9a9f-8f2f1ac8288d"
```
Use rmgr sync --path <owning-repo> --push. No deep-check repair or direct hub UUID
patch substitutes for receiving. Historical unrelated stale projections are not
retirement-authorized by this workplan. Feed receipts back to STATE-WP-0095-T04.
## Offline evidence, 2026-10-04
Prepared registrar suite: 38 focused cases passed (identifiers, registrar,
Custodian legacy identity and exact frozen source cases). Read-only execute=False
against both current owning repos accepted exact histories without source writes.
Implementation is staged; deployment gate is unresolved.
## Approved deployment and actual registrar verification, 2026-10-04
STATE-WP-0095-T03 returned verified main-5dd3df9 / Helm revision72.
Global task-holder refusal passed inside the delivered API against the retired
Custodian parent, with transaction rollback and no committed probe. Five
historical parents/65 tasks retained their entire row snapshots. The gated
three-file registrar patch committed as c41d36c2ef8930628221ef36ec81d8b5d2145ed6;
38 actual owning-source cases passed and Repo Manager primary sync applied.
Normal owner sync admitted all exact frozen histories without changing their
source bytes, UUIDs, parent ownership or row content. Custodian receiving
applied. Activity receiving refused only three empty duplicate projections
that would be retired; no retirement acknowledgement was issued. Registrar
allocated four missing IDs in ordinary Activity ad-hoc sources through its
normal path; frozen historical source files stayed unchanged. T03 remains wait
for explicit disposition and bounded receiving support under STATE-WP-0095-T04
(decision91b7c633-a66f-5d43-9f4b-6565aefb9356).
Evidence: docs/evidence/2026-10-04-historical-registrar-verification.json.
## Ungated T03 preparation, 2026-10-06
Re-ran the registrar suite at 8a47a5c (registrar source unchanged since
c41d36c2): 89 focused registrar/identity/historical cases and the full suite
(226) passed. Read-only hub check: the three refused projections are
683724d9 (custodian-wp-0003), 644017c7 (custodian-wp-0003a) and a5806df9
(custodian-wp-0003b), each with zero tasks; every custodian-WP-0003* source
file binds parent b4eb45a9, which holds the tasks and must not be retired.
Added repo_manager.retirement_disposition, a read-only check that an Activity
expected_retirement_uuids set equals exactly those three empty projections and
excludes the preserved parent. It authorizes nothing; retirement stays with
Activity receiving under STATE-WP-0095-T04.
## Activity disposition received, 2026-10-06
activity-core returned the exact expected_retirement_uuids (message 17ba4201,
evidence activity-core 941fac0, ACTIVITY-WP-0045): 683724d9, 644017c7,
a5806df9. repo_manager.retirement_disposition accepted the set: it equals the
three projections the 2026-10-04 run refused, each holds zero tasks, and it
excludes preserved parent b4eb45a9. Nothing was applied. T03 stays wait: its
acceptance needs a successful Activity owner sync with receipts, which can run
only after the founder decides 91b7c633 and STATE-WP-0095-T04 retires exactly
this set in a reviewed live rollout. Then re-run rmgr sync --path
~/activity-core --push and feed the receipt back to STATE-WP-0095-T04.
## Post-retirement owner sync attempt, 2026-10-06
State Hub reported decision 91b7c633 resolved, main-968a811 / Helm rev 74
deployed, and Activity reconcile at 67a92d2 retiring exactly the three empty
projections (3 retired, 0 created, 0 refused; 4 historical parents / 64 tasks
unchanged). Evidence: state-hub docs/evidence/2026-10-06-activity-stale-retirement-rollout.json.
`rmgr sync --path ~/activity-core --push` stopped before reconciliation with
status pending_commit: uncommitted `state_hub_workstream_id` / `state_hub_task_id`
writeback in activity-core workplans/ACTIVITY-WP-0045-stale-historical-projection-disposition.md
(plus WORK-RECORDS.md). No hub request was made. Repo Manager does not commit in
activity-core; T03 stays wait until Activity Core commits and the sync re-runs.
Re-run 2026-10-06 08:42Z after Activity Core committed the writeback (e809fb2):
status applied, created 0 / updated 45 (ACTIVITY-WP only) / retired 0 / refused 0,
no task changes, notes preserve custodian-WP-0001..0004 UUIDs; hub shows 4 parents,
64 tasks. Not closed: historical parent b4eb45a9 (custodian-WP-0003, source
`status: archived`, evidence snapshot archived) now reads `finished` in the hub,
updated 08:42:03Z during this sync. Needs State Hub review before T03 closes.
Closed 2026-10-06: State Hub traced the b4eb45a9 status change to its own C-04
check, which ran once per file. The custodian-WP-0003a/b/c split views share the
parent UUID, so whichever view ran last set the row. The Forge reconcile did not
cause it. Fixed in state-hub 2b5d8f9 (STATE-WP-ADHOC-2026-10-06-T01); b4eb45a9 is
back to `archived`. Re-check: 4 parents (b4eb45a9 archived; 3a4f47d9, 759b1255,
e22516cc finished) holding 64 tasks (24+12+7+21). Receipt: Activity sync at e809fb2,
primary/railiance01, applied, 0 created / 45 updated (ACTIVITY-WP only) / 0 retired
/ 0 refused, 0 task changes, historical UUIDs preserved. Evidence:
state-hub docs/evidence/2026-10-06-activity-stale-retirement-rollout.json.
## Cluster exact historical source successor, 2026-10-17
This scoped extension preserves the completed Activity/Custodian work above.
The cluster publisher now refuses three closed RAIL-BS source files containing
15 existing UUID-bound bare task labels. Preserve all three source bytes, IDs,
and UUIDs. No new alias or general syntax exemption is introduced. Evidence and
exact receiver-first proposal: cluster-historical-publication-r1 in the Custodian
NetKingdom change packet. The existing SHR-WP-0001 project driver owns the
Repo Manager / State Hub cross-owner wait; NK-WP-0049-T35 benefits from the
blocked cluster publication. No worker or human gate is woken here.
## Prepare exact cluster registrar successor
```task
id: RMGR-WP-0019-T04
status: done
priority: high
```
Prepared full-file SHA256 bindings for three archives and 15 exact task
occurrences; strict scans still return 15 invalid labels and five collisions.
Unknown/live sources, changed/missing/symlinked histories, UUID drift and
qualified/global collisions refuse before allocation. Candidate tests pass 47;
canonical application remains a separate reviewed step. The read-only current
cluster preview proposes seven ordinary UUID bindings in two files and changes
no bytes. One pre-existing ADHOC field is included explicitly: task
RCLUSTER-WP-ADHOC-2026-07-27-T01 in workplans/ADHOC-2026-07-27.md receives
3e3f39d3-5070-5ba3-97db-19f5f361f264 through managed allocation only.
## Admit cluster registrar successor only after receiving verification
```task
id: RMGR-WP-0019-T05
status: wait
priority: high
depends_on: [STATE-WP-0095-T06]
```
Retain the current guard until exact reviewed receiving source has been delivered
and verified through State Hub's normal owner deployment. Match both manifests,
then integrate reviewed registrar source and verify ordinary allocator preview.
No direct API patch, history rename or bypass substitutes for this gate.
## Run ordinary cluster owner publication and verify preservation
```task
id: RMGR-WP-0019-T06
status: wait
priority: high
depends_on: [RMGR-WP-0019-T05]
```
Only then run normal owner consistency/publication with the reviewed expected
file set: RCLUSTER-WP-0018 and the one-field ADHOC managed backfill, plus ordinary
generated indexes. Preserve all three archived files and all 18 existing
parent/task rows. Return actual receiving/publication receipts to STATE-WP-0095-T07.
Do not infer TLS recovery, credential custody or native acceptance from sync.

View file

@ -0,0 +1,26 @@
{
"schema": "receiver-readonly-preflight.v1",
"observed_at": "2026-10-16T23:32:16.582701+00:00",
"source_revision": "51bc70d1ff8f25450a8650d8e50caf053093a02e",
"image_tag": "main-51bc70d",
"production_mutations": false,
"secret_reads": false,
"limitations": [
"Chart/template server admission is not an exact Helm reuse-values upgrade diff.",
"Headroom is dated observation, not a scheduling guarantee or rollout approval.",
"No migration, pod restart, registry pull, live probe transaction or Helm command executes."
],
"target_node_confirmed": true,
"headroom_exit": 0,
"headroom_ok": true,
"headroom_summary": {
"ok": true,
"reasons": [],
"notes": [
"Aggregate remaining millicores is not a kube-scheduler guarantee.",
"Preflight does not lower requests and does not start Helm."
]
},
"server_dryrun_exit": 2,
"status": "server_dryrun_refused_or_unverified"
}

View file

@ -0,0 +1,55 @@
# Source-only publication and later read-only preflight
These helpers contain no Helm upgrade/rollback, kubectl mutation without dry-run,
Repo Manager admission or work-record sync. No credential input is accepted.
Do not call execution switches before root's independent source review and
canonical integration. Normal hooks/scanner remain enabled.
From this r2 directory, after reviewed patch application:
```sh
python3 publish_receiver_source.py
python3 publish_receiver_source.py --execute-source-commit --push-source
```
The second command changes only the explicitly hashed State Hub source paths,
commits and pushes normally. It preserves unrelated index entries and working
brief/index/evidence bytes, checks exact committed path set, and verifies the
remote main ref. A failure never retries or deploys. Retain its sanitized JSON
receipt. It deliberately fails if HEAD or reviewed source bytes drift; prepare
a new reviewed revision instead of changing its expected hashes silently.
After successful commit/push, use the returned full40character source revision:
```sh
python3 lookup_ci_metadata.py --source-revision FULL40CHAR_SOURCE_REVISION
python3 preflight_receiver.py --source-revision FULL40CHAR_SOURCE_REVISION
```
The CI lookup uses State Hub's existing public actions/tasks endpoint and public
registry manifest HEAD. It requires exact head_sha build-and-push success plus
valid sha256 image manifest digest. A chart/docs-only commit can have no image;
unavailable/public-auth refusal stays unverified without a login fallback.
Task results and manifest digest alone do not establish image internal files or
floating hub-core provenance. Capture normal image metadata and compare actual
receiver module SHA, migration source/head, hub-core package/source SHA and
installed package inventory before native approval. Image fetch/inspection
must use the existing approved owner path and never execute the image or
surface bundled credentials. This packet introduces no new credential need.
The preflight pins the existing kubeconfig, confirms railiance01 node IP before
invocation, runs the owner's headroom script (45second timeout) and server dry-run
(60second timeout), and emits fixed metadata only. It retains no raw stderr or
manifest output. It must be run close to a later rollout: dated headroom must
not be reused after workload changes. The owner Make dry-run also admits public
Secret templates and platform resources with dry-run only, never existing Secret
contents. Its values file is a public intended baseline, while real deployment
uses Helm --reuse-values. Therefore exact upgrade diff still needs a separately
reviewed values-preserving Helm dry-run; these checks do not prove equivalence.
No Helm values/manifest dump should be pasted or retained as public evidence.
The image change also includes migration c4d5e6f7a8b9. Require disposable real
PostgreSQL migration and old-image-on-new-schema compatibility tests, migration
lock/time estimates and a verified recovery plan before approval. A Helm atomic
rollback does not downgrade schema, and blindly executing Alembic downgrade
would discard feed data. No migration or new native trial was executed here.

View file

@ -0,0 +1,23 @@
{
"schema": "receiver-ci-metadata.v1",
"source_revision": "4ab66e363ed00a67ad1ea0434854bc4284c7a26e",
"image_tag": "main-4ab66e3",
"no_credentials": true,
"no_logs": true,
"ci_lookup_status": "ok",
"exact_revision_build_tasks": [
{
"id": 12477,
"name": "build-and-push",
"status": "success",
"head_sha": "4ab66e363ed00a67ad1ea0434854bc4284c7a26e",
"started_at": null,
"completed_at": null
}
],
"image_lookup_status": "public_access_refused",
"status": "artifact_unverified",
"limitations": [
"Successful task+registry digest does not prove floating dependency closure or delivered image file hashes."
]
}

View file

@ -0,0 +1,54 @@
#!/usr/bin/env python3
"""Public-only bounded CI/image metadata. No credential lookup/login/logs."""
import argparse
import json
import re
import urllib.error
import urllib.request
LIMIT = 512 * 1024
TASKS = 'https://forgejo.coulomb.social/api/v1/repos/coulomb/state-hub/actions/tasks?limit=50'
REGISTRY = 'https://forgejo.coulomb.social/v2/coulomb/state-hub/manifests/'
def request(url, method='GET', headers=None):
try:
with urllib.request.urlopen(urllib.request.Request(url, method=method, headers=headers or {}), timeout=30) as response:
raw = response.read(LIMIT+1) if method == 'GET' else b''
if len(raw) > LIMIT:
return {'status': 'oversized_refused'}
return {'status': 'ok', 'http_status': response.status, 'body': raw,
'digest': response.headers.get('Docker-Content-Digest'),
'content_type': response.headers.get('Content-Type')}
except urllib.error.HTTPError as exc:
return {'status': 'public_access_refused' if exc.code in (401,403) else 'http_error', 'http_status': exc.code}
except (urllib.error.URLError, TimeoutError):
return {'status': 'metadata_unavailable'}
def lookup(revision):
if not re.fullmatch(r'[0-9a-f]{40}',revision):
raise ValueError('exact40char_revision_required')
result={'schema':'receiver-ci-metadata.v1','source_revision':revision,
'image_tag':'main-'+revision[:7], 'no_credentials':True,'no_logs':True}
tasks=request(TASKS)
result['ci_lookup_status']=tasks['status']
result['exact_revision_build_tasks']=[]
if tasks['status']=='ok':
try:
parsed=json.loads(tasks['body'])
for task in parsed.get('workflow_runs',[]):
if task.get('head_sha')==revision and task.get('name')=='build-and-push':
result['exact_revision_build_tasks'].append({k:task.get(k) for k in ('id','name','status','head_sha','started_at','completed_at')})
except (ValueError,TypeError,AttributeError):
result['ci_lookup_status']='malformed_refused'
image=request(REGISTRY+result['image_tag'],'HEAD',{'Accept':'application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'})
result['image_lookup_status']=image['status']
digest=image.get('digest')
if image['status']=='ok' and isinstance(digest,str) and re.fullmatch(r'sha256:[0-9a-f]{64}',digest):
result['image_digest']=digest
result['status']='artifact_metadata_verified' if any(t.get('status')=='success' for t in result['exact_revision_build_tasks']) and 'image_digest' in result else 'artifact_unverified'
result['limitations']=['Successful task+registry digest does not prove floating dependency closure or delivered image file hashes.']
return result
if __name__=='__main__':
ap=argparse.ArgumentParser();ap.add_argument('--source-revision',required=True);args=ap.parse_args()
result=lookup(args.source_revision);print(json.dumps(result,indent=2));raise SystemExit(0 if result['status']=='artifact_metadata_verified' else 1)

View file

@ -0,0 +1,31 @@
{
"schema": "repo-manager.identifier-assignment.v1",
"ok": true,
"executed": true,
"repo": "state-hub",
"namespace": "helixforge",
"assignments": [
{
"kind": "task",
"record_id": "STATE-WP-0095-T05",
"uuid": "339b9a37-181c-5599-9a89-0393e199bd4b",
"path": "workplans/STATE-WP-0095-historical-source-receiving.md"
},
{
"kind": "task",
"record_id": "STATE-WP-0095-T06",
"uuid": "6af973b4-1964-5d4d-a2f8-8dbd4c01fda4",
"path": "workplans/STATE-WP-0095-historical-source-receiving.md"
},
{
"kind": "task",
"record_id": "STATE-WP-0095-T07",
"uuid": "cfd1ea99-e75b-542e-a434-acefb62f3c7d",
"path": "workplans/STATE-WP-0095-historical-source-receiving.md"
}
],
"historical_invalid_identifiers": [],
"files_changed": [
"workplans/STATE-WP-0095-historical-source-receiving.md"
]
}

View file

@ -0,0 +1,10 @@
{
"schema": "receiver-owner-ssh-image-metadata.v1",
"target": "railiance01",
"image": "forgejo.coulomb.social/coulomb/state-hub:main-51bc70d",
"read_only": true,
"pulled": false,
"exit_code": 127,
"status": "metadata_unverified",
"classification": "docker_cli_unavailable"
}

View file

@ -0,0 +1,51 @@
#!/usr/bin/env python3
"""Bounded read-only owner headroom + chart server admission. Never deploys."""
import argparse
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import re
import subprocess
OWNER=Path('/home/worsch/state-hub')
KUBECONFIG='/home/worsch/.kube/config-railiance01'
def command(argv,env,timeout):
try:
p=subprocess.run(argv,cwd=OWNER,env=env,capture_output=True,timeout=timeout)
return p.returncode,p.stdout
except subprocess.TimeoutExpired:
return 124,b''
def main():
ap=argparse.ArgumentParser();ap.add_argument('--source-revision',required=True);args=ap.parse_args()
if not re.fullmatch('[0-9a-f]{40}',args.source_revision):ap.error('exact40char_revision_required')
env=dict(os.environ);env['KUBECONFIG']=KUBECONFIG
result={'schema':'receiver-readonly-preflight.v1','observed_at':datetime.now(timezone.utc).isoformat(),
'source_revision':args.source_revision,'image_tag':'main-'+args.source_revision[:7],
'production_mutations':False,'secret_reads':False,
'limitations':['Chart/template server admission is not an exact Helm reuse-values upgrade diff.',
'Headroom is dated observation, not a scheduling guarantee or rollout approval.',
'No migration, pod restart, registry pull, live probe transaction or Helm command executes.']}
code,raw=command(['kubectl','--kubeconfig',KUBECONFIG,'get','nodes','-o','json'],env,20)
matched=False
if code==0:
try:matched=any(a.get('address')=='92.205.62.239' for n in json.loads(raw).get('items',[]) for a in n.get('status',{}).get('addresses',[]))
except (ValueError,AttributeError,TypeError):pass
result['target_node_confirmed']=matched
if not matched:
result['status']='target_identity_refused';print(json.dumps(result,indent=2));return 1
code,raw=command(['make','--no-print-directory','railiance-state-hub-headroom'],env,45)
result['headroom_exit']=code
try:
headroom=json.loads(raw);result['headroom_ok']=headroom.get('ok') is True
result['headroom_summary']={k:headroom[k] for k in ('ok','reasons','notes') if k in headroom}
except (ValueError,AttributeError):result['headroom_ok']=False
if code or not result['headroom_ok']:
result['status']='headroom_refused_or_unverified';print(json.dumps(result,indent=2));return 1
code,_=command(['make','--no-print-directory','railiance-state-hub-server-dry-run',
'RAILIANCE_STATE_HUB_IMAGE_TAG='+result['image_tag']],env,60)
result['server_dryrun_exit']=code
result['status']='readonly_checks_passed' if code==0 else 'server_dryrun_refused_or_unverified'
print(json.dumps(result,indent=2));return 0 if code==0 else 1
if __name__=='__main__':raise SystemExit(main())

View file

@ -0,0 +1,121 @@
#!/usr/bin/env python3
"""Exact-path source publication only; no deployment or work-record sync."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
BASE = Path(__file__).resolve().parent
OWNER = Path('/home/worsch/state-hub')
SCANNER = ['/usr/bin/python3', '/home/worsch/repo-manager/src/repo_manager/secret_scan.py',
'--path', str(OWNER), '--binary', '/home/worsch/.local/lib/repo-manager/gitleaks/8.30.1/gitleaks']
class Refusal(RuntimeError):
pass
def sha(raw):
return hashlib.sha256(raw).hexdigest()
def git(owner, *args, check=True):
p = subprocess.run(['git', '-C', str(owner), *args], capture_output=True, timeout=150)
if check and p.returncode:
raise Refusal('git_' + args[0] + '_refused_' + str(p.returncode))
return p
def index_entries(owner):
# Stage bytes remain private to this process. Receipts expose hashes only.
return dict((entry.split(b'\t', 1)[1].decode(), entry.split(b'\t', 1)[0])
for entry in git(owner, 'ls-files', '--stage', '-z').stdout.split(b'\0') if entry)
def dirty_files(owner):
tracked = git(owner, 'diff', '--name-only', 'HEAD', '-z').stdout.split(b'\0')
untracked = git(owner, 'ls-files', '--others', '--exclude-standard', '-z').stdout.split(b'\0')
return sorted({p.decode() for p in tracked + untracked if p})
def preserved_state(owner, owned):
indices = {p: sha(v) for p, v in index_entries(owner).items() if p not in owned}
worktree = {}
for p in dirty_files(owner):
if p in owned:
continue
file = owner / p
if file.is_symlink():
raise Refusal('unrelated_symlink_requires_review')
worktree[p] = sha(file.read_bytes()) if file.exists() else None
return {'index_entries': indices, 'dirty_worktree': worktree}
def validate(owner, expected):
if git(owner, 'rev-parse', 'HEAD').stdout.decode().strip() != expected['base_revision']:
raise Refusal('source_base_drift')
files = expected['files']
names = {f['path'] for f in files}
if len(names) != len(files):
raise Refusal('duplicate_expected_path')
for f in files:
p = owner / f['path']
if p.is_symlink() or not p.is_file() or sha(p.read_bytes()) != f['candidate_sha256']:
raise Refusal('reviewed_source_drift')
# Every unrelated worktree/index entry is retained. No stash/reset/clean.
return names, preserved_state(owner, names)
def commit_source(owner, expected, scanner, execute=False, push=False):
names, before = validate(owner, expected)
result = {'schema': 'receiver-source-publication.v1', 'scope': 'source_only',
'deployed': False, 'repo_manager_enabled': False,
'base_revision': expected['base_revision'], 'files': sorted(names),
'preserved_before': before, 'status': 'validated_not_executed'}
if not execute:
return result
# Stage exact reviewed paths first so the normal scanner inspects them.
# A refusal retains reviewed source/staging for inspection; unrelated state stays intact.
git(owner, 'add', '--', *sorted(names))
scan = subprocess.run(scanner, cwd=owner, capture_output=True, timeout=150)
result['scanner_exit'] = scan.returncode
if scan.returncode:
raise Refusal('normal_source_scanner_refused')
# --only preserves unrelated staged content while committing exact paths.
git(owner, 'commit', '--only', '-m',
'fix(projection): preserve exact frozen cluster historical bindings', '--', *sorted(names))
commit = git(owner, 'rev-parse', 'HEAD').stdout.decode().strip()
changed = set(git(owner, 'diff-tree', '--no-commit-id', '--name-only', '-r', commit).stdout.decode().splitlines())
if changed != names:
raise Refusal('committed_path_set_mismatch_stop')
if preserved_state(owner, names) != before:
raise Refusal('unrelated_state_changed_stop')
result.update(status='committed_not_pushed', source_revision=commit,
preserved_after=before)
if push:
# Explicit ref and bounded push. A nonzero or timeout never triggers retry.
try:
p = git(owner, 'push', 'origin', 'HEAD:refs/heads/main', check=False)
result['push_exit'] = p.returncode
except subprocess.TimeoutExpired:
result.update(status='push_timeout_remote_unverified')
return result
remote = git(owner, 'ls-remote', '--exit-code', 'origin', 'refs/heads/main', check=False)
actual = remote.stdout.split()[0].decode() if remote.returncode == 0 and remote.stdout else None
result['observed_remote_revision'] = actual
result['status'] = 'source_published_verified' if actual == commit else 'push_remote_unverified'
return result
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--execute-source-commit', action='store_true')
ap.add_argument('--push-source', action='store_true')
args = ap.parse_args()
if args.push_source and not args.execute_source_commit:
ap.error('--push-source requires --execute-source-commit')
expected = json.loads((BASE/'source-patch-manifest.json').read_text())['repos']['state-hub']
try:
result = commit_source(OWNER, expected, SCANNER, args.execute_source_commit, args.push_source)
except (Refusal, subprocess.TimeoutExpired) as exc:
result = {'schema': 'receiver-source-publication.v1', 'status': 'refused_or_unverified',
'reason': str(exc) if isinstance(exc, Refusal) else 'bounded_command_timeout',
'deployed': False, 'repo_manager_enabled': False}
print(json.dumps(result, indent=2))
return 0 if result['status'] in {'validated_not_executed','committed_not_pushed','source_published_verified'} else 1
if __name__ == '__main__':
raise SystemExit(main())

View file

@ -0,0 +1,23 @@
{
"schema": "receiver-ci-metadata.v1",
"source_revision": "51bc70d1ff8f25450a8650d8e50caf053093a02e",
"image_tag": "main-51bc70d",
"no_credentials": true,
"no_logs": true,
"ci_lookup_status": "ok",
"exact_revision_build_tasks": [
{
"id": 12624,
"name": "build-and-push",
"status": "running",
"head_sha": "51bc70d1ff8f25450a8650d8e50caf053093a02e",
"started_at": null,
"completed_at": null
}
],
"image_lookup_status": "public_access_refused",
"status": "artifact_unverified",
"limitations": [
"Successful task+registry digest does not prove floating dependency closure or delivered image file hashes."
]
}

View file

@ -0,0 +1,25 @@
{
"schema": "receiver-ci-metadata.v1",
"source_revision": "51bc70d1ff8f25450a8650d8e50caf053093a02e",
"image_tag": "main-51bc70d",
"no_credentials": true,
"no_logs": true,
"ci_lookup_status": "ok",
"exact_revision_build_tasks": [
{
"id": 12624,
"name": "build-and-push",
"status": "success",
"head_sha": "51bc70d1ff8f25450a8650d8e50caf053093a02e",
"started_at": null,
"completed_at": null
}
],
"image_lookup_status": "public_access_refused",
"status": "artifact_unverified",
"limitations": [
"Successful task+registry digest does not prove floating dependency closure or delivered image file hashes."
],
"registry_head_status_interpretation": "Anonymous OCI challenge; root verified manifest through ephemeral anonymous-token flow, not a private credential need.",
"root_verified_image_digest": "sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015"
}

View file

@ -0,0 +1,14 @@
{
"schema": "receiver-ci-metadata.v1",
"source_revision": "51bc70d1ff8f25450a8650d8e50caf053093a02e",
"image_tag": "main-51bc70d",
"no_credentials": true,
"no_logs": true,
"ci_lookup_status": "ok",
"exact_revision_build_tasks": [],
"image_lookup_status": "public_access_refused",
"status": "artifact_unverified",
"limitations": [
"Successful task+registry digest does not prove floating dependency closure or delivered image file hashes."
]
}

View file

@ -0,0 +1,972 @@
--- a/src/repo_manager/historical_sources.py
+++ b/src/repo_manager/historical_sources.py
@@ -70,6 +70,111 @@
"record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED",
"uuid": "1c53d3db-6633-5e2d-987f-10706d73be9e",
"parent_record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED"
+ },
+ {
+ "repo": "railiance-cluster",
+ "path": "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md",
+ "sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9",
+ "record_id": "RAIL-BS-WP-0002",
+ "uuid": "4c63dfc6-9eac-4e79-9f77-8f644ad7147d",
+ "parent_record_id": "RAIL-BS-WP-0002",
+ "tasks": [
+ {
+ "label": "T01",
+ "uuid": "3f042630-eab0-4c6a-9167-e2b28ff20e40",
+ "status": "done"
+ },
+ {
+ "label": "T02",
+ "uuid": "e8510646-46ed-4697-a345-f3d3009eea78",
+ "status": "done"
+ },
+ {
+ "label": "T03",
+ "uuid": "dab2c07f-8aa0-4635-8df6-857e87e93fc5",
+ "status": "done"
+ },
+ {
+ "label": "T04",
+ "uuid": "5c3d40e4-239b-488e-9519-6f7a38d2325f",
+ "status": "done"
+ },
+ {
+ "label": "T05",
+ "uuid": "7f9e0e58-a130-467a-a2d0-b3f2564e496f",
+ "status": "done"
+ }
+ ]
+ },
+ {
+ "repo": "railiance-cluster",
+ "path": "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md",
+ "sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a",
+ "record_id": "RAIL-BS-WP-0003",
+ "uuid": "7ee9ee22-1fae-4567-9194-8d70a9e0f45b",
+ "parent_record_id": "RAIL-BS-WP-0003",
+ "tasks": [
+ {
+ "label": "T01",
+ "uuid": "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc",
+ "status": "done"
+ },
+ {
+ "label": "T02",
+ "uuid": "ab166073-30a7-4702-a037-4091e8706e20",
+ "status": "done"
+ },
+ {
+ "label": "T03",
+ "uuid": "140da396-8e30-4f4d-b88c-c42c0cd46c01",
+ "status": "done"
+ },
+ {
+ "label": "T04",
+ "uuid": "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d",
+ "status": "done"
+ }
+ ]
+ },
+ {
+ "repo": "railiance-cluster",
+ "path": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md",
+ "sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3",
+ "record_id": "RAIL-BS-WP-0004",
+ "uuid": "7e8b0c20-51eb-40c9-9e3b-85dd380d7625",
+ "parent_record_id": "RAIL-BS-WP-0004",
+ "tasks": [
+ {
+ "label": "T01",
+ "uuid": "4526a842-ea31-4874-9231-92ab556cfe7b",
+ "status": "done"
+ },
+ {
+ "label": "T02",
+ "uuid": "a6313e06-1976-46a7-8e31-df4eb2eca880",
+ "status": "done"
+ },
+ {
+ "label": "T03",
+ "uuid": "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3",
+ "status": "done"
+ },
+ {
+ "label": "T04",
+ "uuid": "08233868-d522-4117-bc4e-6c0f52545665",
+ "status": "done"
+ },
+ {
+ "label": "T05",
+ "uuid": "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8",
+ "status": "done"
+ },
+ {
+ "label": "T06",
+ "uuid": "f8e4a094-c367-40eb-b895-da17bc144b07",
+ "status": "done"
+ }
+ ]
}
]''')
@@ -83,3 +188,24 @@
raise ValueError("Historical source binding requires owner review: " + entry["path"])
paths.add(entry["path"])
return paths
+
+
+def historical_source_occurrences(repo_root):
+ """Exact frozen closed task occurrences; never a general local-ID alias."""
+ from repo_manager.parse.workplan import parse_workplan_file
+ from uuid import UUID
+ paths=historical_source_paths(repo_root)
+ occurrences=set(paths)
+ for entry in BINDINGS:
+ if entry['repo']!=repo_root.name or 'tasks' not in entry:continue
+ parsed=parse_workplan_file(repo_root/entry['path'],repo_root=repo_root)
+ actual=[{'label':task.id,'uuid':task.state_hub_task_id,'status':task.status} for task in parsed.tasks]
+ if (parsed.id!=entry['record_id'] or parsed.state_hub_workstream_id!=entry['uuid']
+ or parsed.status not in {'finished','archived','completed'} or actual!=entry['tasks']
+ or any(task['status'] not in {'done','cancel'} for task in actual)):
+ raise ValueError('Historical task binding requires owner review: '+entry['path'])
+ UUID(entry['uuid'])
+ for number,task in enumerate(actual,1):
+ UUID(task['uuid'])
+ occurrences.add(entry['path']+'#task-block-'+str(number))
+ return occurrences
--- a/src/repo_manager/identifiers.py
+++ b/src/repo_manager/identifiers.py
@@ -854,8 +854,9 @@
# allocation candidates. Keep conformance reporting strict while allowing
# ordinary registration to preserve those bindings. Local aliases still
# require the explicit parent allowlist in the kind registry.
- from repo_manager.historical_sources import historical_source_paths
- historical_sources: set[str] = historical_source_paths(repo_root)
+ from repo_manager.historical_sources import historical_source_occurrences
+ frozen_sources = historical_source_occurrences(repo_root)
+ historical_sources: set[str] = set(frozen_sources)
for path in iter_workplan_files(repo_root):
parsed = parse_workplan_file(path, repo_root=repo_root)
if parsed.status not in {"finished", "archived", "completed"} or not parsed.id:
@@ -889,11 +890,18 @@
item for item in identity["invalid_identifiers"]
if item["source"] in historical_sources
]
- if invalid or identity["identity_collisions"]:
+ # Strict scans keep legacy nonconformance. Only collisions wholly contained
+ # in exact hash-bound closed task occurrences can be preserved by receiving;
+ # any unknown/live source or qualified/workplan collision remains fatal.
+ historical_collisions=[item for item in identity['identity_collisions']
+ if item['kind']=='task' and re.fullmatch(r'T[0-9]{2}',item['id'])
+ and all(source in frozen_sources for source in item['sources'])]
+ collisions=[item for item in identity['identity_collisions'] if item not in historical_collisions]
+ if invalid or collisions:
raise ValueError(
"work-record identities are not safe to derive: "
f"invalid={invalid!r}, "
- f"collisions={identity['identity_collisions']!r}"
+ f"collisions={collisions!r}"
)
rendered: dict[Path, str] = {}
@@ -966,6 +974,7 @@
"namespace": namespace,
"assignments": assignments,
"historical_invalid_identifiers": historical,
+ "historical_identity_collisions": historical_collisions,
"files_changed": sorted(str(path.relative_to(repo_root)) for path in rendered),
}
--- /dev/null
+++ b/tests/test_cluster_frozen_history.py
@@ -0,0 +1,52 @@
+import shutil
+from pathlib import Path
+import pytest
+from repo_manager.identifiers import ensure_missing_work_record_identifiers as ensure
+from repo_manager.record_identity import scan_record_identities
+
+FIXTURE=Path(__file__).parent/'fixtures/railiance-cluster'
+@pytest.fixture
+def repo(tmp_path):
+ root=tmp_path/'railiance-cluster';shutil.copytree(FIXTURE,root)
+ live=root/'workplans/RCLUSTER-WP-0018.md';live.write_text('---\nid: RCLUSTER-WP-0018\nstatus: ready\n---\n\n```task\nid: RCLUSTER-WP-0018-T01\nstatus: todo\n```\n')
+ return root,live
+
+def test_exact_three_histories_preserve_15_bindings_and_strict_findings(repo):
+ root,live=repo;archive={p:p.read_bytes() for p in (root/'workplans/archived').glob('*.md')};before=scan_record_identities(root)
+ assert len(before['invalid_identifiers'])==15 and len(before['identity_collisions'])==5
+ preview=ensure(root,execute=False);assert len(preview['assignments'])==2
+ assert preview['historical_invalid_identifiers']==before['invalid_identifiers']
+ assert preview['historical_identity_collisions']==before['identity_collisions']
+ result=ensure(root,execute=True);assert len(result['assignments'])==2
+ assert all(p.read_bytes()==raw for p,raw in archive.items())
+ assert not scan_record_identities(root)['ok']
+ assert ensure(root,execute=False)['assignments']==[]
+
+@pytest.mark.parametrize('change',['bytes','missing','symlink','live','nonterminal','uuid'])
+def test_unknown_history_change_refuses_before_any_allocation(repo,change):
+ root,live=repo;p=next((root/'workplans/archived').glob('*.md'));before=live.read_bytes()
+ raw=p.read_bytes()
+ if change=='missing':p.unlink()
+ elif change=='symlink':p.unlink();target=root/'saved';target.write_bytes(raw);p.symlink_to(target)
+ elif change=='live':p.write_bytes(raw.replace(b'status: finished',b'status: active',1))
+ elif change=='nonterminal':p.write_bytes(raw.replace(b'status: done',b'status: todo',1))
+ elif change=='uuid':p.write_bytes(raw.replace(b'state_hub_task_id:',b'old_task_id:',1))
+ else:p.write_bytes(raw+b'\nunreviewed change\n')
+ with pytest.raises(ValueError):ensure(root)
+ assert live.read_bytes()==before
+
+def test_any_unknown_or_live_source_in_collision_still_refuses(repo):
+ root,live=repo;before=live.read_bytes();other=root/'workplans/unknown.md'
+ other.write_text('---\nid: RCLUSTER-WP-0099\nstatus: active\n---\n\n```task\nid: T01\nstatus: todo\nstate_hub_task_id: 11111111-1111-4111-8111-111111111111\n```\n')
+ with pytest.raises(ValueError,match='identities are not safe'):ensure(root)
+ assert live.read_bytes()==before
+
+def test_other_repo_does_not_inherit_exact_frozen_exception(repo):
+ root,live=repo;other=root.with_name('other');root.rename(other)
+ with pytest.raises(ValueError,match='identities are not safe'):ensure(other)
+
+def test_qualified_collision_stays_global(repo):
+ root,live=repo;before=live.read_bytes()
+ (root/'workplans/other.md').write_text('---\nid: RCLUSTER-WP-0019\nstatus: active\n---\n\n```task\nid: RCLUSTER-WP-0018-T01\nstatus: done\nstate_hub_task_id: 11111111-1111-4111-8111-111111111111\n```\n')
+ with pytest.raises(ValueError,match='collisions='):ensure(root)
+ assert live.read_bytes()==before
--- a/workplans/RMGR-WP-0019-historical-source-admission.md
+++ b/workplans/RMGR-WP-0019-historical-source-admission.md
@@ -4,10 +4,10 @@
title: "Admit exact historical source views after receiving preservation support"
domain: infotech
repo: repo-manager
-status: finished
+status: active
owner: codex
created: "2026-10-04"
-updated: "2026-10-06"
+updated: "2026-10-17"
flavor: implementation
depends_on: []
state_hub_workstream_id: "efe0ecb2-2f3f-5f49-baee-b90ad1d353ab"
@@ -146,3 +146,62 @@
primary/railiance01, applied, 0 created / 45 updated (ACTIVITY-WP only) / 0 retired
/ 0 refused, 0 task changes, historical UUIDs preserved. Evidence:
state-hub docs/evidence/2026-10-06-activity-stale-retirement-rollout.json.
+
+
+## Cluster exact historical source successor, 2026-10-17
+
+This scoped extension preserves the completed Activity/Custodian work above.
+The cluster publisher now refuses three closed RAIL-BS source files containing
+15 existing UUID-bound bare task labels. Preserve all three source bytes, IDs,
+and UUIDs. No new alias or general syntax exemption is introduced. Evidence and
+exact receiver-first proposal: cluster-historical-publication-r1 in the Custodian
+NetKingdom change packet. The existing SHR-WP-0001 project driver owns the
+Repo Manager / State Hub cross-owner wait; NK-WP-0049-T35 benefits from the
+blocked cluster publication. No worker or human gate is woken here.
+
+## Prepare exact cluster registrar successor
+
+```task
+id: RMGR-WP-0019-T04
+status: done
+priority: high
+```
+
+Prepared full-file SHA256 bindings for three archives and 15 exact task
+occurrences; strict scans still return 15 invalid labels and five collisions.
+Unknown/live sources, changed/missing/symlinked histories, UUID drift and
+qualified/global collisions refuse before allocation. Candidate tests pass 47;
+canonical application remains a separate reviewed step. The read-only current
+cluster preview proposes seven ordinary UUID bindings in two files and changes
+no bytes. One pre-existing ADHOC field is included explicitly: task
+RCLUSTER-WP-ADHOC-2026-07-27-T01 in workplans/ADHOC-2026-07-27.md receives
+3e3f39d3-5070-5ba3-97db-19f5f361f264 through managed allocation only.
+
+## Admit cluster registrar successor only after receiving verification
+
+```task
+id: RMGR-WP-0019-T05
+status: wait
+priority: high
+depends_on: [STATE-WP-0095-T06]
+```
+
+Retain the current guard until exact reviewed receiving source has been delivered
+and verified through State Hub's normal owner deployment. Match both manifests,
+then integrate reviewed registrar source and verify ordinary allocator preview.
+No direct API patch, history rename or bypass substitutes for this gate.
+
+## Run ordinary cluster owner publication and verify preservation
+
+```task
+id: RMGR-WP-0019-T06
+status: wait
+priority: high
+depends_on: [RMGR-WP-0019-T05]
+```
+
+Only then run normal owner consistency/publication with the reviewed expected
+file set: RCLUSTER-WP-0018 and the one-field ADHOC managed backfill, plus ordinary
+generated indexes. Preserve all three archived files and all 18 existing
+parent/task rows. Return actual receiving/publication receipts to STATE-WP-0095-T07.
+Do not infer TLS recovery, credential custody or native acceptance from sync.
--- /dev/null
+++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md
@@ -0,0 +1,177 @@
+---
+id: RAIL-BS-WP-0002
+type: workplan
+title: "k3s and Kubernetes Platform Baseline"
+domain: financials
+repo: railiance-cluster
+status: finished
+owner: railiance
+topic_slug: railiance
+repo_goal_id: "70ab2379-fb9d-4fec-a09d-b2a717e4ace8"
+state_hub_workstream_id: "4c63dfc6-9eac-4e79-9f77-8f644ad7147d"
+created: "2026-03-09"
+updated: "2026-03-10"
+completed: "2026-03-10"
+---
+
+> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
+
+# k3s and Kubernetes Platform Baseline
+
+## Goal
+
+Install k3s, Helm, and the baseline Kubernetes services on the converged
+HostEurope node. This workplan picks up exactly where `railiance-hosts`
+leaves off: a hardened, verified OS node that is ready for Kubernetes.
+
+## Pre-condition
+
+`railiance-infra` converge + Goss verify must pass before any task here
+is executed:
+
+```bash
+# In railiance-infra/
+make converge
+make verify # must exit 0
+```
+
+## Boundary
+
+This repo owns everything from k3s upward. It must not re-configure items
+defined in `railiance-infra/spec/server-baseline.yaml`. See ADR-003:
+`railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`.
+
+**Out of scope here:** platform services (PostgreSQL, storage, identity)
+→ `railiance-platform`. Application deployments (Gitea, coulomb services)
+→ `railiance-apps`.
+
+---
+
+## Tasks
+
+### T01 — Ansible playbook: install k3s (server mode)
+
+```task
+id: T01
+status: done
+priority: high
+state_hub_task_id: "3f042630-eab0-4c6a-9167-e2b28ff20e40"
+completed: "2026-03-10"
+```
+
+Harden `ansible/bootstrap.yml` to a production-ready k3s install:
+
+- Use the official k3s install script pinned to a specific version
+ (`INSTALL_K3S_VERSION=v1.35.1+k3s1`)
+- `INSTALL_K3S_EXEC="server --cluster-init --write-kubeconfig-mode=644"`
+ (cluster-init enables embedded etcd for future HA expansion)
+- Wait for node `Ready` before proceeding:
+ ```bash
+ k3s kubectl wait node --all --for=condition=Ready --timeout=120s
+ ```
+- Fetch kubeconfig to the control node as `~/.kube/config-hosteurope`
+
+**Done when:** `k3s kubectl get nodes` returns `Ready` from both the server
+and the control node (via kubeconfig).
+
+---
+
+### T02 — Helm installation
+
+```task
+id: T02
+status: done
+priority: high
+state_hub_task_id: "e8510646-46ed-4697-a345-f3d3009eea78"
+completed: "2026-03-10"
+```
+
+Add a task (or a role `roles/helm/`) that:
+
+1. Downloads the Helm binary (pinned version) to `/usr/local/bin/helm`
+2. Verifies the checksum
+3. Confirms `helm version` succeeds
+
+**Done when:** `helm version` succeeds on the HostEurope node.
+
+---
+
+### T03 — Smoke test: k3s + Helm
+
+```task
+id: T03
+status: done
+priority: high
+state_hub_task_id: "dab2c07f-8aa0-4635-8df6-857e87e93fc5"
+completed: "2026-03-10"
+```
+
+Extend `tests/smoke_kube.sh` to assert:
+
+- `k3s kubectl get nodes` → node in Ready state
+- `helm version` exits 0
+- CoreDNS pod running in `kube-system`
+- Traefik ingress controller pod running (default in k3s)
+
+Run via:
+```bash
+ansible-playbook -i ansible/hosts.ini ansible/smoke.yml
+```
+or directly over SSH if the kubeconfig is available locally.
+
+**Done when:** all assertions pass and the script exits 0.
+
+---
+
+### T04 — Commit kubeconfig management notes
+
+```task
+id: T04
+status: done
+priority: medium
+state_hub_task_id: "5c3d40e4-239b-488e-9519-6f7a38d2325f"
+completed: "2026-03-10"
+```
+
+Document in `docs/kubeconfig.md`:
+
+- Where the kubeconfig is fetched to (`~/.kube/config-hosteurope`)
+- How to merge it into `~/.kube/config`
+- How to switch context: `kubectl config use-context default`
+- Security note: kubeconfig is gitignored (contains cluster CA + client cert)
+
+**Done when:** doc written and committed.
+
+---
+
+### T05 — Add `make k3s-install` and `make smoke` targets
+
+```task
+id: T05
+status: done
+priority: medium
+state_hub_task_id: "7f9e0e58-a130-467a-a2d0-b3f2564e496f"
+completed: "2026-03-10"
+```
+
+Add to Makefile (create one if none exists):
+
+```makefile
+k3s-install: ## Install k3s and Helm on all inventory hosts
+ ansible-playbook -i ansible/hosts.ini ansible/bootstrap.yml
+
+smoke: ## Run Kubernetes smoke tests
+ bash tests/smoke_kube.sh
+```
+
+**Done when:** both targets work and are listed in `make help`.
+
+---
+
+## References
+
+- Repo goal: `70ab2379-fb9d-4fec-a09d-b2a717e4ace8` (Install k3s and Kubernetes Baseline)
+- Domain goal: `6f96c712-60e6-4ea9-ab06-168878eafbce` (Three-Phoenix Secure Kubernetes Infrastructure)
+- Pre-condition: railiance-infra WP-0001 (Secure Single-Server Bootstrap) — completed 2026-03-09
+- Boundary ADR: `railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`
+- k3s releases: https://github.com/k3s-io/k3s/releases
--- /dev/null
+++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md
@@ -0,0 +1,196 @@
+---
+id: RAIL-BS-WP-0003
+type: workplan
+title: "pgpool CrashLoopBackOff on PostgreSQL HA failover — missing secret key"
+domain: financials
+repo: railiance-cluster
+status: finished
+owner: tegwick
+created: "2026-03-10"
+updated: "2026-03-10"
+state_hub_workstream_id: "7ee9ee22-1fae-4567-9194-8d70a9e0f45b"
+---
+
+> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
+
+# Bug Report: pgpool CrashLoopBackOff on PostgreSQL HA failover
+
+## Summary
+
+On 2026-03-10 a PostgreSQL HA failover caused all three postgresql pods to
+restart. pgpool — the connection pooler between Gitea and PostgreSQL — then
+entered CrashLoopBackOff and produced no logs. As a result Gitea's login
+and all write operations hung indefinitely. The root page was still served
+(from Valkey cache) which masked the failure.
+
+The fix was to patch a missing key in a Kubernetes secret. The root cause is
+that the `gitea-12.2.0` Helm chart (postgresql-ha subchart v16.2.2) does not
+populate the `pgpool-password` key in the `gitea-postgresql-ha-postgresql`
+secret, even though the pgpool pod requires it at startup.
+
+---
+
+## Timeline
+
+| Time (UTC) | Event |
+|---|---|
+| ~09:45 | postgresql-0, postgresql-2 pods restarted (repmgr failover) |
+| ~09:45 | pgpool pod restarted and entered CrashLoopBackOff |
+| ~11:00 | User noticed Gitea login hanging; home page still loading |
+| ~13:00 | Root cause identified: missing `pgpool-password` secret key |
+| ~13:10 | Secret patched; pgpool pod deleted and restarted cleanly |
+| ~13:15 | Gitea fully operational |
+
+---
+
+## Root Cause
+
+The Bitnami `pgpool` container startup script reads the file
+`/opt/bitnami/pgpool/secrets/pgpool-password`, which is mounted from the
+`gitea-postgresql-ha-postgresql` Kubernetes Secret via a `subPath` volume
+mount. That secret key was never created by the Helm chart, so the file did
+not exist. The container exited immediately with no logs.
+
+The pod had been running for 20 days without a restart, so this gap was
+never discovered during initial deployment.
+
+---
+
+## Evidence
+
+```bash
+# Secret was missing the pgpool-password key
+sudo k3s kubectl get secret -n default gitea-postgresql-ha-postgresql -o yaml
+# data: keys were password, postgres-password, repmgr-password only
+# pgpool-password was absent
+
+# pgpool pod describe showed 824 back-off restarts over 173 minutes
+# No logs in either current or --previous output
+sudo k3s kubectl logs -n default <pgpool-pod> --previous
+# (empty)
+
+# Gitea process had zero TCP connections to PostgreSQL port 5432
+# but many connections to Valkey port 6379
+cat /proc/<gitea-pid>/net/tcp | grep 1538 # 1538 = 5432 hex — no results
+```
+
+---
+
+## Immediate Fix Applied
+
+```bash
+# Add the missing key (value = sr-check-password = changeme4 = base64: Y2hhbmdlbWU0)
+sudo k3s kubectl patch secret -n default gitea-postgresql-ha-postgresql \
+ --type='json' \
+ -p='[{"op":"add","path":"/data/pgpool-password","value":"Y2hhbmdlbWU0"}]'
+
+# Restart pgpool
+sudo k3s kubectl delete pod -n default <pgpool-pod-name>
+```
+
+---
+
+## Risk: Fix Will Be Lost on helm upgrade
+
+The patched secret is managed by Helm (annotation:
+`meta.helm.sh/release-name: gitea`). A `helm upgrade` will regenerate the
+secret from the chart template, which does not include `pgpool-password`,
+and the bug will recur.
+
+---
+
+## Tasks
+
+### T01 — Add pgpool-password to Helm values
+
+```task
+id: T01
+status: done
+priority: high
+state_hub_task_id: "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc"
+```
+
+Create or update `helm/gitea-values.yaml` (or equivalent) to permanently
+include the pgpool-password so it survives `helm upgrade`:
+
+```yaml
+postgresql-ha:
+ postgresql:
+ pgpoolPassword: <value matching sr-check-password>
+```
+
+**Done when:** `helm upgrade gitea` completes and pgpool starts cleanly
+without manual secret patching.
+
+---
+
+### T02 — Add pgpool health check to smoke test
+
+```task
+id: T02
+status: done
+priority: high
+state_hub_task_id: "ab166073-30a7-4702-a037-4091e8706e20"
+```
+
+Extend `tests/smoke_kube.sh` to assert:
+
+```bash
+# All postgresql-ha pods Running
+kubectl get pods -n default | grep gitea-postgresql-ha | grep -v Running && exit 1
+
+# pgpool specifically not in CrashLoopBackOff
+kubectl get pod -n default -l app.kubernetes.io/component=pgpool \
+ -o jsonpath='{.items[0].status.containerStatuses[0].state}' | grep -v crash
+```
+
+**Done when:** the smoke test catches a pgpool failure within 5 minutes.
+
+---
+
+### T03 — Add HA failover test
+
+```task
+id: T03
+status: done
+priority: high
+state_hub_task_id: "140da396-8e30-4f4d-b88c-c42c0cd46c01"
+```
+
+Create `tests/test_ha_failover.sh` that:
+
+1. Records Gitea login response time (baseline)
+2. Kills the primary PostgreSQL pod: `kubectl delete pod gitea-postgresql-ha-postgresql-0 -n default`
+3. Waits for repmgr to promote a replica (max 60s)
+4. Asserts Gitea login POST still succeeds within 10s
+5. Asserts pgpool pod is Running (not CrashLoopBackOff)
+6. Asserts all postgresql pods return to Running
+
+This test must pass before any PostgreSQL HA deployment is considered done.
+
+**Done when:** script exits 0 against a live cluster.
+
+---
+
+### T04 — Document the incident in docs/
+
+```task
+id: T04
+status: done
+priority: medium
+state_hub_task_id: "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d"
+```
+
+Add `docs/incidents/2026-03-10-pgpool-missing-secret.md` with the full
+timeline, root cause, and fix, so future operators understand what happened
+and how to recover.
+
+**Done when:** doc committed and linked from `docs/README.md`.
+
+---
+
+## References
+
+- Bitnami postgresql-ha chart v16.2.2
+- Gitea Helm chart v12.2.0
+- Related decision: D3 (HA testing policy) in `DECISIONS.md`
--- /dev/null
+++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md
@@ -0,0 +1,275 @@
+---
+id: RAIL-BS-WP-0004
+type: workplan
+title: "Integrated Backup — S2 Kubernetes Runtime Layer"
+domain: financials
+repo: railiance-cluster
+status: finished
+owner: tegwick
+topic_slug: railiance
+state_hub_workstream_id: "7e8b0c20-51eb-40c9-9e3b-85dd380d7625"
+created: "2026-02-25"
+updated: "2026-03-26"
+---
+
+> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
+
+# Integrated Backup — S2 Kubernetes Runtime Layer
+
+## Goal
+
+Implement the Q3 (Operability & Resilience) integrated backup for
+railiance-cluster (S2). Backs up what S2 owns — the Kubernetes runtime state —
+encrypted with age, written to a local directory on the server. No external
+dependencies required.
+
+## Architecture (Decision D4)
+
+Each railiance repo implements its own backup for what it owns. No central
+backup service. See `DECISIONS.md` D4 for full rationale.
+
+**Standard interface every railiance repo must provide:**
+
+```bash
+make backup # encrypt + write to /opt/backup/railiance/<layer>/
+make restore # restore from most recent local backup
+```
+
+Encryption: age, same key pair as SOPS secrets (`.sops.yaml` public key).
+Output: `/opt/backup/railiance/cluster/` on the server.
+
+## What S2 (railiance-cluster) owns and must back up
+
+| Asset | Why it matters |
+|---|---|
+| k3s etcd snapshots | Full cluster state — all workloads, configs, secrets |
+| Helm release values | Runtime values not in git (any manually applied overrides) |
+| kubeconfig | Admin access to the cluster |
+
+**Not S2's responsibility:**
+- Custodian State Hub DB → the-custodian owns this
+- Operator workstation config (`.claude/`, `.gitconfig`) → operator's own concern
+- Application data (Gitea repos, uploads) → S5 (railiance-apps) owns this
+- PostgreSQL data volumes → S3 (railiance-platform) owns this
+
+## Encryption
+
+Reuse the age public key from `.sops.yaml`:
+
+```bash
+AGE_PUBLIC_KEY=$(grep 'age:' .sops.yaml | awk '{print $2}')
+tar -czf - <assets> | age -r "${AGE_PUBLIC_KEY}" -o backup.tar.gz.age
+```
+
+Decryption requires the private key at `~/.config/sops/age/keys.txt`
+(same key used for `sops -d`). No additional key management needed.
+
+## Extension Point EP-RAIL-005
+
+Once all five OAS layers implement this interface, the custodian can
+orchestrate a full-stack backup with:
+
+```bash
+for repo in railiance-infra railiance-cluster railiance-platform \
+ railiance-enablement railiance-apps; do
+ make -C ~/$repo backup
+done
+```
+
+No special protocol needed — just the standard interface.
+
+---
+
+## Tasks
+
+### T01 — Define backup directory and encryption wrapper
+
+```task
+id: T01
+status: done
+priority: high
+state_hub_task_id: "4526a842-ea31-4874-9231-92ab556cfe7b"
+```
+
+Create `tools/cmd/railiance-backup-s2` (replacing the old `railiance-backup`):
+
+- Backup dir: `/opt/backup/railiance/cluster/` (create with `mkdir -p`)
+- Encrypt each artifact with age using public key from `.sops.yaml`
+- Write timestamp-named files: `etcd-<ts>.snap.age`, `helm-values-<ts>.tar.gz.age`, `kubeconfig-<ts>.yaml.age`
+- Keep last 7 of each type
+- Write `.last-backup` stamp
+- Exit 0 on success, non-zero on any failure
+- No network required
+
+Also remove the old `tools/cmd/railiance-backup` (backed up Docker-based
+custodian DB — wrong scope, not applicable to this server).
+
+**Done when:** `make backup` runs on COULOMBCORE without error and files
+appear in `/opt/backup/railiance/cluster/`.
+
+---
+
+### T02 — Back up k3s state (SQLite hot backup)
+
+```task
+id: T02
+status: done
+priority: high
+state_hub_task_id: "a6313e06-1976-46a7-8e31-df4eb2eca880"
+```
+
+k3s has built-in etcd snapshot support:
+
+```bash
+sudo k3s etcd-snapshot save --name railiance-$(date -u +%Y%m%dT%H%M%SZ)
+# Default location: /var/lib/rancher/k3s/server/db/snapshots/
+```
+
+Add to the backup script: take a fresh snapshot, encrypt with age,
+copy to `/opt/backup/railiance/cluster/`.
+
+> **Note — verify etcd is in use before implementing:**
+> `k3s etcd-snapshot` only works if k3s was started with `--cluster-init`.
+> Without it, k3s uses SQLite and this command will fail.
+> Verify first: `sudo k3s etcd-snapshot ls 2>&1`
+
+> **Note — sudo required:** etcd snapshot requires root. See T06 for how
+> this is resolved (backup runs under root's crontab).
+
+**Done when:** backup includes a current etcd snapshot.
+
+---
+
+### T03 — Back up Helm release values
+
+```task
+id: T03
+status: done
+priority: medium
+state_hub_task_id: "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3"
+```
+
+Capture current runtime Helm values for all releases:
+
+```bash
+KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm list -A -o json | \
+ jq -r '.[].name + " " + .namespace' | \
+ while read name ns; do
+ KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm get values "$name" -n "$ns" -o yaml
+ done
+```
+
+Tar and age-encrypt into `helm-values-<ts>.tar.gz.age`.
+
+> **Note — kubeconfig permissions:** `/etc/rancher/k3s/k3s.yaml` is root-readable
+> only by default. The backup script must either run as root (see T06) or k3s
+> must be configured with `--write-kubeconfig-mode=644`. Running as root
+> (via root crontab) is the chosen approach — no config change needed.
+
+**Done when:** backup includes a snapshot of all Helm release values.
+
+---
+
+### T04 — Back up kubeconfig
+
+```task
+id: T04
+status: done
+priority: medium
+state_hub_task_id: "08233868-d522-4117-bc4e-6c0f52545665"
+```
+
+Age-encrypt `~/.kube/config-hosteurope` (or `/etc/rancher/k3s/k3s.yaml`)
+into `kubeconfig-<ts>.yaml.age` in the backup directory.
+
+**Done when:** backup includes the encrypted kubeconfig.
+
+---
+
+### T05 — make restore target
+
+```task
+id: T05
+status: done
+priority: medium
+state_hub_task_id: "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8"
+```
+
+Add `tools/cmd/railiance-restore-s2` that decrypts and lists available
+backups, with guided restore for the etcd snapshot case.
+
+Restore of etcd from snapshot:
+```bash
+sudo k3s server --cluster-reset \
+ --cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/<name>
+```
+
+**Done when:** `make restore` prints available backups and a restore guide.
+
+---
+
+### T06 — Install cron job and run restore drill
+
+```task
+id: T06
+status: done
+priority: medium
+state_hub_task_id: "f8e4a094-c367-40eb-b895-da17bc144b07"
+```
+
+#### Solving the sudo problem
+
+The backup script needs root for two reasons:
+- `k3s etcd-snapshot save` requires root
+- `/etc/rancher/k3s/k3s.yaml` (kubeconfig) is root-readable only
+
+**Solution: run the cron under root's crontab.**
+
+This is the correct pattern for system-level backup jobs. It avoids a
+proliferating sudoers whitelist (one entry per command, brittle to maintain)
+and matches how tools like `rsnapshot`, `bacula`, and `borgbackup` work in
+production. The backup writes to `/opt/backup/` which is root-owned anyway.
+
+Install the cron as root:
+
+```bash
+sudo crontab -e
+# Add:
+0 2 * * * make -C /home/tegwick/railiance-cluster backup >> /opt/backup/railiance/cluster/backup.log 2>&1
+```
+
+Note: use the absolute path to the repo — `~` does not expand reliably in
+root's crontab unless HOME is set.
+
+Verify it is installed:
+```bash
+sudo crontab -l | grep railiance
+```
+
+#### Restore drill
+
+Once T01–T04 are done, run a decrypt-and-verify drill:
+
+```bash
+# Decrypt the etcd snapshot and verify it is a valid snapshot file
+sudo age -d -i ~/.config/sops/age/keys.txt \
+ /opt/backup/railiance/cluster/etcd-$(ls /opt/backup/railiance/cluster/etcd-*.snap.age | sort -r | head -1 | xargs basename | sed 's/etcd-//;s/.snap.age//').snap.age \
+ | file -
+
+# Record the drill
+echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) restore drill OK" \
+ >> /opt/backup/railiance/cluster/restore-drill.log
+```
+
+**Done when:** cron installed under root, drill completes without error,
+log entry written.
+
+---
+
+## References
+
+- Decision D4: Integrated backup per capability (`DECISIONS.md`)
+- Decision D2: Nextcloud as optional offsite extension (still valid, not a requirement)
+- OAS Q3: Operability & Resilience
+- Extension point EP-RAIL-005: Custodian full-stack backup orchestration
+- k3s etcd snapshots: https://docs.k3s.io/datastore/backup-restore

View file

@ -0,0 +1,56 @@
{
"schema": "cluster-frozen-successor-review.v1",
"stable_proposal_id": "STATE-WP-0095-T06-cluster-frozen-receiving-r1",
"revision": 2,
"status": "prepared_for_independent_source_review_not_deployment",
"files": [
{
"path": "REVIEW.md",
"sha256": "c8f13ed5d67d4743ea8e81bf720367022c820a060d7b225e84338eab2907493f"
},
{
"path": "source-patch-manifest.json",
"sha256": "f75391603c703b71b76eaf3432adef5022e73d61e4d3b22da4820f583dd435b4"
},
{
"path": "state-hub.patch",
"sha256": "5cb52c26c902ed1cd95156f25c2e4d9107e6b2a808212960aa766074314b819d"
},
{
"path": "repo-manager.patch",
"sha256": "232552c3d601d185984be5a8e5bc9937bc9ddda7327923b5f0440f6deb2c87fc"
},
{
"path": "publish_receiver_source.py",
"sha256": "98c24e9df3384d39e188019a249265a39896221d898082bcab5bbef64ba40ec8"
},
{
"path": "lookup_ci_metadata.py",
"sha256": "05e388ad81334316653770430b153b8a040427159e7e33b730edc4a0ecf3a520"
},
{
"path": "preflight_receiver.py",
"sha256": "1140ad022eba074cffd8bd62ab71643389a7f6992d2d59765ea0fd85bb812b92"
},
{
"path": "test_publication_helpers.py",
"sha256": "160f9efe77e6533ca154b1876ab1221fe60029d8a59b68e7d18010731c71a156"
},
{
"path": "bounded-preflight-commands.md",
"sha256": "f4cfbeea5758b5e48234f41eb2960adf55cd0dbb78cd4caecf64d28bc1f44cc1"
},
{
"path": "deployed-to-current-source-closure.json",
"sha256": "a8dd674b9740b2d72425acfadb10b00abd7692ccf6b11cf81dbdda4d246c6d21"
},
{
"path": "managed-statehub-uuid-assignment.json",
"sha256": "9310f0889ba801414a5e9274ef3ff64c0b083749cce927231bd80c712428bdf6"
},
{
"path": "current-base-public-ci-metadata.json",
"sha256": "4a397725f7d4a861463ce8c2f15d0aecab9091c5376b9b882a9bb7d5b88598fc"
}
]
}

View file

@ -0,0 +1,24 @@
{
"schema": "receiver-serverdryrun-phase-diagnosis.v1",
"phase": "platform_db_secret_template",
"exit_code": 1,
"raw_diagnostics_retained": false,
"fixed_indicators": {
"Secret": true,
"state-hub-db-credentials": true,
"type": false,
"immutable": false,
"Invalid value": false,
"data": false,
"metadata.resourceVersion": false,
"required": true,
"strict": false,
"admission": true,
"annotation": true,
"Too long": false,
"forbidden": true,
"patch": true,
"invalid": true
},
"safe_field_names": []
}

View file

@ -0,0 +1,8 @@
{
"schema": "receiver-make-serverdryrun-diagnosis.v1",
"command_scope": "existing_owner_make_server_dryrun",
"exit_code": 2,
"raw_diagnostics_retained": false,
"classification": "invalid_resource",
"safe_details": []
}

View file

@ -0,0 +1,47 @@
{
"schema": "receiver-nonsecret-serverdryrun-remainder.v1",
"image_tag": "main-51bc70d",
"production_mutations": false,
"raw_diagnostics_retained": false,
"secret_template_phases": "refused_by_existing_policy_and_excluded_from_remainder",
"phases": [
{
"phase": "render_nonsecret_app_chart",
"status": "passed",
"exit_code": 0,
"classification": null
},
{
"phase": "platform_db_cluster",
"status": "passed",
"exit_code": 0,
"classification": null
},
{
"phase": "platform_db_networkpolicies",
"status": "passed",
"exit_code": 0,
"classification": null
},
{
"phase": "app_namespace",
"status": "passed",
"exit_code": 0,
"classification": null
},
{
"phase": "nonsecret_app_chart_admission",
"status": "passed",
"exit_code": 0,
"classification": null
}
],
"rendered_public_kinds": [
"ConfigMap",
"Deployment",
"Job",
"Service",
"ServiceAccount"
],
"status": "nonsecret_remainder_passed"
}

View file

@ -0,0 +1,23 @@
{
"phase": "platform_db_secret_template",
"exit_code": 1,
"classification": "secret_last_applied_annotation_admission_refusal",
"fixed_indicators": {
"last-applied-configuration": true,
"kubectl.kubernetes.io": true,
"validatingadmissionpolicy": true,
"admission webhook": false,
"secret-last-applied": true,
"secret-annotation": false,
"forbidden:": false,
"denied the request": false,
"does not have": false,
"not a valid": false,
"cannot be": false,
"must not": false,
"must be": false,
"required": true,
"plaintext": false,
"plain-text": false
}
}

View file

@ -0,0 +1,99 @@
{
"schema": "cluster-historical-source-patches.v1",
"status": "isolated_review_candidate_not_applied",
"binding_manifests_identical": true,
"repos": {
"repo-manager": {
"base_revision": "9ebde0a9bd60d5c82910494fbe6bd1b891455cee",
"files": [
{
"path": "src/repo_manager/historical_sources.py",
"action": "modify",
"base_sha256": "a465aa595867c5e80918168158f8a27bf429d469b2e5249eca98b4e5ef84ce03",
"candidate_sha256": "838a0422b81e1215e3ec148c3104dd603d9dfc476610bb5617d98ccf5d3be218"
},
{
"path": "src/repo_manager/identifiers.py",
"action": "modify",
"base_sha256": "68dbbd3f137f326a49d20ba3f5855d6b00b4d62eb01dc904b51a08d77552d2bf",
"candidate_sha256": "4e35fcd009c20822574fd33171176bd14a1ff2dd5af8b0bb2820954974593a2a"
},
{
"path": "tests/test_cluster_frozen_history.py",
"action": "add",
"base_sha256": null,
"candidate_sha256": "0490ef3eb474f653e4a6a52b711999876bef061c07de3e7a6d327a8ba54acaf3"
},
{
"path": "workplans/RMGR-WP-0019-historical-source-admission.md",
"action": "modify",
"base_sha256": "638af4000aa442a57df8eddef2d8b5aa5dffb9dbc9ccf547d9e04cf25012ffe4",
"candidate_sha256": "6378d96b70259ae81435b6ce8f97864698bdc5b6679cabbe9dc61a1fc1d09699"
},
{
"path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md",
"action": "add",
"base_sha256": null,
"candidate_sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9"
},
{
"path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md",
"action": "add",
"base_sha256": null,
"candidate_sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a"
},
{
"path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md",
"action": "add",
"base_sha256": null,
"candidate_sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3"
}
],
"patch_sha256": "232552c3d601d185984be5a8e5bc9937bc9ddda7327923b5f0440f6deb2c87fc"
},
"state-hub": {
"base_revision": "4ab66e363ed00a67ad1ea0434854bc4284c7a26e",
"files": [
{
"path": "api/services/historical_projection.py",
"action": "modify",
"base_sha256": "4d7cd127462bedaaf1c0915c6a11f58b25d28ae6f000eb89c95d903bf4a0e94e",
"candidate_sha256": "e3c4bf965764b32a6df5d61c71272ee22569df9a9ddae2a3fc3df97ec554e0b2"
},
{
"path": "tests/test_cluster_historical_projection.py",
"action": "add",
"base_sha256": null,
"candidate_sha256": "183f0574aa01f69269a06d162b29b5b95c3415f6fce0560bb92188c09be26507"
},
{
"path": "workplans/STATE-WP-0095-historical-source-receiving.md",
"action": "modify",
"base_sha256": "9be4b0661f0cc817d55dcf02f959339cb6c13f56956b111a6ecd87d1bec79405",
"candidate_sha256": "aeb307597dba0d92594d55242bd69ed6a391e7fdd58bb16c839ec8cbce44b349"
},
{
"path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md",
"action": "add",
"base_sha256": null,
"candidate_sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9"
},
{
"path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md",
"action": "add",
"base_sha256": null,
"candidate_sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a"
},
{
"path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md",
"action": "add",
"base_sha256": null,
"candidate_sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3"
}
],
"patch_sha256": "5cb52c26c902ed1cd95156f25c2e4d9107e6b2a808212960aa766074314b819d"
}
},
"revision": "r2",
"supersedes": "r1 manifest and REVIEW rollout assumptions only; originals preserved"
}