diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/idempotency.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/idempotency.py new file mode 100644 index 00000000..26d9c2c4 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/idempotency.py @@ -0,0 +1,45 @@ +"""Simple file-backed idempotency for dual-run commands.""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any + + +def _store_path() -> Path: + raw = os.environ.get("RM_IDEMPOTENCY_PATH", "~/.repo-manager/idempotency.json") + return Path(raw).expanduser() + + +def _load() -> dict[str, Any]: + path = _store_path() + if not path.is_file(): + return {} + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + + +def _save(data: dict[str, Any]) -> None: + path = _store_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + + +def payload_hash(payload: dict[str, Any]) -> str: + blob = json.dumps(payload, sort_keys=True, default=str) + return hashlib.sha256(blob.encode()).hexdigest() + + +def get(key: str) -> dict[str, Any] | None: + return _load().get(key) + + +def put(key: str, payload_hash_value: str, result: dict[str, Any]) -> None: + data = _load() + data[key] = {"payload_hash": payload_hash_value, "result": result} + _save(data) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/identifiers.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/identifiers.py new file mode 100644 index 00000000..7dbbdd08 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/identifiers.py @@ -0,0 +1,1185 @@ +"""Deterministic work-record identifiers and live-collision preflight.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import subprocess +import tempfile +import uuid +from collections import defaultdict +from pathlib import Path +from typing import Any + +import httpx +import yaml + +from repo_manager.cache import source_fingerprint +from repo_manager.gitops import head_sha +from repo_manager.parse.workplan import ( + _TASK_BLOCK_RE, + _parse_yaml_block, + iter_workplan_files, + parse_workplan_file, +) +from repo_manager.prefix_registry import iter_repo_roots +from repo_manager.record_identity import classify_record_id, scan_record_identities +from repo_manager.time import utc_now_text + +DERIVATION_NAMESPACE_UUID = uuid.UUID("a4058507-5c4a-5a00-ab06-fffa4fb46009") +DERIVATION_VERSION = "repo-manager.work-record-uuid.v1" +FLEET_NAMESPACE_SCHEMA = "repo-manager.fleet-namespace.v1" +DEFAULT_NAMESPACE_CONTRACT = Path(__file__).resolve().parents[2] / "config" / "fleet-namespace.yaml" +LIVE_WORKPLAN_STATUSES = frozenset({"proposed", "ready", "active", "blocked", "backlog"}) +_NAMESPACE_RE = re.compile(r"^[a-z0-9][a-z0-9.-]{0,62}$") + + +def load_fleet_namespace(path: Path = DEFAULT_NAMESPACE_CONTRACT) -> str: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if not isinstance(data, dict) or data.get("schema") != FLEET_NAMESPACE_SCHEMA: + raise ValueError(f"unsupported fleet namespace contract: {path}") + namespace = str(data.get("namespace") or "").strip() + if not _NAMESPACE_RE.fullmatch(namespace): + raise ValueError(f"invalid fleet namespace in contract: {path}") + return namespace + + +def derivation_name(namespace: str, identifier: str) -> str: + """Return the exact UTF-8 UUIDv5 name input defined by contract v1.""" + namespace = namespace.strip() + identifier = identifier.strip() + if not _NAMESPACE_RE.fullmatch(namespace): + raise ValueError("namespace must be lowercase DNS-label style") + canonical_kind = next( + ( + kind + for kind in ("workplan", "task") + if classify_record_id(kind, identifier) == "canonical" + ), + None, + ) + if canonical_kind is None: + raise ValueError("identifier must be a canonical workplan or task id") + return f"{namespace}\n{identifier}" + + +def derive_work_record_uuid(namespace: str, identifier: str) -> uuid.UUID: + return uuid.uuid5(DERIVATION_NAMESPACE_UUID, derivation_name(namespace, identifier)) + + +def _migration_mapping( + *, + namespace: str, + repo: Path, + path: str, + kind: str, + identifier: str, + current_uuid: str | None, +) -> dict[str, str | None]: + derived_uuid = str(derive_work_record_uuid(namespace, identifier)) + if not current_uuid: + action = "assign" + elif current_uuid == derived_uuid: + action = "unchanged" + else: + action = "replace" + return { + "repo": repo.name, + "path": path, + "kind": kind, + "record_id": identifier, + "current_uuid": current_uuid, + "derived_uuid": derived_uuid, + "action": action, + } + + +def scan_live_identifier_collisions(root: Path) -> dict[str, Any]: + """Report live workplan/task identifiers that cannot safely be derived.""" + by_id: dict[str, list[dict[str, str]]] = defaultdict(list) + repos = iter_repo_roots(root) + for repo in repos: + for path in iter_workplan_files(repo): + parsed = parse_workplan_file(path, repo_root=repo) + if parsed.status not in LIVE_WORKPLAN_STATUSES: + continue + if parsed.id: + by_id[parsed.id].append( + {"repo": repo.name, "path": parsed.path, "kind": "workplan", "status": parsed.status} + ) + for task in parsed.tasks: + if task.id and task.status not in {"done", "cancel"}: + by_id[task.id].append( + { + "repo": repo.name, + "path": parsed.path, + "kind": "task", + "status": task.status or "unknown", + } + ) + collisions = {identifier: entries for identifier, entries in sorted(by_id.items()) if len(entries) > 1} + return { + "ok": not collisions, + "root": str(root.resolve()), + "repos_scanned": [repo.name for repo in repos], + "live_identifiers": len(by_id), + "collisions": collisions, + "safe_to_derive": not collisions, + } + + +def plan_identifier_migration(root: Path, namespace: str) -> dict[str, Any]: + """Plan a live-record UUID migration without changing files or databases. + + The report is the durable old-to-derived mapping required by RMGR-WP-0005. + Eligibility is deliberately computed per repository: a repository with one + colliding or malformed live identifier is skipped as a whole. + """ + # Validate even for an empty fleet. + derivation_name(namespace, "RMGR-WP-0000") + root = root.resolve() + collision_report = scan_live_identifier_collisions(root) + collision_ids = set(collision_report["collisions"]) + repositories: list[dict[str, Any]] = [] + totals = {"repositories": 0, "eligible": 0, "skipped": 0, "records": 0, "replace": 0, "assign": 0, "unchanged": 0} + + for repo in iter_repo_roots(root): + fingerprint, _source_files = source_fingerprint(repo) + mappings: list[dict[str, str | None]] = [] + blockers: list[dict[str, str]] = [] + for path in iter_workplan_files(repo): + parsed = parse_workplan_file(path, repo_root=repo) + if parsed.status not in LIVE_WORKPLAN_STATUSES: + continue + if not parsed.id: + blockers.append({"path": parsed.path, "reason": "live workplan has no canonical id"}) + continue + if parsed.id in collision_ids: + blockers.append({"path": parsed.path, "record_id": parsed.id, "reason": "live identifier collision"}) + try: + mappings.append( + _migration_mapping( + namespace=namespace, + repo=repo, + path=parsed.path, + kind="workplan", + identifier=parsed.id, + current_uuid=parsed.state_hub_workstream_id, + ) + ) + except ValueError as exc: + blockers.append({"path": parsed.path, "record_id": parsed.id, "reason": str(exc)}) + + for task in parsed.tasks: + if task.status in {"done", "cancel"}: + continue + if not task.id: + blockers.append({"path": parsed.path, "reason": "unfinished task has no canonical id"}) + continue + if task.id in collision_ids: + blockers.append({"path": parsed.path, "record_id": task.id, "reason": "live identifier collision"}) + try: + mappings.append( + _migration_mapping( + namespace=namespace, + repo=repo, + path=parsed.path, + kind="task", + identifier=task.id, + current_uuid=task.state_hub_task_id, + ) + ) + except ValueError as exc: + blockers.append({"path": parsed.path, "record_id": task.id, "reason": str(exc)}) + + if not mappings and not blockers: + continue + eligible = not blockers + repositories.append( + { + "repo": repo.name, + "path": str(repo), + "planned_head_sha": head_sha(repo), + "source_fingerprint": fingerprint, + "eligible": eligible, + "atomic_unit": True, + "blockers": blockers, + "mappings": mappings, + } + ) + totals["repositories"] += 1 + totals["eligible" if eligible else "skipped"] += 1 + totals["records"] += len(mappings) + for mapping in mappings: + totals[str(mapping["action"])] += 1 + + report = { + "schema": "repo-manager.identifier-migration-plan.v1", + "ok": totals["skipped"] == 0, + "ready_to_apply": totals["skipped"] == 0, + "root": str(root), + "namespace": namespace.strip(), + "derivation_version": DERIVATION_VERSION, + "namespace_uuid": str(DERIVATION_NAMESPACE_UUID), + "scope": "live workplans and unfinished tasks", + "apply_policy": "all-or-nothing per repository", + "generated_at": utc_now_text(), + "totals": totals, + "repositories": repositories, + } + canonical = json.dumps(report, sort_keys=True, separators=(",", ":")).encode("utf-8") + report["plan_sha256"] = hashlib.sha256(canonical).hexdigest() + return report + + +def verify_identifier_migration_plan( + plan: dict[str, Any], *, repo_slug: str | None = None +) -> dict[str, Any]: + """Verify a saved plan against its seal and current repository sources.""" + errors: list[dict[str, str]] = [] + if plan.get("schema") != "repo-manager.identifier-migration-plan.v1": + errors.append({"scope": "plan", "reason": "unsupported schema"}) + expected_seal = plan.get("plan_sha256") + unsealed = {key: value for key, value in plan.items() if key != "plan_sha256"} + canonical = json.dumps(unsealed, sort_keys=True, separators=(",", ":")).encode("utf-8") + actual_seal = hashlib.sha256(canonical).hexdigest() + if expected_seal != actual_seal: + errors.append({"scope": "plan", "reason": "plan SHA-256 mismatch"}) + try: + declared_namespace = load_fleet_namespace() + except (OSError, ValueError) as exc: + errors.append({"scope": "namespace", "reason": str(exc)}) + declared_namespace = None + if plan.get("namespace") != declared_namespace: + errors.append( + { + "scope": "namespace", + "reason": f"plan={plan.get('namespace')!r}, declared={declared_namespace!r}", + } + ) + if not plan.get("ready_to_apply"): + errors.append({"scope": "plan", "reason": "plan is not ready_to_apply"}) + + repositories = list(plan.get("repositories") or []) + if repo_slug is not None: + repositories = [item for item in repositories if item.get("repo") == repo_slug] + if not repositories: + errors.append({"scope": repo_slug, "reason": "repository is absent from plan"}) + + checked_repositories = 0 + repository_results: list[dict[str, Any]] = [] + for repository in repositories: + repo = Path(str(repository.get("path") or "")) + scope = str(repository.get("repo") or repo.name or "repository") + before = len(errors) + if not repository.get("eligible"): + errors.append({"scope": scope, "reason": "repository is ineligible"}) + repository_results.append({"repo": scope, "ok": False}) + continue + if not repo.is_dir(): + errors.append({"scope": scope, "reason": f"repository path is missing: {repo}"}) + repository_results.append({"repo": scope, "ok": False}) + continue + checked_repositories += 1 + head_changed = head_sha(repo) != repository.get("planned_head_sha") + current_fingerprint, _source_files = source_fingerprint(repo) + if current_fingerprint != repository.get("source_fingerprint"): + errors.append({"scope": scope, "reason": "authoritative source changed after planning"}) + repository_results.append( + {"repo": scope, "ok": len(errors) == before, "head_changed": head_changed} + ) + + return { + "schema": "repo-manager.identifier-migration-verification.v1", + "ok": not errors, + "namespace": plan.get("namespace"), + "plan_sha256": expected_seal, + "repo_filter": repo_slug, + "checked_repositories": checked_repositories, + "repository_results": repository_results, + "errors": errors, + "source_preconditions_satisfied": not errors, + "apply_authorized": False, + "remaining_gate": "transactional central-projection migration and rollback", + } + + +def _git_cutover_preflight(repo: Path, *, expected_head_sha: str | None) -> dict[str, Any]: + errors: list[str] = [] + + def git(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=repo, + capture_output=True, + text=True, + check=False, + ) + + current_head = head_sha(repo) + if current_head is None: + errors.append("repository has no Git HEAD") + elif current_head != expected_head_sha: + errors.append("Git HEAD changed after migration planning") + + status = git("status", "--porcelain") + if status.returncode != 0: + errors.append(status.stderr.strip() or "could not inspect Git worktree") + elif status.stdout.strip(): + errors.append("worktree is not clean") + + upstream = git("rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}") + upstream_name = upstream.stdout.strip() if upstream.returncode == 0 else None + behind: int | None = None + ahead: int | None = None + if upstream_name is None: + errors.append("current branch has no upstream") + else: + counts = git("rev-list", "--left-right", "--count", "@{u}...HEAD") + try: + behind, ahead = (int(value) for value in counts.stdout.split()) + except (TypeError, ValueError): + errors.append(counts.stderr.strip() or "could not compare HEAD with upstream") + else: + if behind or ahead: + errors.append("branch does not exactly match its upstream") + + remote = git("remote", "get-url", "origin") + origin = remote.stdout.strip() if remote.returncode == 0 else None + if origin is None: + errors.append("origin remote is missing") + elif any( + marker in origin + for marker in ("gitea-remote", "gitea.coulomb.social", "92.205.130.254") + ): + errors.append("origin targets the retired Gitea lineage") + + return { + "ok": not errors, + "head_sha": current_head, + "expected_head_sha": expected_head_sha, + "upstream": upstream_name, + "behind": behind, + "ahead": ahead, + "origin": origin, + "errors": errors, + } + + +def _projection_migration_preflight( + mappings: list[dict[str, Any]], api_bases: list[str] +) -> dict[str, Any]: + """Classify every replacement without assuming which side currently exists. + + A projection may legitimately be mixed when forge reconciliation created + some deterministic targets before the sealed migration ran. Legacy-source + and derived-target are both convergent states. An exact task duplicate is + also convergent when the legacy row has no conflicting canonical identity + and both API representations have identical business fields; State Hub + rechecks that equivalence and proves the legacy row is unreferenced inside + the apply transaction. Other both-present and all neither-present states + remain hard refusals. + """ + duplicate_fields = ( + "workplan_id", + "title", + "description", + "status", + "priority", + "assignee", + "due_date", + "blocking_reason", + "needs_human", + "intervention_note", + "parent_task_id", + ) + + def mergeable_task_duplicate( + mapping: dict[str, Any], current: httpx.Response, derived: httpx.Response + ) -> bool: + if mapping.get("kind") != "task": + return False + try: + legacy_payload = current.json() + derived_payload = derived.json() + except ValueError: + return False + record_id = mapping.get("record_id") + return ( + isinstance(legacy_payload, dict) + and isinstance(derived_payload, dict) + and legacy_payload.get("id") == mapping.get("current_uuid") + and derived_payload.get("id") == mapping.get("derived_uuid") + and legacy_payload.get("record_id") in {None, record_id} + and derived_payload.get("record_id") == record_id + and all( + legacy_payload.get(field) == derived_payload.get(field) + for field in duplicate_fields + ) + ) + + projections: list[dict[str, Any]] = [] + errors: list[dict[str, str]] = [ + { + "scope": str(mapping.get("record_id") or "unknown"), + "reason": "projection-aware approval does not yet support UUID assignments", + } + for mapping in mappings + if mapping.get("action") == "assign" + ] + for raw_base in api_bases: + api_base = raw_base.strip().rstrip("/") + try: + url = httpx.URL(api_base) + if ( + url.scheme not in {"http", "https"} + or not url.host + or url.username + or url.password + or url.query + or url.fragment + ): + raise ValueError + except (TypeError, ValueError): + errors.append( + {"scope": raw_base, "reason": "projection API base must be a plain HTTP(S) origin"} + ) + continue + + checks: list[dict[str, Any]] = [] + try: + with httpx.Client(timeout=10.0, follow_redirects=False) as client: + for mapping in mappings: + if mapping.get("action") != "replace": + continue + route = "workplans" if mapping.get("kind") == "workplan" else "tasks" + current = client.get(f"{api_base}/{route}/{mapping.get('current_uuid')}") + derived = client.get(f"{api_base}/{route}/{mapping.get('derived_uuid')}") + status_pair = (current.status_code, derived.status_code) + state = { + (200, 404): "legacy_source", + (404, 200): "derived_target", + (200, 200): "both_present", + (404, 404): "neither_present", + }.get(status_pair, "unexpected_status") + if state == "both_present" and mergeable_task_duplicate( + mapping, current, derived + ): + state = "duplicate_target" + check_ok = state in { + "legacy_source", + "derived_target", + "duplicate_target", + } + check = { + "record_id": mapping.get("record_id"), + "kind": mapping.get("kind"), + "current_status": current.status_code, + "derived_status": derived.status_code, + "state": state, + "ok": check_ok, + } + checks.append(check) + if not check_ok: + errors.append( + { + "scope": f"{api_base}:{mapping.get('record_id')}", + "reason": ( + "projection is ambiguous: both legacy and derived UUIDs exist" + if state == "both_present" + else "projection is incomplete: neither legacy nor derived UUID exists" + if state == "neither_present" + else "projection returned an unexpected HTTP status" + ), + } + ) + except httpx.HTTPError: + errors.append({"scope": api_base, "reason": "projection API is unavailable"}) + state_counts = { + state: sum(check["state"] == state for check in checks) + for state in ( + "legacy_source", + "derived_target", + "duplicate_target", + "both_present", + "neither_present", + "unexpected_status", + ) + } + convergent_states = {check["state"] for check in checks if check["ok"]} + mode = ( + "legacy_migration" + if convergent_states == {"legacy_source"} + else "file_convergence" + if convergent_states == {"derived_target"} + else "duplicate_convergence" + if convergent_states == {"duplicate_target"} + else "mixed_convergence" + if convergent_states + else "blocked" + ) + projections.append( + { + "api_base": api_base, + "ok": all(check["ok"] for check in checks), + "mode": mode, + "state_counts": state_counts, + "replacement_checks": checks, + } + ) + + return { + "required": True, + "ok": bool(projections) and not errors, + "projections": projections, + "errors": errors, + } + + +def plan_identifier_migration_batch( + plan: dict[str, Any], + *, + repo_slugs: list[str], + projection_api_bases: list[str] | None = None, +) -> dict[str, Any]: + """Pin a reviewed repository batch after source and Git preflight. + + The result is an approval package, not apply authority. Live database and + file mutation still require an explicit decision citing ``batch_sha256``. + """ + plan_sha256 = _verified_plan_seal(plan) + requested = [slug.strip() for slug in repo_slugs if slug.strip()] + errors: list[dict[str, str]] = [] + if not requested: + errors.append({"scope": "batch", "reason": "at least one repository is required"}) + if len(set(requested)) != len(requested): + errors.append({"scope": "batch", "reason": "repository list contains duplicates"}) + + repositories_by_slug: dict[str, list[dict[str, Any]]] = defaultdict(list) + for repository in plan.get("repositories", []): + repositories_by_slug[str(repository.get("repo") or "")].append(repository) + + projection_api_bases = list(projection_api_bases or []) + batch_repositories: list[dict[str, Any]] = [] + totals = {"repositories": 0, "records": 0, "replace": 0, "assign": 0, "unchanged": 0} + for slug in requested: + matches = repositories_by_slug.get(slug, []) + if len(matches) != 1: + errors.append( + { + "scope": slug, + "reason": "repository must occur exactly once in the source plan", + } + ) + continue + repository = matches[0] + source_verification = verify_identifier_migration_plan(plan, repo_slug=slug) + repo = Path(str(repository.get("path") or "")).resolve() + git_preflight = _git_cutover_preflight( + repo, + expected_head_sha=repository.get("planned_head_sha"), + ) + mappings = list(repository.get("mappings") or []) + action_counts = { + action: sum(mapping.get("action") == action for mapping in mappings) + for action in ("replace", "assign", "unchanged") + } + actionable = action_counts["replace"] + action_counts["assign"] + if actionable == 0: + errors.append({"scope": slug, "reason": "repository has no migration action"}) + for error in source_verification["errors"]: + errors.append({"scope": slug, "reason": error["reason"]}) + for reason in git_preflight["errors"]: + errors.append({"scope": slug, "reason": reason}) + projection_preflight = None + if projection_api_bases: + projection_preflight = _projection_migration_preflight( + mappings, projection_api_bases + ) + errors.extend(projection_preflight["errors"]) + + batch_repositories.append( + { + "repo": slug, + "path": str(repo), + "planned_head_sha": repository.get("planned_head_sha"), + "source_fingerprint": repository.get("source_fingerprint"), + "source_verified": source_verification["ok"], + "git_preflight": git_preflight, + "projection_preflight": projection_preflight, + "mapping_counts": {"records": len(mappings), **action_counts}, + "ready": ( + source_verification["ok"] + and git_preflight["ok"] + and actionable > 0 + and (projection_preflight is None or projection_preflight["ok"]) + ), + } + ) + totals["repositories"] += 1 + totals["records"] += len(mappings) + for action, count in action_counts.items(): + totals[action] += count + + result = { + "schema": "repo-manager.identifier-migration-batch.v1", + "ok": not errors, + "ready_for_approval": not errors, + "apply_authorized": False, + "approval_required": True, + "projection_api_bases": [base.strip().rstrip("/") for base in projection_api_bases], + "namespace": plan.get("namespace"), + "source_plan_sha256": plan_sha256, + "batch_policy": "repository-atomic, sequential, stop on first failure", + "rollback_order": "reverse files if written, then reverse central projection", + "generated_at": utc_now_text(), + "totals": totals, + "repositories": batch_repositories, + "errors": errors, + } + canonical = json.dumps(result, sort_keys=True, separators=(",", ":")).encode("utf-8") + result["batch_sha256"] = hashlib.sha256(canonical).hexdigest() + return result + + +def verify_identifier_migration_batch( + batch: dict[str, Any], *, plan: dict[str, Any] +) -> dict[str, Any]: + """Verify a saved batch seal and repeat every selected source/Git check.""" + errors: list[dict[str, str]] = [] + expected_batch_sha256 = batch.get("batch_sha256") + unsealed = {key: value for key, value in batch.items() if key != "batch_sha256"} + canonical = json.dumps(unsealed, sort_keys=True, separators=(",", ":")).encode("utf-8") + actual_batch_sha256 = hashlib.sha256(canonical).hexdigest() + if batch.get("schema") != "repo-manager.identifier-migration-batch.v1": + errors.append({"scope": "batch", "reason": "unsupported batch schema"}) + if expected_batch_sha256 != actual_batch_sha256: + errors.append({"scope": "batch", "reason": "batch SHA-256 mismatch"}) + + try: + plan_sha256 = _verified_plan_seal(plan) + except (OSError, ValueError) as exc: + errors.append({"scope": "plan", "reason": str(exc)}) + plan_sha256 = None + if batch.get("source_plan_sha256") != plan_sha256: + errors.append({"scope": "plan", "reason": "batch source plan SHA-256 mismatch"}) + if batch.get("apply_authorized") is not False or batch.get("approval_required") is not True: + errors.append( + { + "scope": "batch", + "reason": "batch must remain unauthorized and approval-required", + } + ) + + saved_repositories = list(batch.get("repositories") or []) + repo_slugs = [str(item.get("repo") or "") for item in saved_repositories] + if not repo_slugs or any(not slug for slug in repo_slugs): + errors.append({"scope": "batch", "reason": "batch has no valid repository scope"}) + fresh = None + else: + try: + fresh = plan_identifier_migration_batch( + plan, + repo_slugs=repo_slugs, + projection_api_bases=list(batch.get("projection_api_bases") or []), + ) + except (OSError, ValueError) as exc: + errors.append({"scope": "batch", "reason": str(exc)}) + fresh = None + + if fresh is not None: + errors.extend(fresh["errors"]) + fresh_by_slug = {item["repo"]: item for item in fresh["repositories"]} + for saved in saved_repositories: + slug = str(saved.get("repo") or "") + current = fresh_by_slug.get(slug) + if current is None: + errors.append({"scope": slug, "reason": "repository disappeared from batch"}) + continue + for field in ("planned_head_sha", "source_fingerprint", "mapping_counts"): + if saved.get(field) != current.get(field): + errors.append({"scope": slug, "reason": f"saved {field} differs from plan"}) + + return { + "schema": "repo-manager.identifier-migration-batch-verification.v1", + "ok": not errors, + "ready_for_decision": not errors, + "apply_authorized": False, + "approval_required": True, + "batch_sha256": expected_batch_sha256, + "source_plan_sha256": plan_sha256, + "repositories": repo_slugs, + "errors": errors, + } + + +def _verified_plan_seal(plan: dict[str, Any]) -> str: + if plan.get("schema") != "repo-manager.identifier-migration-plan.v1": + raise ValueError("unsupported identifier migration plan schema") + expected = plan.get("plan_sha256") + if not isinstance(expected, str) or not re.fullmatch(r"[0-9a-f]{64}", expected): + raise ValueError("migration plan has no valid SHA-256 seal") + unsealed = {key: value for key, value in plan.items() if key != "plan_sha256"} + canonical = json.dumps(unsealed, sort_keys=True, separators=(",", ":")).encode("utf-8") + if hashlib.sha256(canonical).hexdigest() != expected: + raise ValueError("migration plan SHA-256 mismatch") + if plan.get("namespace") != load_fleet_namespace(): + raise ValueError("migration plan namespace does not match the fleet declaration") + return expected + + +def _yaml_scalar(value: str | None) -> str | None: + return value.strip().strip('"').strip("'") if value is not None else None + + +def _patch_yaml_field( + raw: str, + *, + key: str, + expected: str | None, + target: str | None, + scope: str, +) -> str: + pattern = re.compile(rf"^(?P{re.escape(key)}:\s*)(?P.*)$", re.MULTILINE) + match = pattern.search(raw) + current = _yaml_scalar(match.group("value")) if match else None + if current != expected: + raise ValueError(f"{scope}: expected {key}={expected!r}, found {current!r}") + if target is None: + if match is None: + return raw + start, end = match.span() + if end < len(raw) and raw[end : end + 1] == "\n": + end += 1 + return raw[:start] + raw[end:] + replacement = f'{key}: "{target}"' + if match is not None: + return raw[: match.start()] + replacement + raw[match.end() :] + separator = "" if not raw or raw.endswith("\n") else "\n" + return f"{raw}{separator}{replacement}\n" + + +def _patch_workplan_identifier( + text: str, + *, + record_id: str, + expected: str | None, + target: str | None, +) -> str: + if not text.startswith("---"): + raise ValueError(f"{record_id}: missing YAML frontmatter") + parts = text.split("---", 2) + if len(parts) != 3: + raise ValueError(f"{record_id}: malformed YAML frontmatter") + metadata = _parse_yaml_block(parts[1].strip()) + if metadata.get("id") != record_id: + raise ValueError(f"{record_id}: workplan id does not match source file") + patched = _patch_yaml_field( + parts[1].lstrip("\n"), + key="state_hub_workstream_id", + expected=expected, + target=target, + scope=record_id, + ) + return f"---\n{patched.rstrip()}\n---{parts[2]}" + + +def _patch_task_identifier( + text: str, + *, + record_id: str, + expected: str | None, + target: str | None, +) -> str: + matched = 0 + + def replace(match: re.Match[str]) -> str: + nonlocal matched + raw = match.group(1) + metadata = _parse_yaml_block(raw.strip()) + if metadata.get("id") != record_id: + return match.group(0) + matched += 1 + patched = _patch_yaml_field( + raw, + key="state_hub_task_id", + expected=expected, + target=target, + scope=record_id, + ) + return f"```task\n{patched.rstrip()}\n```" + + patched_text = _TASK_BLOCK_RE.sub(replace, text) + if matched != 1: + raise ValueError(f"{record_id}: expected exactly one task block, found {matched}") + return patched_text + + +def _atomic_write(path: Path, content: str) -> None: + mode = path.stat().st_mode + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.identifier-migration-", + dir=path.parent, + ) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as temporary: + temporary.write(content) + temporary.flush() + os.fsync(temporary.fileno()) + os.chmod(temporary_name, mode) + os.replace(temporary_name, path) + finally: + if os.path.exists(temporary_name): + os.unlink(temporary_name) + + +def ensure_missing_work_record_identifiers( + repo_root: Path, + *, + namespace: str | None = None, + execute: bool = True, +) -> dict[str, Any]: + """Assign only missing canonical UUIDs, deterministically, in one repository. + + Existing UUIDs are deliberately preserved. Replacing a legacy UUID is a + separately sealed migration; ordinary registration must never perform it. + Every source file is validated and rendered before any file is changed. + """ + repo_root = repo_root.resolve() + namespace = namespace or load_fleet_namespace() + identity = scan_record_identities(repo_root) + # Completed, already-bound suffix tasks are historical evidence, not + # allocation candidates. Keep conformance reporting strict while allowing + # ordinary registration to preserve those bindings. Local aliases still + # require the explicit parent allowlist in the kind registry. + from repo_manager.historical_sources import historical_source_occurrences + frozen_sources = historical_source_occurrences(repo_root) + historical_sources: set[str] = set(frozen_sources) + for path in iter_workplan_files(repo_root): + parsed = parse_workplan_file(path, repo_root=repo_root) + if parsed.status not in {"finished", "archived", "completed"} or not parsed.id: + continue + for number, task in enumerate(parsed.tasks, start=1): + if task.status not in {"done", "cancel"} or not task.state_hub_task_id: + continue + # RAILIANCE-WP-0004 is a closed historical workplan whose source + # used the old I01..I07 item labels. These exact, UUID-bound + # labels are preserved like the older suffix records below; they + # must not become a general I-suffix grandfather rule. + legacy_item = ( + parsed.id == "RAILIANCE-WP-0004" + and re.fullmatch(r"RAILIANCE-WP-0004-I(?:0[1-7])", task.id or "") + ) + if not ( + re.fullmatch(re.escape(parsed.id) + r"-T[0-9]{2}[a-z]", task.id or "") + or legacy_item + ): + continue + try: + uuid.UUID(task.state_hub_task_id) + except ValueError: + continue + historical_sources.add(f"{parsed.path}#task-block-{number}") + invalid = [ + item for item in identity["invalid_identifiers"] + if item["source"] not in historical_sources + ] + historical = [ + item for item in identity["invalid_identifiers"] + if item["source"] in historical_sources + ] + # Strict scans keep legacy nonconformance. Only collisions wholly contained + # in exact hash-bound closed task occurrences can be preserved by receiving; + # any unknown/live source or qualified/workplan collision remains fatal. + historical_collisions=[item for item in identity['identity_collisions'] + if item['kind']=='task' and re.fullmatch(r'T[0-9]{2}',item['id']) + and all(source in frozen_sources for source in item['sources'])] + collisions=[item for item in identity['identity_collisions'] if item not in historical_collisions] + if invalid or collisions: + raise ValueError( + "work-record identities are not safe to derive: " + f"invalid={invalid!r}, " + f"collisions={collisions!r}" + ) + + rendered: dict[Path, str] = {} + assignments: list[dict[str, str]] = [] + skipped: list[dict[str, str]] = [] + for path in iter_workplan_files(repo_root): + parsed = parse_workplan_file(path, repo_root=repo_root) + text = path.read_text(encoding="utf-8") + changed = False + if parsed.id and not parsed.state_hub_workstream_id: + if classify_record_id("workplan", parsed.id) != "canonical": + skipped.append( + {"kind": "workplan", "record_id": parsed.id, "path": parsed.path} + ) + else: + derived = str(derive_work_record_uuid(namespace, parsed.id)) + text = _patch_workplan_identifier( + text, + record_id=parsed.id, + expected=None, + target=derived, + ) + assignments.append( + { + "kind": "workplan", + "record_id": parsed.id, + "uuid": derived, + "path": parsed.path, + } + ) + changed = True + for task in parsed.tasks: + if not task.id or task.state_hub_task_id: + continue + if classify_record_id("task", task.id) != "canonical": + skipped.append({"kind": "task", "record_id": task.id, "path": parsed.path}) + continue + derived = str(derive_work_record_uuid(namespace, task.id)) + text = _patch_task_identifier( + text, + record_id=task.id, + expected=None, + target=derived, + ) + assignments.append( + { + "kind": "task", + "record_id": task.id, + "uuid": derived, + "path": parsed.path, + } + ) + changed = True + if changed: + rendered[path] = text + + if skipped: + raise ValueError( + "missing UUIDs on grandfathered or noncanonical records require " + f"an explicit identity decision: {skipped!r}" + ) + if execute: + for path, content in rendered.items(): + _atomic_write(path, content) + return { + "schema": "repo-manager.identifier-assignment.v1", + "ok": True, + "executed": execute, + "repo": repo_root.name, + "namespace": namespace, + "assignments": assignments, + "historical_invalid_identifiers": historical, + "historical_identity_collisions": historical_collisions, + "files_changed": sorted(str(path.relative_to(repo_root)) for path in rendered), + } + + +def migrate_repository_identifier_files( + plan: dict[str, Any], + *, + repo_slug: str, + confirm_plan_sha256: str, + direction: str = "forward", + execute: bool = False, +) -> dict[str, Any]: + """Validate and optionally rewrite one repository's UUID fields atomically. + + This is the file half of the governed cutover. The caller coordinates it + with State Hub's database transaction and invokes ``direction='reverse'`` + if the file or subsequent consistency phase fails. + """ + plan_sha256 = _verified_plan_seal(plan) + if confirm_plan_sha256 != plan_sha256: + raise ValueError("--confirm-plan-sha256 does not match the sealed plan") + if direction not in {"forward", "reverse"}: + raise ValueError("direction must be 'forward' or 'reverse'") + + repositories = [item for item in plan.get("repositories", []) if item.get("repo") == repo_slug] + if len(repositories) != 1: + raise ValueError(f"repository {repo_slug!r} must occur exactly once in the plan") + repository = repositories[0] + if repository.get("eligible") is not True or repository.get("atomic_unit") is not True: + raise ValueError(f"repository {repo_slug!r} is not eligible and atomic") + + if direction == "forward": + verification = verify_identifier_migration_plan(plan, repo_slug=repo_slug) + if not verification["ok"]: + reasons = "; ".join(error["reason"] for error in verification["errors"]) + raise ValueError(f"repository source preconditions failed: {reasons}") + + repo_root = Path(str(repository.get("path") or "")).resolve() + originals: dict[Path, str] = {} + patched: dict[Path, str] = {} + replacements = 0 + assignments = 0 + for mapping in repository.get("mappings", []): + action = mapping.get("action") + if action == "unchanged": + continue + if action not in {"replace", "assign"}: + raise ValueError(f"unsupported mapping action {action!r}") + if mapping.get("repo") != repo_slug: + raise ValueError("mapping repository does not match the atomic unit") + path = (repo_root / str(mapping.get("path") or "")).resolve() + if repo_root not in path.parents or not path.is_file(): + raise ValueError(f"mapping source is missing or escapes repository: {path}") + if path not in originals: + originals[path] = path.read_text(encoding="utf-8") + patched[path] = originals[path] + + old_id = mapping.get("current_uuid") + new_id = mapping.get("derived_uuid") + expected = old_id if direction == "forward" else new_id + target = new_id if direction == "forward" else old_id + record_id = str(mapping.get("record_id") or "") + if mapping.get("kind") == "workplan": + patched[path] = _patch_workplan_identifier( + patched[path], record_id=record_id, expected=expected, target=target + ) + elif mapping.get("kind") == "task": + patched[path] = _patch_task_identifier( + patched[path], record_id=record_id, expected=expected, target=target + ) + else: + raise ValueError(f"unsupported record kind {mapping.get('kind')!r}") + replacements += action == "replace" + assignments += action == "assign" + + touched = [str(path.relative_to(repo_root)) for path in sorted(patched)] + if execute: + written: list[Path] = [] + try: + for path, content in patched.items(): + if content != originals[path]: + _atomic_write(path, content) + written.append(path) + except Exception: + restore_errors: list[str] = [] + for path in reversed(written): + try: + _atomic_write(path, originals[path]) + except Exception as exc: # noqa: BLE001 + restore_errors.append(f"{path}: {exc}") + if restore_errors: + raise RuntimeError( + "file migration failed and rollback was incomplete: " + "; ".join(restore_errors) + ) + raise + + return { + "schema": "repo-manager.identifier-file-migration.v1", + "ok": True, + "executed": execute, + "direction": direction, + "repo": repo_slug, + "plan_sha256": plan_sha256, + "files_touched": touched, + "replacements": replacements, + "assignments": assignments, + "database_coordinated": False, + } + + +def migrate_repository_projection( + plan: dict[str, Any], + *, + repo_slug: str, + confirm_plan_sha256: str, + api_base: str, + direction: str = "forward", + expected_instance_label: str | None = "railiance01", + transport: httpx.BaseTransport | None = None, +) -> dict[str, Any]: + """Apply or reverse the sealed projection half on the authoritative hub.""" + plan_sha256 = _verified_plan_seal(plan) + if confirm_plan_sha256 != plan_sha256: + raise ValueError("--confirm-plan-sha256 does not match the sealed plan") + if direction not in {"forward", "reverse"}: + raise ValueError("direction must be 'forward' or 'reverse'") + verification = verify_identifier_migration_plan(plan, repo_slug=repo_slug) + if not verification["ok"]: + reasons = "; ".join(error["reason"] for error in verification["errors"]) + raise ValueError(f"repository source preconditions failed: {reasons}") + repositories = [ + item for item in plan.get("repositories", []) if item.get("repo") == repo_slug + ] + repository = repositories[0] + git_preflight = _git_cutover_preflight( + Path(str(repository.get("path") or "")).resolve(), + expected_head_sha=repository.get("planned_head_sha"), + ) + if not git_preflight["ok"]: + raise ValueError( + "repository Git preconditions failed: " + "; ".join(git_preflight["errors"]) + ) + api_base = api_base.strip().rstrip("/") + try: + url = httpx.URL(api_base) + if ( + url.scheme not in {"http", "https"} + or not url.host + or url.username + or url.password + or url.query + or url.fragment + ): + raise ValueError + except (TypeError, ValueError) as exc: + raise ValueError("projection API base must be a plain HTTP(S) origin") from exc + + headers = { + "Idempotency-Key": f"rmgr-identifier-migration:{direction}:{repo_slug}:{plan_sha256}", + "X-StateHub-Source-Agent": "repo-manager", + } + try: + with httpx.Client( + base_url=api_base, + timeout=httpx.Timeout(120.0, connect=5.0), + follow_redirects=False, + transport=transport, + ) as client: + health = client.get("/state/health") + health.raise_for_status() + identity = health.json() + if identity.get("instance_role") != "primary" or ( + expected_instance_label is not None + and identity.get("instance_label") != expected_instance_label + ): + raise ValueError("identifier migration requires the expected primary State Hub") + response = client.post( + f"/identifier-migrations/repositories/{repo_slug}/" + f"{'apply' if direction == 'forward' else 'reverse'}", + json={ + "plan": plan, + "expected_plan_sha256": plan_sha256, + "primary_confirmed": True, + }, + headers=headers, + ) + response.raise_for_status() + except httpx.HTTPStatusError as exc: + try: + detail = exc.response.json() + except ValueError: + detail = exc.response.text[:500] + raise ValueError(f"projection migration rejected: {detail}") from exc + except httpx.HTTPError as exc: + raise ValueError(f"projection migration unavailable: {exc}") from exc + receipt = response.json() + return { + "schema": "repo-manager.identifier-projection-migration.v1", + "ok": True, + "direction": direction, + "repo": repo_slug, + "plan_sha256": plan_sha256, + "api_base": api_base, + "instance": identity, + "source_verification": verification, + "git_preflight": git_preflight, + "receipt": receipt, + } diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/index_store.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/index_store.py new file mode 100644 index 00000000..9e4b660a --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/index_store.py @@ -0,0 +1,109 @@ +"""Local JSON projection store for the vertical-slice proof (pre-Postgres).""" + +from __future__ import annotations + +import json +import os +import tempfile +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +from repo_manager.time import utc_now_text + + +def _now() -> str: + return utc_now_text() + + +@dataclass +class WorkRecordEntry: + kind: str + id: str | None + status: str | None + title: str | None + source_path: str + uuid: str | None = None + parent_id: str | None = None + extra: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class RepoIndex: + slug: str + repo_root: str + head_sha: str | None + observed_at: str + source_fingerprint: str | None = None + source_files: list[str] = field(default_factory=list) + work_records: list[WorkRecordEntry] = field(default_factory=list) + events: list[dict[str, Any]] = field(default_factory=list) + schema: str = "repo_manager.index.v1" + + def to_dict(self) -> dict[str, Any]: + return { + "schema": self.schema, + "slug": self.slug, + "repo_root": self.repo_root, + "head_sha": self.head_sha, + "observed_at": self.observed_at, + "source_fingerprint": self.source_fingerprint, + "source_files": self.source_files, + "work_records": [asdict(r) for r in self.work_records], + "events": self.events, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> RepoIndex: + records = [ + WorkRecordEntry(**{k: v for k, v in r.items() if k in WorkRecordEntry.__dataclass_fields__}) + for r in data.get("work_records", []) + ] + return cls( + slug=data["slug"], + repo_root=data["repo_root"], + head_sha=data.get("head_sha"), + observed_at=data.get("observed_at") or _now(), + source_fingerprint=data.get("source_fingerprint"), + source_files=list(data.get("source_files") or []), + work_records=records, + events=list(data.get("events") or []), + schema=data.get("schema") or "repo_manager.index.v0", + ) + + +def default_index_path(repo_root: Path) -> Path: + return repo_root / ".repo-manager" / "index.json" + + +def save_index(index: RepoIndex, path: Path | None = None) -> Path: + path = path or default_index_path(Path(index.repo_root)) + path.parent.mkdir(parents=True, exist_ok=True) + temporary: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=path.parent, + prefix=f".{path.name}.", + delete=False, + ) as handle: + temporary = Path(handle.name) + handle.write(json.dumps(index.to_dict(), indent=2) + "\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + temporary = None + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + return path + + +def load_index(path: Path) -> RepoIndex: + return RepoIndex.from_dict(json.loads(path.read_text(encoding="utf-8"))) + + +def append_event(index: RepoIndex, event: dict[str, Any]) -> None: + event = {**event, "emitted_at": event.get("emitted_at") or _now()} + index.events.append(event) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/observe.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/observe.py new file mode 100644 index 00000000..8618f89a --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/observe.py @@ -0,0 +1,210 @@ +"""Build repository observation + work-record index from files.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import yaml + +from repo_manager.cache import source_fingerprint +from repo_manager.classification import require_valid_classification +from repo_manager.gitops import head_sha, is_git_repo +from repo_manager.index_store import RepoIndex, WorkRecordEntry, _now +from repo_manager.parse.record import iter_record_files, parse_record_file +from repo_manager.parse.register import iter_register_files, parse_register_file +from repo_manager.parse.workplan import iter_workplan_files, parse_workplan_file +from repo_manager.record_identity import scan_record_identities + + +class RecordIdentityCollisionError(ValueError): + """Raised when one canonical record id points at conflicting UUIDs.""" + + +def _slug_from_path(repo_root: Path) -> str: + return re.sub(r"[^a-z0-9]+", "-", repo_root.name.lower()).strip("-") or "repo" + + +def load_classification(repo_root: Path) -> dict | None: + path = repo_root / ".repo-classification.yaml" + if not path.is_file(): + return None + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if isinstance(data, dict) and "repo_classification" in data: + classification = data["repo_classification"] + else: + classification = data if isinstance(data, dict) else None + if not isinstance(classification, dict): + return None + return require_valid_classification(classification) + + +def observe_repository(repo_root: Path, *, slug: str | None = None) -> tuple[dict, RepoIndex]: + """Return (RepositorySnapshot-like dict, rebuilt RepoIndex).""" + repo_root = repo_root.resolve() + slug = slug or _slug_from_path(repo_root) + classification = load_classification(repo_root) + intent_path = None + description = None + for name in ("INTENT.md", "GOAL.md"): + p = repo_root / name + if p.is_file(): + intent_path = name + # first non-empty line of body as weak description + text = p.read_text(encoding="utf-8") + for line in text.splitlines(): + line = line.strip() + if line and not line.startswith("#") and not line.startswith("---") and not line.startswith(">"): + description = line[:200] + break + break + + records: list[WorkRecordEntry] = [] + for path in iter_workplan_files(repo_root): + wp = parse_workplan_file(path, repo_root=repo_root) + records.append( + WorkRecordEntry( + kind="workplan", + id=wp.id, + status=wp.status, + title=wp.title, + source_path=wp.path, + uuid=wp.state_hub_workstream_id, + extra={ + key: wp.frontmatter[key] + for key in ("depends_on", "related", "needs_human", "intervention_note") + if key in wp.frontmatter + }, + ) + ) + for task_number, task in enumerate(wp.tasks, start=1): + records.append( + WorkRecordEntry( + kind="task", + id=task.id, + status=task.status, + title=task.title, + source_path=wp.path, + uuid=task.state_hub_task_id, + parent_id=wp.id, + extra={ + "source_occurrence": f"{wp.path}#task-block-{task_number}", + **{ + key: task.raw[key] + for key in ( + "depends_on", + "needs_human", + "intervention_note", + "blocking_reason", + ) + if key in task.raw + }, + }, + ) + ) + + for path in iter_record_files(repo_root): + for record_number, record in enumerate(parse_record_file(path, repo_root=repo_root), start=1): + records.append( + WorkRecordEntry( + kind=record.kind, + id=record.id, + status=record.status, + title=record.title, + source_path=record.source_path, + uuid=record.uuid, + extra={ + "record": record.raw, + "source_occurrence": f"{record.source_path}#record-block-{record_number}", + }, + ) + ) + + for path in iter_register_files(repo_root): + for entry in parse_register_file(path, repo_root=repo_root): + records.append( + WorkRecordEntry( + kind=f"register:{entry.kind}", + id=entry.id, + status=entry.status, + title=entry.title, + source_path=entry.source_path, + extra={"register_kind": entry.kind, "entry": entry.raw}, + ) + ) + + identity = scan_record_identities(repo_root) + if identity["identity_collisions"]: + raise RecordIdentityCollisionError( + f"work-record identity collision: {identity['identity_collisions']}" + ) + + diagnostic_events: list[dict] = [] + for duplicate in identity["duplicate_source_occurrences"]: + matches = [ + record + for record in records + if record.kind == duplicate["kind"] and record.id == duplicate["id"] + ] + if len(matches) < 2: + continue + keeper = matches[0] + keeper.extra.pop("source_occurrence", None) + keeper.extra["source_occurrences"] = duplicate["sources"] + records = [record for record in records if record is keeper or record not in matches] + diagnostic_events.append( + { + "type": "repo.work_record.duplicate_source_occurrence", + "severity": "warning", + "record_kind": duplicate["kind"], + "record_id": duplicate["id"], + "uuid": duplicate["uuid"], + "source_occurrences": duplicate["sources"], + "cleanup_required": True, + "canon": identity["canon"], + } + ) + + for record in records: + record.extra.pop("source_occurrence", None) + + sha = head_sha(repo_root) if is_git_repo(repo_root) else None + fingerprint, source_files = source_fingerprint(repo_root) + index = RepoIndex( + slug=slug, + repo_root=str(repo_root), + head_sha=sha, + observed_at=_now(), + source_fingerprint=fingerprint, + source_files=source_files, + work_records=records, + events=diagnostic_events, + ) + + snapshot = { + "api_version": "0.1", + "slug": slug, + "lifecycle": "active", + "domain": (classification or {}).get("domain"), + "classification": classification, + "purpose": { + "description": description, + "intent_path": intent_path, + }, + "locations": { + "repo_root": str(repo_root), + }, + "revision": { + "head_sha": sha, + "observed_at": index.observed_at, + }, + "index": { + "workplan_count": sum(1 for r in records if r.kind == "workplan"), + "task_count": sum(1 for r in records if r.kind == "task"), + "register_entry_count": sum(1 for r in records if r.kind.startswith("register:")), + "intake_count": sum(1 for r in records if r.kind == "intake"), + "decision_count": sum(1 for r in records if r.kind == "decision"), + "record_count": len(records), + }, + } + return snapshot, index diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/owner_interfaces.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/owner_interfaces.py new file mode 100644 index 00000000..86d1669a --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/owner_interfaces.py @@ -0,0 +1,177 @@ +"""Validation and rendering for owner-consumable task interfaces.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import yaml + +API_VERSION = "helixforge.repo-manager/v1" +KIND = "OwnerTaskInterface" +DISPOSITIONS = ("approved", "amended", "rejected") +PRIORITIES = frozenset({"high", "medium", "low"}) + + +def _error(code: str, path: Path, message: str) -> dict[str, str]: + return {"code": code, "path": str(path), "message": message} + + +def _text(value: Any) -> str | None: + if not isinstance(value, str): + return None + value = value.strip() + return value or None + + +def _string_list(value: Any) -> list[str] | None: + if not isinstance(value, list) or not value: + return None + values = [_text(item) for item in value] + if any(item is None for item in values): + return None + return [item for item in values if item is not None] + + +def _interface_paths(path: Path) -> list[Path]: + path = path.expanduser().resolve() + if path.is_file(): + return [path] + return sorted(path.glob("*.yaml")) + + +def validate_owner_interfaces(path: Path, *, owner: str | None = None) -> dict[str, Any]: + """Validate task interfaces and return their directly consumable content.""" + path = path.expanduser().resolve() + errors: list[dict[str, str]] = [] + interfaces: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + + paths = _interface_paths(path) + if not paths: + errors.append(_error("interface_not_found", path, "no interface YAML files found")) + + for interface_path in paths: + try: + raw = yaml.safe_load(interface_path.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError) as exc: + errors.append(_error("interface_invalid", interface_path, str(exc))) + continue + if not isinstance(raw, dict): + errors.append(_error("interface_invalid", interface_path, "root must be a mapping")) + continue + if raw.get("apiVersion") != API_VERSION or raw.get("kind") != KIND: + errors.append( + _error( + "interface_kind_invalid", + interface_path, + f"expected apiVersion {API_VERSION!r} and kind {KIND!r}", + ) + ) + continue + + metadata = raw.get("metadata") + source = raw.get("source") + target = raw.get("target") + approval = raw.get("approval") + task = raw.get("task") + if not all(isinstance(item, dict) for item in (metadata, source, target, approval, task)): + errors.append( + _error( + "interface_shape_invalid", + interface_path, + "metadata, source, target, approval, and task must be mappings", + ) + ) + continue + + interface_id = _text(metadata.get("id")) + title = _text(metadata.get("title")) + source_repo = _text(source.get("repo")) + source_task = _text(source.get("workplan_task")) + target_repo = _text(target.get("repo")) + owner_agent = _text(target.get("owner_agent")) + task_title = _text(task.get("title")) + objective = _text(task.get("objective")) + priority = _text(task.get("priority")) + deliverables = _string_list(task.get("deliverables")) + acceptance = _string_list(task.get("acceptance")) + dispositions = approval.get("dispositions") + + required = { + "metadata.id": interface_id, + "metadata.title": title, + "source.repo": source_repo, + "source.workplan_task": source_task, + "target.repo": target_repo, + "target.owner_agent": owner_agent, + "task.title": task_title, + "task.objective": objective, + } + missing = [name for name, value in required.items() if value is None] + if missing: + errors.append( + _error( + "interface_required_field_missing", + interface_path, + "missing non-empty fields: " + ", ".join(missing), + ) + ) + continue + if priority not in PRIORITIES: + errors.append( + _error( + "interface_priority_invalid", + interface_path, + "task.priority must be high, medium, or low", + ) + ) + continue + if deliverables is None or acceptance is None: + errors.append( + _error( + "interface_task_invalid", + interface_path, + "task.deliverables and task.acceptance must be non-empty string lists", + ) + ) + continue + if dispositions != list(DISPOSITIONS): + errors.append( + _error( + "interface_approval_invalid", + interface_path, + "approval.dispositions must be [approved, amended, rejected]", + ) + ) + continue + if "status" in raw or "status" in metadata: + errors.append( + _error( + "interface_lifecycle_forbidden", + interface_path, + "an interface is not a work record and must not carry lifecycle status", + ) + ) + continue + if interface_id in seen_ids: + errors.append( + _error("interface_id_duplicate", interface_path, f"duplicate id {interface_id!r}") + ) + continue + seen_ids.add(interface_id) + + if owner is None or owner in {target_repo, owner_agent}: + document = dict(raw) + document["path"] = str(interface_path) + interfaces.append(document) + + return { + "ok": not errors, + "apiVersion": API_VERSION, + "kind": KIND, + "root": str(path), + "interface_count": len(interfaces), + "interfaces": interfaces, + "errors": errors, + } diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/parse/__init__.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/parse/__init__.py new file mode 100644 index 00000000..2d056013 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/parse/__init__.py @@ -0,0 +1,3 @@ +from repo_manager.parse.workplan import ParsedTask, ParsedWorkplan, parse_workplan_file + +__all__ = ["ParsedTask", "ParsedWorkplan", "parse_workplan_file"] diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/parse/record.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/parse/record.py new file mode 100644 index 00000000..611049a5 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/parse/record.py @@ -0,0 +1,80 @@ +"""Parse repository-owned intake and decision records from Markdown YAML fences.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +_YAML_FENCE_RE = re.compile(r"```ya?ml\s*\n(.*?)\n```", re.DOTALL | re.IGNORECASE) +SUPPORTED_RECORD_KINDS = frozenset({"intake", "decision"}) + + +@dataclass +class ParsedRecord: + kind: str + id: str + title: str | None + status: str | None + uuid: str | None + source_path: str + raw: dict[str, Any] = field(default_factory=dict) + block_start: int = 0 + block_end: int = 0 + + +def parse_record_text(text: str, *, relative_path: str) -> list[ParsedRecord]: + """Return intake/decision records embedded in YAML fenced blocks.""" + records: list[ParsedRecord] = [] + for match in _YAML_FENCE_RE.finditer(text): + try: + data = yaml.safe_load(match.group(1)) or {} + except yaml.YAMLError: + continue + if not isinstance(data, dict) or data.get("kind") not in SUPPORTED_RECORD_KINDS: + continue + record_id = data.get("id") + if record_id is None: + continue + kind = str(data["kind"]) + uuid_field = "state_hub_intake_id" if kind == "intake" else "state_hub_decision_id" + records.append( + ParsedRecord( + kind=kind, + id=str(record_id), + title=str(data["title"]) if data.get("title") is not None else None, + status=str(data["status"]) if data.get("status") is not None else None, + uuid=str(data[uuid_field]) if data.get(uuid_field) is not None else None, + source_path=relative_path, + raw=data, + block_start=match.start(), + block_end=match.end(), + ) + ) + return records + + +def parse_record_file(path: Path, *, repo_root: Path) -> list[ParsedRecord]: + return parse_record_text( + path.read_text(encoding="utf-8"), + relative_path=str(path.relative_to(repo_root)), + ) + + +def iter_record_files(repo_root: Path) -> list[Path]: + """Find only the governed record locations; do not interpret arbitrary docs.""" + files: set[Path] = set() + for relative in ("intakes", "decisions", "docs/intakes", "docs/decisions", "workplans"): + root = repo_root / relative + if root.is_dir(): + files.update(path for path in root.rglob("*.md") if not path.name.startswith(".")) + # Fleet repositories use both title-case conventions. Keep these explicit: + # arbitrary top-level Markdown remains outside the governed record surface. + for name in ("INTAKES.md", "DECISIONS.md", "intakes.md", "decisions.md"): + path = repo_root / name + if path.is_file(): + files.add(path) + return sorted(files) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/parse/register.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/parse/register.py new file mode 100644 index 00000000..29a88163 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/parse/register.py @@ -0,0 +1,73 @@ +"""Parser for repository-owned register files.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +REGISTER_SCHEMA = "repo-manager.register.v0" +SUPPORTED_REGISTER_KINDS = frozenset( + { + "sbom-inventory", + "repo-goals", + "upstream-contributions", + "technical-debt", + "extension-points", + "register-entries", + } +) + + +@dataclass +class ParsedRegisterEntry: + kind: str + id: str + title: str | None + status: str | None + source_path: str + raw: dict[str, Any] = field(default_factory=dict) + + +def register_path(repo_root: Path, kind: str) -> Path: + return repo_root / "registers" / f"{kind}.yaml" + + +def parse_register_file(path: Path, *, repo_root: Path) -> list[ParsedRegisterEntry]: + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError): + return [] + if not isinstance(data, dict): + return [] + kind = str(data.get("kind") or path.stem) + if kind not in SUPPORTED_REGISTER_KINDS: + return [] + entries = data.get("entries") or [] + if not isinstance(entries, list): + return [] + source_path = str(path.relative_to(repo_root)) + parsed: list[ParsedRegisterEntry] = [] + for item in entries: + if not isinstance(item, dict) or not item.get("id"): + continue + parsed.append( + ParsedRegisterEntry( + kind=kind, + id=str(item["id"]), + title=str(item["title"]) if item.get("title") is not None else None, + status=str(item["status"]) if item.get("status") is not None else None, + source_path=source_path, + raw=item, + ) + ) + return parsed + + +def iter_register_files(repo_root: Path) -> list[Path]: + directory = repo_root / "registers" + if not directory.is_dir(): + return [] + return sorted(path for path in directory.glob("*.yaml") if not path.name.startswith(".")) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/parse/workplan.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/parse/workplan.py new file mode 100644 index 00000000..eb1b2aac --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/parse/workplan.py @@ -0,0 +1,114 @@ +"""Workplan file parser (P0 extract — simplified from State Hub consistency_check).""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +_TASK_BLOCK_RE = re.compile(r"```task\s*\n(.*?)\n```", re.DOTALL) +_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+)$", re.MULTILINE) + + +def _parse_yaml_block(raw: str) -> dict[str, Any]: + try: + data = yaml.safe_load(raw) or {} + except yaml.YAMLError: + return {} + return data if isinstance(data, dict) else {} + + +def parse_frontmatter(text: str) -> tuple[dict[str, Any], str]: + if not text.startswith("---"): + return {}, text + parts = text.split("---", 2) + if len(parts) < 3: + return {}, text + return _parse_yaml_block(parts[1].strip()), parts[2] + + +@dataclass +class ParsedTask: + id: str | None + title: str | None + status: str | None + state_hub_task_id: str | None + raw: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class ParsedWorkplan: + path: str + id: str | None + title: str | None + status: str | None + state_hub_workstream_id: str | None + frontmatter: dict[str, Any] + tasks: list[ParsedTask] + + +def parse_task_blocks(body: str) -> list[ParsedTask]: + headings = [ + (m.start(), len(m.group(1)), m.group(2).strip()) + for m in _HEADING_RE.finditer(body) + ] + results: list[ParsedTask] = [] + for m in _TASK_BLOCK_RE.finditer(body): + meta = _parse_yaml_block(m.group(1).strip()) + prev = [(pos, level, text) for pos, level, text in headings if pos < m.start()] + title = meta.get("title") + if not title and prev: + title = prev[-1][2] + results.append( + ParsedTask( + id=str(meta["id"]) if meta.get("id") is not None else None, + title=str(title) if title else None, + status=str(meta["status"]) if meta.get("status") is not None else None, + state_hub_task_id=( + str(meta["state_hub_task_id"]).strip().strip('"') + if meta.get("state_hub_task_id") is not None + else None + ), + raw=meta, + ) + ) + return results + + +def parse_workplan_text(text: str, *, relative_path: str) -> ParsedWorkplan: + fm, body = parse_frontmatter(text) + return ParsedWorkplan( + path=relative_path, + id=str(fm["id"]) if fm.get("id") is not None else None, + title=str(fm["title"]) if fm.get("title") is not None else None, + status=str(fm["status"]) if fm.get("status") is not None else None, + state_hub_workstream_id=( + str(fm["state_hub_workstream_id"]).strip().strip('"') + if fm.get("state_hub_workstream_id") is not None + else None + ), + frontmatter=fm, + tasks=parse_task_blocks(body), + ) + + +def parse_workplan_file(path: Path, *, repo_root: Path) -> ParsedWorkplan: + text = path.read_text(encoding="utf-8") + rel = str(path.relative_to(repo_root)) + return parse_workplan_text(text, relative_path=rel) + + +def iter_workplan_files(repo_root: Path) -> list[Path]: + wp_dir = repo_root / "workplans" + if not wp_dir.is_dir(): + return [] + files: list[Path] = [] + for p in sorted(wp_dir.rglob("*.md")): + if p.name.startswith("."): + continue + # skip archived copies optionally still include them + files.append(p) + return files diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/prefix_registry.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/prefix_registry.py new file mode 100644 index 00000000..20568ef3 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/prefix_registry.py @@ -0,0 +1,112 @@ +"""Fleet workplan-prefix ownership and uniqueness (RMGR-WP-0004-T08).""" + +from __future__ import annotations + +from collections import defaultdict +from pathlib import Path +from typing import Any + +import yaml + +from repo_manager.standards import FLAVOR_MARKER_PREFIX, workplan_ids + +DEFAULT_REGISTRY = Path(__file__).resolve().parents[2] / "config" / "workplan-prefix-registry.yaml" + + +def load_registry(path: Path | None = None) -> dict[str, Any]: + target = path or DEFAULT_REGISTRY + data = yaml.safe_load(target.read_text(encoding="utf-8")) or {} + if not isinstance(data, dict): + return {} + return data + + +def _skip_dir(name: str) -> bool: + return name.startswith(".") or name in { + "node_modules", + "__pycache__", + ".venv", + "venv", + "archive", + } + + +def iter_repo_roots(root: Path) -> list[Path]: + root = root.resolve() + if (root / "workplans").is_dir() and (root / ".git").exists(): + return [root] + found: list[Path] = [] + if not root.is_dir(): + return found + for child in sorted(root.iterdir()): + if not child.is_dir() or _skip_dir(child.name): + continue + if (child / "workplans").is_dir(): + found.append(child) + return found + + +def scan_prefixes(root: Path, *, registry_path: Path | None = None) -> dict[str, Any]: + """Detect prefix sharing, identifier reuse, flavor-derived prefixes, number reuse. + + Detection only. Does not rewrite files or remediations. + """ + registry = load_registry(registry_path) + owners = {str(k): str(v) for k, v in (registry.get("owners") or {}).items()} + retired = {str(item.get("prefix")) for item in registry.get("retired") or [] if item.get("prefix")} + + by_prefix: dict[str, set[str]] = defaultdict(set) + by_id: dict[str, list[tuple[str, str]]] = defaultdict(list) + numbers: dict[tuple[str, str], list[str]] = defaultdict(list) + + repos_scanned = [] + for repo in iter_repo_roots(root): + repos_scanned.append(repo.name) + for rel, prefix, number in workplan_ids(repo): + by_prefix[prefix].add(repo.name) + ident = f"{prefix}-{number}" + by_id[ident].append((repo.name, rel)) + numbers[(repo.name, prefix)].append(number) + + shared = { + prefix: sorted(repos) + for prefix, repos in sorted(by_prefix.items()) + if len(repos) > 1 + } + reused = { + ident: [{"repo": repo, "path": path} for repo, path in entries] + for ident, entries in sorted(by_id.items()) + if len({repo for repo, _path in entries}) > 1 or len(entries) > 1 + } + flavor_derived = sorted(repo for repo in by_prefix.get(FLAVOR_MARKER_PREFIX, [])) + retired_in_use = { + prefix: sorted(repos) + for prefix, repos in by_prefix.items() + if prefix in retired + } + owner_mismatch = { + prefix: {"declared_owner": owners[prefix], "seen_in": sorted(repos)} + for prefix, repos in by_prefix.items() + if prefix in owners and any(repo != owners[prefix] for repo in repos) + } + + reused_numbers = [] + for (repo, prefix), nums in numbers.items(): + seen: set[str] = set() + for num in nums: + if num in seen: + reused_numbers.append({"repo": repo, "prefix": prefix, "number": num}) + seen.add(num) + + return { + "ok": not shared and not flavor_derived and not reused_numbers and not reused, + "root": str(root.resolve()), + "repos_scanned": repos_scanned, + "shared_prefixes": shared, + "reused_identifiers": reused, + "flavor_derived_prefix": flavor_derived, + "retired_in_use": retired_in_use, + "owner_mismatch": owner_mismatch, + "reused_numbers_in_repo": reused_numbers, + "detection_only": True, + } diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/projection_sync.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/projection_sync.py new file mode 100644 index 00000000..4254ff0e --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/projection_sync.py @@ -0,0 +1,267 @@ +"""Fast, guarded trigger for State Hub's forge-derived repository projection.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +import tempfile +import time +from pathlib import Path +from typing import Any + +import httpx + +from repo_manager.gitops import GitError, commit_paths, head_sha, push_ff +from repo_manager.identifiers import ensure_missing_work_record_identifiers + + +def _git(repo: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(["git", *args], cwd=repo, capture_output=True, text=True, check=False) + + +def _ahead_behind(repo: Path) -> dict[str, Any]: + upstream = _git(repo, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}") + if upstream.returncode != 0: + raise ValueError("current branch has no upstream; central can only derive pushed work") + counts = _git(repo, "rev-list", "--left-right", "--count", "@{u}...HEAD") + if counts.returncode != 0: + raise ValueError(counts.stderr.strip() or "could not compare HEAD with upstream") + try: + behind, ahead = (int(value) for value in counts.stdout.split()) + except ValueError as exc: + raise ValueError("could not parse Git ahead/behind state") from exc + return {"upstream": upstream.stdout.strip(), "behind": behind, "ahead": ahead} + + +def _dirty_authoritative_paths(repo: Path) -> list[str]: + result = _git(repo, "status", "--porcelain", "--", "workplans") + if result.returncode != 0: + raise ValueError(result.stderr.strip() or "could not inspect workplan files") + return [line[3:] for line in result.stdout.splitlines() if len(line) > 3] + + +def _pending_path(repo: Path) -> Path: + resolved = _git( + repo, "rev-parse", "--git-path", "repo-manager/pending-work-record-projection.json" + ) + if resolved.returncode != 0 or not resolved.stdout.strip(): + raise ValueError("could not resolve the repository-local Git state directory") + path = Path(resolved.stdout.strip()) + return path if path.is_absolute() else repo / path + + +def _write_pending(repo: Path, payload: dict[str, Any]) -> Path: + path = _pending_path(repo) + path.parent.mkdir(parents=True, exist_ok=True) + temporary: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", delete=False + ) as handle: + temporary = Path(handle.name) + handle.write(json.dumps(payload, indent=2) + "\n") + temporary.replace(path) + temporary = None + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + return path + + +def sync_repository_projection( + repo_root: Path, + *, + api_base: str, + repo_slug: str | None = None, + expected_instance_label: str | None = "railiance01", + commit_identifiers: bool = True, + push: bool = False, + acknowledge_retirements: bool = False, + queue_on_unavailable: bool = True, + transport: httpx.BaseTransport | None = None, +) -> dict[str, Any]: + """Assign missing IDs, prove the commit is pushed, then reconcile once.""" + started = time.perf_counter() + repo_root = repo_root.resolve() + slug = repo_slug or repo_root.name + + dirty = _dirty_authoritative_paths(repo_root) + if dirty: + return { + "ok": False, + "status": "pending_commit", + "error": "authoritative workplan files have uncommitted changes", + "dirty_paths": dirty, + } + + assignments = ensure_missing_work_record_identifiers(repo_root) + if assignments["files_changed"] and commit_identifiers: + try: + commit_paths( + repo_root, + assignments["files_changed"], + message=( + "repo.work.assign_missing_identifiers\n\n" + "source: repo-manager\nreason: deterministic projection registration\n" + ), + ) + except GitError as exc: + return {"ok": False, "status": "failed", "error": str(exc), "identifiers": assignments} + + dirty = _dirty_authoritative_paths(repo_root) + if dirty: + return { + "ok": False, + "status": "pending_commit", + "error": "authoritative workplan files have uncommitted changes", + "dirty_paths": dirty, + "identifiers": assignments, + } + + try: + git = _ahead_behind(repo_root) + except ValueError as exc: + return {"ok": False, "status": "failed", "error": str(exc), "identifiers": assignments} + if git["behind"]: + return { + "ok": False, + "status": "behind_upstream", + "error": "branch is behind upstream; reconcile Git before projection", + "git": git, + } + if git["ahead"] and push: + pushed, message = push_ff(repo_root) + if not pushed: + return { + "ok": False, + "status": "push_failed", + "error": message, + "git": git, + } + git = _ahead_behind(repo_root) + if git["ahead"]: + return { + "ok": True, + "status": "pending_push", + "message": "commit is not yet visible to the authoritative forge", + "git": git, + "identifiers": assignments, + } + + commit = head_sha(repo_root) + if not commit: + return {"ok": False, "status": "failed", "error": "repository HEAD is unavailable"} + + request_path = f"/repos/{slug}/work-record-projection/reconcile" + request_body = { + "expected_commit": commit, + "acknowledge_retirements": acknowledge_retirements, + } + # The review acknowledgement changes request semantics at the same commit. + # Bind the key to the complete POST, retaining identical keys for retries. + request_digest = hashlib.sha256(json.dumps( + {"method": "POST", "path": request_path, "body": request_body}, + sort_keys=True, separators=(",", ":"), + ).encode()).hexdigest() + pending = { + "schema": "repo-manager.pending-work-record-projection.v1", + "repo_slug": slug, + "expected_commit": commit, + "api_base": api_base.rstrip("/"), + "acknowledge_retirements": acknowledge_retirements, + "expected_instance_label": expected_instance_label, + } + headers = { + "Idempotency-Key": f"rmgr-projection:v2:{request_digest}", + "X-StateHub-Source-Agent": "repo-manager", + } + try: + with httpx.Client( + base_url=api_base.rstrip("/"), + timeout=httpx.Timeout(120.0, connect=5.0), + follow_redirects=True, + transport=transport, + ) as client: + health = client.get("/state/health") + health.raise_for_status() + identity = health.json() + identity_ok = identity.get("instance_role") == "primary" and ( + expected_instance_label is None + or identity.get("instance_label") == expected_instance_label + ) + if not identity_ok: + if not queue_on_unavailable: + return { + "ok": False, + "status": "wrong_instance", + "instance": identity, + } + path = _write_pending(repo_root, {**pending, "reason": "wrong_instance"}) + return { + "ok": True, + "status": "queued", + "reason": "wrong_instance", + "instance": identity, + "pending_path": str(path), + } + response = client.post( + request_path, + json=request_body, + headers=headers, + ) + response.raise_for_status() + receipt = response.json() + except (httpx.ConnectError, httpx.TimeoutException, httpx.NetworkError) as exc: + if not queue_on_unavailable: + return {"ok": False, "status": "unavailable", "error": str(exc)} + path = _write_pending(repo_root, {**pending, "reason": "hub_unavailable"}) + return { + "ok": True, + "status": "queued", + "reason": "hub_unavailable", + "pending_path": str(path), + } + except httpx.HTTPStatusError as exc: + detail: Any + try: + detail = exc.response.json() + except ValueError: + detail = exc.response.text[:500] + return { + "ok": False, + "status": "rejected", + "http_status": exc.response.status_code, + "error": detail, + } + + # A successful HTTP response alone is not an exact-commit primary receipt. + outcome = receipt.get("outcome") if isinstance(receipt, dict) else None + if not ( + isinstance(receipt, dict) + and receipt.get("schema") == "state-hub.repository-projection-reconcile.v1" + and receipt.get("instance_role") == "primary" + and receipt.get("instance_label") == identity.get("instance_label") + and receipt.get("expected_commit") == commit + and receipt.get("derived_commit") == commit + and isinstance(outcome, dict) + and outcome.get("repo_slug") == slug + and outcome.get("commit") == commit + and outcome.get("status") in {"applied", "noop", "refused"} + ): + return {"ok": False, "status": "invalid_receipt", "repo_slug": slug, + "commit": commit, "error": "response does not bind the requested primary/repository/commit"} + outcome_status = outcome["status"] + if outcome_status in {"applied", "noop"}: + _pending_path(repo_root).unlink(missing_ok=True) + return { + "ok": outcome_status in {"applied", "noop"}, + "status": outcome_status, + "repo_slug": slug, + "commit": commit, + "requests": 2, + "elapsed_ms": round((time.perf_counter() - started) * 1000, 1), + "git": git, + "identifiers": assignments, + "receipt": receipt, + } diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/provenance.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/provenance.py new file mode 100644 index 00000000..b20a06e8 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/provenance.py @@ -0,0 +1,135 @@ +"""Install and report on coding-assistant commit provenance.""" + +from __future__ import annotations + +import re +import subprocess +from collections import defaultdict +from pathlib import Path +from typing import Any + +import yaml + +_TRAILER_RE = re.compile(r"^(Assistant(?:-Model|-Process|-Session)?):\s*(.+)$", re.MULTILINE) + + +def install_hook(hooks_path: Path) -> dict[str, Any]: + """Configure the user's Git installation to use the governed hook directory.""" + hooks_path = hooks_path.expanduser().resolve() + hook = hooks_path / "prepare-commit-msg" + if not hook.is_file(): + return {"ok": False, "error": f"missing hook: {hook}"} + if not hook.stat().st_mode & 0o111: + return {"ok": False, "error": f"hook is not executable: {hook}"} + completed = subprocess.run( + ["git", "config", "--global", "core.hooksPath", str(hooks_path)], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode: + return {"ok": False, "error": completed.stderr.strip() or "git config failed"} + return {"ok": True, "hooks_path": str(hooks_path), "hook": str(hook)} + + +def _cutover(repo_root: Path) -> str | None: + path = repo_root / "config" / "assistant-provenance.yaml" + if not path.is_file(): + return None + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + value = data.get("cutover_commit") if isinstance(data, dict) else None + return str(value) if value else None + + +def _commits(repo_root: Path, rev: str, max_count: int | None) -> list[dict[str, Any]]: + command = ["git", "log", "--reverse", "--format=%H%x1f%aI%x1f%an%x1f%ae%x1f%B%x1e"] + if max_count: + command.append(f"--max-count={max_count}") + command.append(rev) + completed = subprocess.run(command, cwd=repo_root, check=True, capture_output=True, text=True) + commits: list[dict[str, Any]] = [] + for raw in completed.stdout.split("\x1e"): + raw = raw.strip("\n") + if not raw: + continue + parts = raw.split("\x1f", 4) + if len(parts) != 5: + continue + sha, authored_at, author, email, body = parts + trailers = {key: value.strip() for key, value in _TRAILER_RE.findall(body)} + commits.append( + { + "sha": sha, + "authored_at": authored_at, + "author": author, + "email": email, + "assistant": trailers.get("Assistant"), + "model": trailers.get("Assistant-Model"), + "process": trailers.get("Assistant-Process"), + "session": trailers.get("Assistant-Session"), + } + ) + return commits + + +def assistant_report(repo_root: Path, *, rev: str = "HEAD", max_count: int | None = None) -> dict[str, Any]: + """Derive assistant activity and interleaved-session signals from Git alone.""" + repo_root = repo_root.resolve() + commits = _commits(repo_root, rev, max_count) + cutover = _cutover(repo_root) + cutover_seen = False + assistants: dict[str, dict[str, Any]] = defaultdict( + lambda: {"commit_count": 0, "models": set(), "sessions": set(), "processes": set()} + ) + unattributed = {"before_cutover": 0, "after_cutover": 0, "known_automation": 0} + session_sequence: list[str] = [] + + for commit in commits: + if cutover and commit["sha"].startswith(cutover): + cutover_seen = True + assistant = commit["assistant"] + if assistant: + entry = assistants[assistant] + entry["commit_count"] += 1 + for field, bucket in (("model", "models"), ("session", "sessions"), ("process", "processes")): + if commit[field]: + entry[bucket].add(commit[field]) + if commit["session"]: + session_sequence.append(commit["session"]) + elif commit["author"] == "custodian-sync" or commit["email"] == "custodian-sync@railiance.local": + unattributed["known_automation"] += 1 + elif cutover_seen: + unattributed["after_cutover"] += 1 + else: + unattributed["before_cutover"] += 1 + + interleaved: set[tuple[str, str]] = set() + positions: dict[str, list[int]] = defaultdict(list) + for position, session in enumerate(session_sequence): + positions[session].append(position) + sessions = sorted(positions) + for index, first in enumerate(sessions): + for second in sessions[index + 1 :]: + merged = [session_sequence[pos] for pos in sorted(positions[first] + positions[second])] + compressed = [value for pos, value in enumerate(merged) if pos == 0 or value != merged[pos - 1]] + if len(compressed) >= 3: + interleaved.add((first, second)) + + return { + "ok": True, + "repo": str(repo_root), + "revision": rev, + "cutover_commit": cutover, + "commit_count": len(commits), + "assistants": { + name: { + "commit_count": item["commit_count"], + "models": sorted(item["models"]), + "sessions": sorted(item["sessions"]), + "processes": sorted(item["processes"]), + } + for name, item in sorted(assistants.items()) + }, + "interleaved_sessions": [list(pair) for pair in sorted(interleaved)], + "unattributed": unattributed, + } diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/record_identity.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/record_identity.py new file mode 100644 index 00000000..82c54a79 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/record_identity.py @@ -0,0 +1,139 @@ +"""Work-record identity validation and duplicate-source reconciliation.""" + +from __future__ import annotations + +import re +from collections import defaultdict +from pathlib import Path +from typing import Any + +import yaml + +from repo_manager.parse.record import iter_record_files, parse_record_file +from repo_manager.parse.workplan import iter_workplan_files, parse_workplan_file + +DEFAULT_KIND_REGISTRY = Path(__file__).resolve().parents[2] / "config" / "work-record-types.yaml" +CANON_WORK_RECORD_TYPES = "the-custodian/canon/standards/work-record-types.yaml" + + +def load_kind_registry(path: Path | None = None) -> dict[str, Any]: + data = yaml.safe_load((path or DEFAULT_KIND_REGISTRY).read_text(encoding="utf-8")) or {} + if not isinstance(data, dict) or not isinstance(data.get("kinds"), list): + raise TypeError("invalid work-record kind registry") + return data + + +def classify_record_id( + kind: str, + identifier: str, + *, + parent_id: str | None = None, + registry_path: Path | None = None, +) -> str | None: + """Return canonical/grandfathered when the canon registry accepts an id.""" + registry = load_kind_registry(registry_path) + entry = next((item for item in registry["kinds"] if item.get("kind") == kind), None) + if not isinstance(entry, dict): + return None + if any(re.fullmatch(pattern, identifier) for pattern in entry.get("id_patterns") or []): + return "canonical" + for legacy in entry.get("legacy_patterns") or []: + if legacy.get("grandfathered") and re.fullmatch( + str(legacy.get("pattern") or ""), identifier + ): + return "grandfathered" + # Historical task blocks may use a local TNN label. Accept these only + # under an explicitly enumerated parent workplan; the parent scope is + # part of identity and no source id/UUID is rewritten. + if parent_id: + for legacy in entry.get("scoped_legacy_patterns") or []: + if ( + legacy.get("grandfathered") + and re.fullmatch(str(legacy.get("pattern") or ""), identifier) + and re.fullmatch(str(legacy.get("parent_pattern") or ""), parent_id) + ): + return "grandfathered" + return None + + +def scan_record_identities(repo_root: Path, *, registry_path: Path | None = None) -> dict[str, Any]: + """Describe invalid ids, benign duplicate sources, and hard collisions.""" + occurrences: dict[tuple[str, str, str | None], list[dict[str, Any]]] = defaultdict(list) + invalid: list[dict[str, str]] = [] + + def add( + kind: str, + identifier: str | None, + record_uuid: str | None, + source: str, + *, + parent_id: str | None = None, + ) -> None: + if not identifier: + return + classification = classify_record_id( + kind, identifier, parent_id=parent_id, registry_path=registry_path + ) + item = { + "kind": kind, + "id": identifier, + "uuid": record_uuid, + "source": source, + **({"parent_id": parent_id} if parent_id else {}), + } + # Only the explicitly scoped local aliases use parent scope. Qualified + # canonical IDs remain globally collision checked. + local_alias = ( + kind == "task" + and parent_id is not None + and re.fullmatch(r"T[0-9]{2}", identifier) is not None + and classification == "grandfathered" + ) + occurrences[(kind, identifier, parent_id if local_alias else None)].append(item) + if classification is None or (local_alias and not record_uuid): + invalid.append({"kind": kind, "id": identifier, "source": source}) + + for path in iter_workplan_files(repo_root): + workplan = parse_workplan_file(path, repo_root=repo_root) + add("workplan", workplan.id, workplan.state_hub_workstream_id, workplan.path) + for number, task in enumerate(workplan.tasks, start=1): + add( + "task", + task.id, + task.state_hub_task_id, + f"{workplan.path}#task-block-{number}", + parent_id=workplan.id, + ) + for path in iter_record_files(repo_root): + for number, record in enumerate(parse_record_file(path, repo_root=repo_root), start=1): + add(record.kind, record.id, record.uuid, f"{record.source_path}#record-block-{number}") + + duplicates: list[dict[str, Any]] = [] + collisions: list[dict[str, Any]] = [] + for (kind, identifier, _parent_id), items in sorted( + occurrences.items(), key=lambda pair: (pair[0][0], pair[0][1], pair[0][2] or "") + ): + if len(items) < 2: + continue + uuids = {item["uuid"] for item in items} + result = { + "kind": kind, + "id": identifier, + "uuid": items[0]["uuid"] if len(uuids) == 1 else None, + "sources": sorted(item["source"] for item in items), + } + if items[0].get("parent_id"): + result["parent_id"] = items[0]["parent_id"] + if len(uuids) == 1 and None not in uuids: + duplicates.append(result) + else: + result["uuids"] = sorted(str(value) for value in uuids) + collisions.append(result) + + return { + "ok": not invalid and not collisions, + "invalid_identifiers": invalid, + "duplicate_source_occurrences": duplicates, + "identity_collisions": collisions, + "canon": CANON_WORK_RECORD_TYPES, + } diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/registrar.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/registrar.py new file mode 100644 index 00000000..0b21f774 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/registrar.py @@ -0,0 +1,30 @@ +"""Interim single-writer for hub identifiers (ADR-007 decision 2). + +Until deterministic derivation (RMGR-WP-0005-T03) ships, exactly one +instance may mint ``state_hub_workstream_id`` / ``state_hub_task_id`` +into repository files. Other instances may read and project; they must +not write new hub primary keys into git-tracked files. + +The registrar is the automated production instance (hostname prefix +``railiance``). Override with ``STATEHUB_REGISTRAR=1`` or ``=0``. +""" + +from __future__ import annotations + +import os +import socket + +_TRUE = frozenset({"1", "true", "yes", "on"}) +_FALSE = frozenset({"0", "false", "no", "off"}) +REGISTRAR_HOST_PREFIX = "railiance" + + +def is_identifier_registrar(*, hostname: str | None = None) -> bool: + """Return True if this process may mint hub IDs into repository files.""" + raw = os.environ.get("STATEHUB_REGISTRAR", "").strip().lower() + if raw in _TRUE: + return True + if raw in _FALSE: + return False + host = (hostname if hostname is not None else socket.gethostname()).strip().lower() + return host.startswith(REGISTRAR_HOST_PREFIX) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/repository_publisher.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/repository_publisher.py new file mode 100644 index 00000000..caebe835 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/repository_publisher.py @@ -0,0 +1,723 @@ +"""Publish repository classifications through the frozen ``port.repo`` contract.""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import re +import subprocess +import threading +from collections import OrderedDict +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any +from urllib.parse import quote, urlparse +from uuid import UUID + +import httpx +import yaml + +from repo_manager import __version__ +from repo_manager.classification import ClassificationError, require_valid_classification +from repo_manager.observe import load_classification +from repo_manager.time import format_utc, utc_now + +CONTRACT_ID = "helixforge.repository-classification-projection" +CONTRACT_VERSION = "1.0.0" +CLASSIFICATION_CONTRACT_ID = "RMGR-CONTRACT-CLASSIFICATION-0001" +CLASSIFICATION_CONTRACT_VERSION = "1.0" +REGISTRY_SCHEMA = "repo-manager.repository-registry.v1" +LIFECYCLES = frozenset({"active", "archived", "retired"}) +_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") + + +class RegistryError(ValueError): + """The operator-owned repository registry cannot be used safely.""" + + +class ProjectionCursorError(ValueError): + """A projection page cursor is malformed, stale, or has been tampered with.""" + + +@dataclass(frozen=True, slots=True) +class RepositoryRegistration: + repository_id: str + slug: str + lifecycle: str + path: Path | None = None + forgejo_repository: str | None = None + + +@dataclass(frozen=True, slots=True) +class ClassificationSnapshot: + snapshot_id: str + generated_at: str + source_revision: str + repositories: tuple[dict[str, Any], ...] + diagnostics: tuple[dict[str, Any], ...] + + +def load_repository_registry(path: Path) -> tuple[RepositoryRegistration, ...]: + """Load the private operator registry used to join UUIDs to local checkouts.""" + try: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError) as exc: + raise RegistryError(f"cannot read repository registry: {exc}") from exc + if not isinstance(raw, dict) or raw.get("schema") != REGISTRY_SCHEMA: + raise RegistryError(f"repository registry must declare schema {REGISTRY_SCHEMA!r}") + rows = raw.get("repositories") + if not isinstance(rows, list): + raise RegistryError("repository registry `repositories` must be a list") + + registrations: list[RepositoryRegistration] = [] + ids: set[str] = set() + slugs: set[str] = set() + for number, row in enumerate(rows, start=1): + if not isinstance(row, dict): + raise RegistryError(f"repository registry row {number} must be an object") + try: + repository_id = str(UUID(str(row["repository_id"]))) + slug = str(row["slug"]) + lifecycle = str(row["lifecycle"]) + raw_path = row.get("path") + raw_forgejo = row.get("forgejo_repository") + except (KeyError, TypeError, ValueError) as exc: + raise RegistryError(f"invalid repository registry row {number}: {exc}") from exc + if repository_id in ids: + raise RegistryError(f"duplicate repository_id {repository_id}") + if slug in slugs: + raise RegistryError(f"duplicate repository slug {slug!r}") + if not _SLUG_RE.fullmatch(slug) or len(slug) > 120: + raise RegistryError(f"invalid repository slug {slug!r}") + if lifecycle not in LIFECYCLES: + raise RegistryError( + f"repository {slug!r} lifecycle must be one of {', '.join(sorted(LIFECYCLES))}" + ) + if (raw_path is None) == (raw_forgejo is None): + raise RegistryError( + f"repository {slug!r} must declare exactly one of path or forgejo_repository" + ) + repo_path = Path(str(raw_path)) if raw_path is not None else None + forgejo_repository = str(raw_forgejo) if raw_forgejo is not None else None + if repo_path is not None and not repo_path.is_absolute(): + raise RegistryError(f"repository {slug!r} path must be absolute") + if forgejo_repository is not None and forgejo_repository != f"coulomb/{slug}": + raise RegistryError( + f"repository {slug!r} Forgejo identity must be exactly 'coulomb/{slug}'" + ) + ids.add(repository_id) + slugs.add(slug) + registrations.append( + RepositoryRegistration( + repository_id=repository_id, + slug=slug, + lifecycle=lifecycle, + path=repo_path, + forgejo_repository=forgejo_repository, + ) + ) + return tuple(sorted(registrations, key=lambda item: item.repository_id)) + + +def import_state_hub_registry( + fleet_root: Path, + *, + api_base: str, + timeout_seconds: float = 10.0, + workers: int = 4, + retries: int = 2, + source: str = "local", + transport: httpx.BaseTransport | None = None, +) -> dict[str, Any]: + """Bootstrap Repo Manager identities from the retiring authoritative registrar. + + The bulk State Hub repository route is intentionally not used: direct + identity reads keep the migration bounded and let independent repositories + complete concurrently. Host paths come only from the operator-selected + local fleet root; remote State Hub path values are never trusted. + """ + parsed = urlparse(api_base) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("State Hub API base must be an absolute HTTP(S) origin") + if timeout_seconds <= 0 or workers < 1 or retries < 0: + raise ValueError("timeout_seconds/workers must be positive and retries non-negative") + if source not in {"local", "forgejo"}: + raise ValueError("registry import source must be 'local' or 'forgejo'") + paths = _discover_classified_repositories(fleet_root) + if not paths: + raise ValueError("fleet root contains no classified Git repositories") + with httpx.Client( + base_url=api_base.rstrip("/"), + timeout=httpx.Timeout(timeout_seconds), + follow_redirects=True, + transport=transport, + ) as client: + try: + health = client.get("/state/health") + health.raise_for_status() + identity = health.json() + except (httpx.HTTPError, ValueError) as exc: + raise ValueError(f"State Hub registry source is unavailable: {exc}") from exc + if not isinstance(identity, dict): + raise RegistryError("State Hub health response is not a JSON object") + if identity.get("instance_role") != "primary": + raise ValueError("State Hub registry import requires an instance_role=primary source") + + def fetch(path: Path) -> tuple[Path, dict[str, Any]]: + for attempt in range(retries + 1): + try: + response = client.get(f"/repos/{path.name}") + response.raise_for_status() + payload = response.json() + if not isinstance(payload, dict): + raise RegistryError("repository response is not a JSON object") + return path, payload + except (httpx.TimeoutException, httpx.NetworkError): + if attempt == retries: + raise + raise AssertionError("unreachable retry loop") + + rows: list[dict[str, Any]] = [] + diagnostics: list[dict[str, str]] = [] + with ThreadPoolExecutor(max_workers=workers) as executor: + pending = {executor.submit(fetch, path): path for path in paths} + for future in as_completed(pending): + path = pending[future] + try: + local_path, payload = future.result() + repository_id = str(UUID(str(payload["id"]))) + slug = str(payload["slug"]) + lifecycle = str(payload.get("status") or "active") + if lifecycle not in LIFECYCLES: + raise ValueError(f"unsupported lifecycle {lifecycle!r}") + if not _SLUG_RE.fullmatch(slug): + raise ValueError(f"invalid canonical slug {slug!r}") + except httpx.HTTPStatusError as exc: + missing = exc.response.status_code == 404 + diagnostics.append( + { + "severity": "warning" if missing else "error", + "code": ( + "registry_import.unregistered_checkout" + if missing + else "registry_import.repository_failed" + ), + "repository": path.name, + "message": str(exc)[:500], + } + ) + except (httpx.HTTPError, KeyError, TypeError, ValueError) as exc: + diagnostics.append( + { + "severity": "error", + "code": "registry_import.repository_failed", + "repository": path.name, + "message": str(exc)[:500], + } + ) + else: + rows.append( + { + "repository_id": repository_id, + "slug": slug, + "lifecycle": lifecycle, + **( + {"path": str(local_path.resolve())} + if source == "local" + else {"forgejo_repository": f"coulomb/{slug}"} + ), + } + ) + + rows.sort(key=lambda row: row["repository_id"]) + diagnostics.sort(key=lambda row: row["repository"]) + ids = [row["repository_id"] for row in rows] + slugs = [row["slug"] for row in rows] + if len(ids) != len(set(ids)) or len(slugs) != len(set(slugs)): + diagnostics.append( + { + "severity": "error", + "code": "registry_import.identity_collision", + "repository": "fleet", + "message": "State Hub returned duplicate repository UUIDs or canonical slugs", + } + ) + return { + "ok": not any(item["severity"] == "error" for item in diagnostics), + "schema": REGISTRY_SCHEMA, + "source": { + "system": "state-hub", + "instance_role": identity.get("instance_role"), + "instance_label": identity.get("instance_label"), + }, + "generated_at": format_utc(utc_now()), + "repositories": rows, + "diagnostics": diagnostics, + } + + +class ForgejoClassificationClient: + """Read exact default-branch classifications from the Forgejo API.""" + + def __init__( + self, + base_url: str, + *, + token: str | None = None, + timeout_seconds: float = 10.0, + retries: int = 2, + transport: httpx.BaseTransport | None = None, + ) -> None: + parsed = urlparse(base_url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("Forgejo base URL must be an absolute HTTP(S) origin") + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError("Forgejo base URL must not contain credentials, query, or fragment") + if timeout_seconds <= 0 or retries < 0: + raise ValueError("Forgejo timeout must be positive and retries non-negative") + headers = {"Authorization": f"token {token}"} if token else None + self._client = httpx.Client( + base_url=base_url.rstrip("/"), + headers=headers, + timeout=httpx.Timeout(timeout_seconds), + follow_redirects=True, + transport=transport, + ) + self.retries = retries + + def observe( + self, registration: RepositoryRegistration, *, observed_at: str + ) -> dict[str, Any]: + repository = registration.forgejo_repository + if repository is None: + raise ValueError("registration has no Forgejo repository source") + owner, slug = repository.split("/", 1) + owner_path = quote(owner, safe="") + slug_path = quote(slug, safe="") + prefix = f"/api/v1/repos/{owner_path}/{slug_path}" + metadata = self._get_object(prefix) + if metadata.get("full_name") != repository: + raise ValueError("Forgejo repository identity does not match the registry") + branch = metadata.get("default_branch") + if not isinstance(branch, str) or not branch or branch.startswith("/"): + raise ValueError("Forgejo default branch is unavailable") + branch_payload = self._get_object(f"{prefix}/branches/{quote(branch, safe='')}") + commit = branch_payload.get("commit") + head_sha = commit.get("id") if isinstance(commit, dict) else None + if not isinstance(head_sha, str) or not re.fullmatch(r"[a-f0-9]{40,64}", head_sha): + raise ValueError("Forgejo default-branch revision is invalid") + content = self._get_object( + f"{prefix}/contents/.repo-classification.yaml", + params={"ref": head_sha}, + ) + if content.get("type", "file") != "file" or content.get("encoding") != "base64": + raise ValueError("Forgejo classification content has an unsupported representation") + encoded = content.get("content") + if not isinstance(encoded, str): + raise TypeError("Forgejo classification content is missing") + try: + raw = base64.b64decode(encoded, validate=True) + except (ValueError, TypeError) as exc: + raise ValueError("Forgejo classification content is not valid base64") from exc + if not raw or len(raw) > 131072: + raise ValueError("Forgejo classification content size is outside the safe bound") + classification = _classification_from_bytes(raw) + normalized = _normalize_classification(classification) + return { + "operation": "upsert", + "repository_id": registration.repository_id, + "slug": registration.slug, + "lifecycle": registration.lifecycle, + "classification": normalized, + "revision": { + "head_sha": head_sha, + "source_fingerprint": hashlib.sha256(raw).hexdigest(), + "observed_at": observed_at, + }, + } + + def close(self) -> None: + self._client.close() + + def _get_object( + self, path: str, *, params: dict[str, str] | None = None + ) -> dict[str, Any]: + for attempt in range(self.retries + 1): + try: + response = self._client.get(path, params=params) + response.raise_for_status() + payload = response.json() + if not isinstance(payload, dict): + raise TypeError("Forgejo response is not a JSON object") + return payload + except (httpx.TimeoutException, httpx.NetworkError): + if attempt == self.retries: + raise ValueError(f"Forgejo request failed after retries: {path}") from None + except httpx.HTTPStatusError as exc: + raise ValueError( + f"Forgejo request returned HTTP {exc.response.status_code}: {path}" + ) from exc + except ValueError: + raise + raise AssertionError("unreachable retry loop") + + +def build_classification_snapshot( + registrations: tuple[RepositoryRegistration, ...], + *, + observed_at: datetime | None = None, + forgejo_client: ForgejoClassificationClient | None = None, + remote_workers: int = 8, +) -> ClassificationSnapshot: + """Observe one atomic, full fleet generation without publishing host-local paths.""" + instant = observed_at or utc_now() + generated_at = format_utc(instant) + repositories: list[dict[str, Any]] = [] + diagnostics: list[dict[str, Any]] = [] + + def observe(registration: RepositoryRegistration) -> dict[str, Any]: + if registration.path is not None: + return _observe_registration(registration, observed_at=generated_at) + if forgejo_client is None: + raise ValueError("Forgejo registry source requires a configured Forgejo client") + return forgejo_client.observe(registration, observed_at=generated_at) + + ordered = sorted(registrations, key=lambda item: item.repository_id) + local = [registration for registration in ordered if registration.path is not None] + remote = [registration for registration in ordered if registration.forgejo_repository is not None] + observations: dict[str, dict[str, Any]] = {} + errors: dict[str, Exception] = {} + for registration in local: + try: + observations[registration.repository_id] = observe(registration) + except (ClassificationError, OSError, ValueError) as exc: + errors[registration.repository_id] = exc + if remote: + if remote_workers < 1: + raise ValueError("remote_workers must be positive") + with ThreadPoolExecutor(max_workers=remote_workers) as executor: + pending = {executor.submit(observe, registration): registration for registration in remote} + for future in as_completed(pending): + registration = pending[future] + try: + observations[registration.repository_id] = future.result() + except (ClassificationError, OSError, TypeError, ValueError) as exc: + errors[registration.repository_id] = exc + + for registration in ordered: + if registration.repository_id in errors: + exc = errors[registration.repository_id] + diagnostics.append( + { + "severity": "error", + "code": "repo_projection.source_invalid", + "message": f"{registration.slug}: {str(exc)[:900]}", + "repository_id": registration.repository_id, + } + ) + elif registration.repository_id in observations: + repositories.append(observations[registration.repository_id]) + + source_revision = _sha256( + { + "registry_schema": REGISTRY_SCHEMA, + "repositories": repositories, + "diagnostics": diagnostics, + } + ) + snapshot_id = _sha256( + { + "contract_id": CONTRACT_ID, + "contract_version": CONTRACT_VERSION, + "generated_at": generated_at, + "source_revision": source_revision, + } + ) + return ClassificationSnapshot( + snapshot_id=snapshot_id, + generated_at=generated_at, + source_revision=source_revision, + repositories=tuple(repositories), + diagnostics=tuple(diagnostics), + ) + + +def projection_page( + snapshot: ClassificationSnapshot, + *, + offset: int, + page_size: int, + cursor_secret: bytes, + page_cursor: str | None, +) -> dict[str, Any]: + if not 1 <= page_size <= 500: + raise ValueError("page_size must be between 1 and 500") + if offset < 0 or offset > len(snapshot.repositories): + raise ProjectionCursorError("projection cursor offset exceeds the snapshot") + page_rows = snapshot.repositories[offset : offset + page_size] + next_offset = offset + len(page_rows) + final_page = next_offset >= len(snapshot.repositories) + next_cursor = None + if not final_page: + next_cursor = encode_projection_cursor( + snapshot_id=snapshot.snapshot_id, + offset=next_offset, + page_size=page_size, + secret=cursor_secret, + ) + return { + "contract_id": CONTRACT_ID, + "contract_version": CONTRACT_VERSION, + "source": { + "system": "repo-manager", + "classification_contract_id": CLASSIFICATION_CONTRACT_ID, + "classification_contract_version": CLASSIFICATION_CONTRACT_VERSION, + "producer_version": __version__, + }, + "snapshot": { + "snapshot_id": snapshot.snapshot_id, + "mode": "full", + "generated_at": snapshot.generated_at, + "source_revision": snapshot.source_revision, + "page_cursor": page_cursor, + "next_cursor": next_cursor, + "final_page": final_page, + "total_repository_count": len(snapshot.repositories), + }, + "repositories": list(page_rows), + "diagnostics": list(snapshot.diagnostics) if offset == 0 else [], + } + + +class ClassificationPublisher: + """Bounded in-memory holder for in-flight, immutable snapshot transfers.""" + + def __init__( + self, + registry_path: Path, + *, + cursor_secret: str | bytes, + page_size: int = 100, + max_snapshots: int = 4, + forgejo_base_url: str | None = None, + forgejo_token: str | None = None, + forgejo_timeout_seconds: float = 10.0, + forgejo_retries: int = 2, + remote_workers: int = 8, + forgejo_transport: httpx.BaseTransport | None = None, + now: Callable[[], datetime] = utc_now, + ) -> None: + secret = cursor_secret.encode() if isinstance(cursor_secret, str) else cursor_secret + if len(secret) < 32: + raise ValueError("cursor_secret must contain at least 32 bytes") + if not 1 <= page_size <= 500: + raise ValueError("page_size must be between 1 and 500") + if max_snapshots < 1: + raise ValueError("max_snapshots must be positive") + if remote_workers < 1: + raise ValueError("remote_workers must be positive") + self.registry_path = registry_path + self.cursor_secret = secret + self.page_size = page_size + self.max_snapshots = max_snapshots + self.remote_workers = remote_workers + self.now = now + self._forgejo_client = ( + ForgejoClassificationClient( + forgejo_base_url, + token=forgejo_token, + timeout_seconds=forgejo_timeout_seconds, + retries=forgejo_retries, + transport=forgejo_transport, + ) + if forgejo_base_url is not None + else None + ) + self._snapshots: OrderedDict[str, ClassificationSnapshot] = OrderedDict() + self._lock = threading.Lock() + + def fetch_page(self, cursor: str | None = None) -> dict[str, Any]: + with self._lock: + return self._fetch_page(cursor) + + def _fetch_page(self, cursor: str | None = None) -> dict[str, Any]: + if cursor is None: + snapshot = build_classification_snapshot( + load_repository_registry(self.registry_path), + observed_at=self.now(), + forgejo_client=self._forgejo_client, + remote_workers=self.remote_workers, + ) + self._snapshots[snapshot.snapshot_id] = snapshot + self._snapshots.move_to_end(snapshot.snapshot_id) + while len(self._snapshots) > self.max_snapshots: + self._snapshots.popitem(last=False) + return projection_page( + snapshot, + offset=0, + page_size=self.page_size, + cursor_secret=self.cursor_secret, + page_cursor=None, + ) + + cursor_data = decode_projection_cursor(cursor, secret=self.cursor_secret) + snapshot = self._snapshots.get(cursor_data["snapshot_id"]) + if snapshot is None: + raise ProjectionCursorError("projection cursor snapshot has expired") + self._snapshots.move_to_end(snapshot.snapshot_id) + return projection_page( + snapshot, + offset=cursor_data["offset"], + page_size=cursor_data["page_size"], + cursor_secret=self.cursor_secret, + page_cursor=cursor, + ) + + def close(self) -> None: + if self._forgejo_client is not None: + self._forgejo_client.close() + + +def encode_projection_cursor( + *, snapshot_id: str, offset: int, page_size: int, secret: bytes +) -> str: + payload = json.dumps( + {"snapshot_id": snapshot_id, "offset": offset, "page_size": page_size}, + separators=(",", ":"), + sort_keys=True, + ).encode() + signature = hmac.new(secret, payload, hashlib.sha256).digest() + return f"{_b64encode(payload)}.{_b64encode(signature)}" + + +def decode_projection_cursor(cursor: str, *, secret: bytes) -> dict[str, Any]: + try: + encoded_payload, encoded_signature = cursor.split(".", 1) + payload = _b64decode(encoded_payload) + signature = _b64decode(encoded_signature) + expected = hmac.new(secret, payload, hashlib.sha256).digest() + if not hmac.compare_digest(signature, expected): + raise ProjectionCursorError("projection cursor signature is invalid") + data = json.loads(payload) + snapshot_id = str(data["snapshot_id"]) + offset = int(data["offset"]) + page_size = int(data["page_size"]) + except ProjectionCursorError: + raise + except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise ProjectionCursorError("projection cursor is malformed") from exc + if not re.fullmatch(r"[a-f0-9]{64}", snapshot_id): + raise ProjectionCursorError("projection cursor snapshot id is invalid") + if offset < 1 or not 1 <= page_size <= 500: + raise ProjectionCursorError("projection cursor bounds are invalid") + return {"snapshot_id": snapshot_id, "offset": offset, "page_size": page_size} + + +def _observe_registration( + registration: RepositoryRegistration, *, observed_at: str +) -> dict[str, Any]: + repo_root = registration.path + if repo_root is None: + raise ValueError("registration has no local checkout source") + if not repo_root.is_dir() or not (repo_root / ".git").exists(): + raise ValueError("registered checkout is missing or is not a Git working copy") + classification_path = repo_root / ".repo-classification.yaml" + if not classification_path.is_file(): + raise ValueError("authoritative .repo-classification.yaml is missing") + classification_status = subprocess.run( + ["git", "status", "--porcelain", "--", ".repo-classification.yaml"], + cwd=repo_root, + capture_output=True, + text=True, + check=False, + ) + if classification_status.returncode != 0 or classification_status.stdout.strip(): + raise ValueError("authoritative classification has uncommitted changes") + classification = load_classification(repo_root) + if classification is None: + raise ValueError("authoritative classification is empty") + head_sha = _git_head(repo_root) + if head_sha is None: + raise ValueError("repository HEAD is unavailable") + normalized = _normalize_classification(classification) + return { + "operation": "upsert", + "repository_id": registration.repository_id, + "slug": registration.slug, + "lifecycle": registration.lifecycle, + "classification": normalized, + "revision": { + "head_sha": head_sha, + "source_fingerprint": hashlib.sha256(classification_path.read_bytes()).hexdigest(), + "observed_at": observed_at, + }, + } + + +def _git_head(repo_root: Path) -> str | None: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo_root, + capture_output=True, + text=True, + check=False, + ) + value = result.stdout.strip().lower() + return value if result.returncode == 0 and re.fullmatch(r"[a-f0-9]{40,64}", value) else None + + +def _classification_from_bytes(raw: bytes) -> dict[str, Any]: + try: + document = yaml.safe_load(raw.decode("utf-8")) or {} + except (UnicodeDecodeError, yaml.YAMLError) as exc: + raise ValueError(f"classification YAML is invalid: {exc}") from exc + if not isinstance(document, dict): + raise TypeError("classification document must be an object") + classification = document.get("repo_classification") + if not isinstance(classification, dict): + raise TypeError("classification document has no repo_classification object") + return require_valid_classification(classification) + + +def _normalize_classification(classification: dict[str, Any]) -> dict[str, Any]: + return { + "category": classification["category"], + "domain": classification["domain"], + "secondary_domains": sorted(classification.get("secondary_domains") or []), + "capability_tags": sorted(classification.get("capability_tags") or []), + "business_stake": sorted(classification.get("business_stake") or []), + "business_mechanics": sorted(classification.get("business_mechanics") or []), + } + + +def _discover_classified_repositories(root: Path) -> tuple[Path, ...]: + resolved = root.expanduser().resolve() + if (resolved / ".git").exists() and (resolved / ".repo-classification.yaml").is_file(): + return (resolved,) + if not resolved.is_dir(): + raise ValueError(f"fleet root does not exist: {resolved}") + return tuple( + child + for child in sorted(resolved.iterdir()) + if child.is_dir() + and (child / ".git").exists() + and (child / ".repo-classification.yaml").is_file() + ) + + +def _sha256(value: Any) -> str: + encoded = json.dumps(value, separators=(",", ":"), sort_keys=True).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _b64encode(value: bytes) -> str: + return base64.urlsafe_b64encode(value).rstrip(b"=").decode() + + +def _b64decode(value: str) -> bytes: + return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/retirement_disposition.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/retirement_disposition.py new file mode 100644 index 00000000..2ca6ba4b --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/retirement_disposition.py @@ -0,0 +1,81 @@ +"""Read-only check of an Activity Core retirement disposition (RMGR-WP-0019-T03). + +The 2026-10-04 Activity receiving run refused exactly three empty duplicate +projections. Before STATE-WP-0095-T04 may apply them, the owner's +``expected_retirement_uuids`` must match what Repo Manager observed: the same +three projections, none bound to tasks, and never the source-bound parent. +This module only compares data; it performs no hub or source writes. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +# Projections the receiving run refused as "would be retired" (evidence file +# docs/evidence/2026-10-04-historical-registrar-verification.json) with the +# hub ids observed read-only on 2026-10-06. +REFUSED_DUPLICATE_PROJECTIONS: dict[str, dict[str, str]] = { + "683724d9-4249-5e0e-88e0-7baff4330d58": { + "slug": "custodian-wp-0003", + "backing_relative_path": "workplans/archived/260707-custodian-WP-0003-event-bridge.md", + }, + "644017c7-c5f2-5a64-81c3-d8ee806f8251": { + "slug": "custodian-wp-0003a", + "backing_relative_path": "workplans/custodian-WP-0003a-event-bridge-model-rules-registry.md", + }, + "a5806df9-5b9a-50da-81c1-7e606076bed9": { + "slug": "custodian-wp-0003b", + "backing_relative_path": "workplans/custodian-WP-0003b-event-bridge-parser-workflow-triggers.md", + }, +} + +# Source-bound parent every custodian-WP-0003* file names; it must survive. +PRESERVED_PARENT_UUIDS = frozenset({"b4eb45a9-69e3-4ab0-b00c-67a53c3117c5"}) + + +def check_disposition( + expected_retirement_uuids: list[str], + task_counts: dict[str, int] | None = None, +) -> list[str]: + """Return problems; an empty list means the disposition is consistent.""" + problems: list[str] = [] + given = [u.lower() for u in expected_retirement_uuids] + if len(given) != len(set(given)): + problems.append("duplicate UUIDs in expected_retirement_uuids") + given_set = set(given) + known = set(REFUSED_DUPLICATE_PROJECTIONS) + for uuid in sorted(given_set & PRESERVED_PARENT_UUIDS): + problems.append(f"{uuid} is the preserved source-bound parent") + for uuid in sorted(given_set - known - PRESERVED_PARENT_UUIDS): + problems.append(f"{uuid} was not refused by the receiving run") + for uuid in sorted(known - given_set): + slug = REFUSED_DUPLICATE_PROJECTIONS[uuid]["slug"] + problems.append(f"{uuid} ({slug}) refused but missing from disposition") + for uuid in sorted(given_set & known): + count = (task_counts or {}).get(uuid) + if count is None: + problems.append(f"{uuid} task count not verified") + elif count != 0: + problems.append(f"{uuid} holds {count} tasks; not an empty projection") + return problems + + +def main(argv: list[str]) -> int: + """Usage: python -m repo_manager.retirement_disposition DISPOSITION.json + + The JSON needs ``expected_retirement_uuids`` and may carry ``task_counts``. + """ + if len(argv) != 2: + print(main.__doc__, file=sys.stderr) + return 2 + data = json.loads(Path(argv[1]).read_text()) + problems = check_disposition( + data.get("expected_retirement_uuids", []), data.get("task_counts") + ) + print(json.dumps({"ok": not problems, "problems": problems}, indent=2)) + return 0 if not problems else 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/runtime.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/runtime.py new file mode 100644 index 00000000..cdac4abd --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/runtime.py @@ -0,0 +1,155 @@ +"""Read-only HTTP runtime for Repo Manager projection publication.""" + +from __future__ import annotations + +import hmac +import os +import secrets +from contextlib import asynccontextmanager +from dataclasses import dataclass +from pathlib import Path + +from fastapi import FastAPI, Header, HTTPException, Query, Response, status + +from repo_manager import __version__ +from repo_manager.repository_publisher import ( + ClassificationPublisher, + ProjectionCursorError, + RegistryError, + load_repository_registry, +) + + +@dataclass(frozen=True, slots=True) +class PublisherSettings: + registry_path: Path | None = None + cursor_secret: str | None = None + api_token: str | None = None + page_size: int = 100 + max_snapshots: int = 4 + forgejo_base_url: str | None = None + forgejo_token: str | None = None + forgejo_timeout_seconds: float = 10.0 + forgejo_retries: int = 2 + remote_workers: int = 8 + + @classmethod + def from_env(cls) -> PublisherSettings: + registry = os.getenv("REPO_MANAGER_REGISTRY_PATH") + token = os.getenv("REPO_MANAGER_FORGEJO_TOKEN") + token_file = os.getenv("REPO_MANAGER_FORGEJO_TOKEN_FILE") + if token and token_file: + raise ValueError( + "set only one of REPO_MANAGER_FORGEJO_TOKEN and REPO_MANAGER_FORGEJO_TOKEN_FILE" + ) + if token_file: + try: + token = Path(token_file).read_text(encoding="utf-8").strip() + except OSError as exc: + raise ValueError(f"cannot read Forgejo token file: {exc}") from exc + if not token: + raise ValueError("Forgejo token file is empty") + return cls( + registry_path=Path(registry) if registry else None, + cursor_secret=os.getenv("REPO_MANAGER_CURSOR_SECRET") or secrets.token_urlsafe(32), + api_token=os.getenv("REPO_MANAGER_API_TOKEN"), + page_size=int(os.getenv("REPO_MANAGER_PROJECTION_PAGE_SIZE", "100")), + max_snapshots=int(os.getenv("REPO_MANAGER_PROJECTION_MAX_SNAPSHOTS", "4")), + forgejo_base_url=os.getenv( + "REPO_MANAGER_FORGEJO_BASE_URL", "https://forgejo.coulomb.social" + ), + forgejo_token=token, + forgejo_timeout_seconds=float( + os.getenv("REPO_MANAGER_FORGEJO_TIMEOUT_SECONDS", "10") + ), + forgejo_retries=int(os.getenv("REPO_MANAGER_FORGEJO_RETRIES", "2")), + remote_workers=int(os.getenv("REPO_MANAGER_FORGEJO_WORKERS", "8")), + ) + + +def create_app(*, settings: PublisherSettings | None = None) -> FastAPI: + resolved = settings or PublisherSettings.from_env() + publisher: ClassificationPublisher | None = None + configuration_error: str | None = None + try: + if resolved.registry_path is None: + raise ValueError("REPO_MANAGER_REGISTRY_PATH is required") + cursor_secret = resolved.cursor_secret or secrets.token_urlsafe(32) + publisher = ClassificationPublisher( + resolved.registry_path, + cursor_secret=cursor_secret, + page_size=resolved.page_size, + max_snapshots=resolved.max_snapshots, + forgejo_base_url=resolved.forgejo_base_url, + forgejo_token=resolved.forgejo_token, + forgejo_timeout_seconds=resolved.forgejo_timeout_seconds, + forgejo_retries=resolved.forgejo_retries, + remote_workers=resolved.remote_workers, + ) + except ValueError as exc: + configuration_error = str(exc) + + @asynccontextmanager + async def lifespan(_app: FastAPI): + yield + if publisher is not None: + publisher.close() + + app = FastAPI( + title="Repo Manager projection publisher", + version=__version__, + description="Read-only port.repo repository projection publication.", + lifespan=lifespan, + ) + app.state.publisher = publisher + app.state.configuration_error = configuration_error + + @app.get("/healthz", tags=["system"]) + async def healthz() -> dict[str, str]: + return {"status": "ok", "service": "repo-manager", "version": __version__} + + @app.get("/readyz", tags=["system"]) + async def readyz(response: Response) -> dict[str, str | int]: + error = app.state.configuration_error + if error is None: + try: + registrations = load_repository_registry(resolved.registry_path) # type: ignore[arg-type] + except RegistryError as exc: + error = str(exc) + if error is not None: + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + return {"status": "degraded", "error": error} + return {"status": "ok", "repository_count": len(registrations)} + + @app.get( + "/ports/repositories/classifications", + tags=["port.repo"], + openapi_extra={"x-port-id": "port.repo", "x-direction": "out"}, + ) + def repository_classifications( + cursor: str | None = Query(default=None, max_length=1000), + authorization: str | None = Header(default=None), + ) -> dict: + if resolved.api_token is not None and not hmac.compare_digest( + authorization or "", f"Bearer {resolved.api_token}" + ): + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="unauthorized") + active = app.state.publisher + if active is None: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=app.state.configuration_error, + ) + try: + return active.fetch_page(cursor) + except ProjectionCursorError as exc: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc + except RegistryError as exc: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(exc) + ) from exc + + return app + + +app = create_app() diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/sbom_client.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/sbom_client.py new file mode 100644 index 00000000..d31447b0 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/sbom_client.py @@ -0,0 +1,379 @@ +"""Deprecated Repo Manager CLI adapter for the SBOM Nexus product owner.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from urllib.parse import quote + +import httpx + +SNAPSHOT_SCHEMA = "sbom-nexus.snapshot.v1" +SNAPSHOT_REQUIRED_FIELDS = frozenset( + { + "schema", + "ok", + "repo_slug", + "source_revision", + "generated_at", + "entry_count", + "entries", + "sources", + "licence_report", + "errors", + } +) +PREVIEW_CONTEXT = { + "mode": "local-preview", + "authoritative": False, + "persisted": False, + "advances_last_attempt_at": False, + "advances_last_success_at": False, + "creates_snapshot_history": False, +} +SERVICE_CONTEXT = { + "mode": "authoritative-service", + "authoritative": True, + "product_owner": "sbom-nexus", +} + + +class SBOMContractError(ValueError): + """Raised when a Nexus response is outside the pinned consumer contract.""" + + +class SBOMServiceError(RuntimeError): + """A sanitized, deterministic failure from the authoritative service client.""" + + def __init__( + self, + code: str, + method: str, + path: str, + *, + status_code: int | None = None, + mutation_may_have_committed: bool = False, + ) -> None: + self.code = code + self.method = method + self.path = path + self.status_code = status_code + self.mutation_may_have_committed = mutation_may_have_committed + status = f" HTTP {status_code}" if status_code is not None else "" + super().__init__(f"SBOM Nexus {method} {path} failed ({code}{status})") + + def to_dict(self) -> dict[str, Any]: + return { + "code": self.code, + "method": self.method, + "path": self.path, + "status_code": self.status_code, + "mutation_may_have_committed": self.mutation_may_have_committed, + } + + +@dataclass(frozen=True) +class SBOMNexusConfig: + """Explicit service configuration; credentials are deliberately repr-hidden.""" + + base_url: str + timeout_seconds: float = 30.0 + bearer_token: str | None = field(default=None, repr=False) + + def __post_init__(self) -> None: + url = httpx.URL(self.base_url) + if url.scheme not in {"http", "https"} or not url.host: + raise ValueError("SBOM Nexus base_url must be an absolute HTTP(S) URL") + if url.username or url.password or url.query or url.fragment: + raise ValueError("SBOM Nexus base_url must not contain credentials, query, or fragment") + if not 0.1 <= self.timeout_seconds <= 300.0: + raise ValueError("SBOM Nexus timeout_seconds must be between 0.1 and 300") + object.__setattr__(self, "base_url", self.base_url.rstrip("/")) + + @classmethod + def from_environment(cls) -> SBOMNexusConfig: + base_url = os.getenv("SBOM_NEXUS_URL") + if not base_url: + raise ValueError("SBOM_NEXUS_URL is required for authoritative service mode") + try: + timeout = float(os.getenv("SBOM_NEXUS_TIMEOUT_SECONDS", "30")) + except ValueError as exc: + raise ValueError("SBOM_NEXUS_TIMEOUT_SECONDS must be numeric") from exc + return cls( + base_url=base_url, + timeout_seconds=timeout, + bearer_token=os.getenv("SBOM_NEXUS_TOKEN") or None, + ) + + +class SBOMNexusClient: + """Bounded client for Nexus-owned authoritative repository SBOM state.""" + + def __init__( + self, + config: SBOMNexusConfig, + *, + transport: httpx.BaseTransport | None = None, + ) -> None: + self.config = config + self._transport = transport + + def upsert_repository( + self, + repo_slug: str, + *, + nexus_checkout_path: str | None, + active: bool = True, + source_ref: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Project identity/path; the path must resolve inside Nexus's controlled source plane.""" + body: dict[str, Any] = {"checkout_path": nexus_checkout_path, "active": active} + if source_ref is not None: + body["source_ref"] = source_ref + result = self._request_object( + "PUT", + f"/repositories/{quote(repo_slug, safe='')}", + body=body, + operation="repository-projection", + writes_state=True, + required_fields=frozenset( + {"slug", "active"} | ({"source_ref"} if source_ref is not None else set()) + ), + ) + if source_ref is not None and result.get("source_ref") != source_ref: + raise SBOMServiceError( + "source_ref_projection_mismatch", + "PUT", + f"/repositories/{quote(repo_slug, safe='')}", + mutation_may_have_committed=True, + ) + return result + + def ingest_repository( + self, + repo_slug: str, + *, + expected_source_revision: str | None = None, + operation_id: str | None = None, + ) -> dict[str, Any]: + """Persist one Nexus scan outcome without automatic mutation retries.""" + headers = {"Idempotency-Key": operation_id} if operation_id else None + result = self._request_object( + "POST", + f"/sbom/{quote(repo_slug, safe='')}/ingest", + headers=headers, + operation="ingest", + writes_state=True, + required_fields=frozenset( + {"repo_slug", "snapshot_id", "status", "entry_count", "snapshot_at"} + ), + ) + if expected_source_revision and result.get("status") == "ingested": + actual = result.get("source_revision") + if actual != expected_source_revision: + raise SBOMServiceError( + "source_revision_mismatch", + "POST", + f"/sbom/{quote(repo_slug, safe='')}/ingest", + mutation_may_have_committed=True, + ) + return result + + def latest_snapshot(self, repo_slug: str) -> dict[str, Any]: + return self._request_object( + "GET", + f"/sbom/{quote(repo_slug, safe='')}", + operation="latest-snapshot", + writes_state=False, + required_fields=frozenset( + { + "repo_slug", + "last_attempt_at", + "last_success_at", + "last_status", + "entry_count", + "snapshot_id", + "entries", + } + ), + ) + + def licence_report(self) -> dict[str, Any]: + return self._request_object( + "GET", + "/sbom/report/licences/", + operation="licence-report", + writes_state=False, + required_fields=frozenset({"groups", "copyleft_direct_prod", "copyleft_direct_count"}), + ) + + def _request_object( + self, + method: str, + path: str, + *, + body: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + operation: str, + writes_state: bool, + required_fields: frozenset[str], + ) -> dict[str, Any]: + request_headers = dict(headers or {}) + if self.config.bearer_token: + request_headers["Authorization"] = f"Bearer {self.config.bearer_token}" + try: + with httpx.Client( + base_url=self.config.base_url, + timeout=self.config.timeout_seconds, + headers=request_headers, + transport=self._transport, + ) as client: + response = client.request(method, path, json=body) + except httpx.TimeoutException as exc: + raise SBOMServiceError("timeout", method, path) from exc + except httpx.RequestError as exc: + raise SBOMServiceError("transport_error", method, path) from exc + + if response.status_code >= 400: + raise SBOMServiceError( + "http_error", + method, + path, + status_code=response.status_code, + mutation_may_have_committed=writes_state and response.status_code >= 500, + ) + try: + payload = response.json() + except ValueError as exc: + raise SBOMServiceError( + "invalid_json", + method, + path, + mutation_may_have_committed=writes_state, + ) from exc + if not isinstance(payload, dict): + raise SBOMServiceError( + "invalid_response_type", + method, + path, + mutation_may_have_committed=writes_state, + ) + if required_fields - payload.keys(): + raise SBOMServiceError( + "contract_error", + method, + path, + mutation_may_have_committed=writes_state, + ) + payload["repo_manager_context"] = { + **SERVICE_CONTEXT, + "operation": operation, + "writes_state": writes_state, + } + return payload + + +def validate_snapshot_contract(payload: dict[str, Any]) -> None: + """Accept additive fields but reject unknown schemas or missing required fields.""" + if payload.get("schema") != SNAPSHOT_SCHEMA: + raise SBOMContractError( + f"unsupported SBOM Nexus schema {payload.get('schema')!r}; expected {SNAPSHOT_SCHEMA!r}" + ) + missing = sorted(SNAPSHOT_REQUIRED_FIELDS - payload.keys()) + if missing: + raise SBOMContractError(f"SBOM Nexus snapshot is missing required fields: {missing}") + + +def _mark_preview(payload: dict[str, Any]) -> dict[str, Any]: + payload["repo_manager_context"] = dict(PREVIEW_CONTEXT) + payload["delegated_by"] = "repo-manager" + payload["product_owner"] = "sbom-nexus" + return payload + + +def scan_repository_via_nexus(repo_root: Path, *, slug: str | None = None) -> dict[str, Any]: + """Delegate a non-authoritative local preview to the Nexus-owned CLI.""" + executable = os.getenv("SBOM_NEXUS_CLI") or shutil.which("sbom-nexus") + if not executable: + return _error( + "sbom-nexus CLI not found; install the sbom-nexus package or set " + "SBOM_NEXUS_CLI to its executable path" + ) + + command = [executable, "scan", str(repo_root)] + if slug: + command.extend(["--slug", slug]) + try: + completed = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + ) + except OSError as exc: + return _error(f"could not execute SBOM Nexus CLI: {exc}") + + try: + payload = json.loads(completed.stdout) + except json.JSONDecodeError: + detail = completed.stderr.strip() or completed.stdout.strip() or "no output" + return _error( + f"SBOM Nexus CLI returned invalid JSON (exit {completed.returncode}): {detail}" + ) + if not isinstance(payload, dict): + return _error("SBOM Nexus CLI returned a non-object JSON response") + + try: + validate_snapshot_contract(payload) + except SBOMContractError as exc: + return _error(str(exc), reason="sbom-nexus-contract") + + _mark_preview(payload) + if completed.returncode and payload.get("ok", True): + payload["ok"] = False + payload.setdefault("errors", []).append( + { + "reason": "sbom-nexus-exit", + "detail": completed.stderr.strip() or f"exit {completed.returncode}", + } + ) + return payload + + +def licence_report_from_snapshot(snapshot: dict[str, Any]) -> dict[str, Any]: + """Preserve the old rmgr report-shaped view over a Nexus snapshot.""" + return { + "ok": bool(snapshot.get("ok")), + "repo_slug": snapshot.get("repo_slug"), + "source_revision": snapshot.get("source_revision"), + "generated_at": snapshot.get("generated_at"), + "entry_count": int(snapshot.get("entry_count") or 0), + "licence_report": snapshot.get("licence_report") + or { + "groups": [], + "copyleft_direct_prod": [], + "copyleft_direct_count": 0, + }, + "errors": snapshot.get("errors") or [], + "repo_manager_context": dict(PREVIEW_CONTEXT), + "delegated_by": "repo-manager", + "product_owner": "sbom-nexus", + } + + +def _error(detail: str, *, reason: str = "sbom-nexus-delegation") -> dict[str, Any]: + return _mark_preview( + { + "schema": SNAPSHOT_SCHEMA, + "ok": False, + "entry_count": 0, + "entries": [], + "sources": [], + "errors": [{"reason": reason, "detail": detail}], + } + ) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/secret_scan.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/secret_scan.py new file mode 100644 index 00000000..c2203bd2 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/secret_scan.py @@ -0,0 +1,428 @@ +"""Content-silent adapter for staged-blob secret detection with pinned Gitleaks. + +This is a repository control adapter, not credential custody or an authorization +boundary. It does not install hooks, alter indexes, send content off-host, or +claim history/push/CI coverage. Its caller must treat unverified as refusal. +""" +from __future__ import annotations + +import argparse +import contextlib +import ctypes +import fcntl +import functools +import hashlib +import json +import os +import re +import shutil +import signal +import resource +import stat +import sys +import subprocess +import tempfile +import time +from pathlib import Path + +# Resolve libc and prctl before fork; child_limits performs no library loading. +_LIBC = ctypes.CDLL(None, use_errno=True) +_PRCTL = _LIBC.prctl +_PRCTL.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong] +_PRCTL.restype = ctypes.c_int + +VERSION = '8.30.1' +BINARY_SHA256 = '88f91962aa2f93ac6ab281d553b9e125f5197bbbce38f9f2437f7299c32e5509' +MAX_BLOB_BYTES = 8 * 1024 * 1024 +MAX_TOTAL_BYTES = 64 * 1024 * 1024 +MAX_BLOBS = 500 +MAX_BATCH_BLOBS = 64 +MAX_BATCH_BYTES = 8 * 1024 * 1024 +# PIPE_BUF bytes fit the original stdin writer's single atomic write. Larger +# blobs retain the stdin path, avoiding a change to fragment boundaries. +DIRECTORY_BLOB_BYTES = 4096 +MAX_REPORT_BYTES = 1024 * 1024 +SCAN_SECONDS = 120 +CONFIG = '[extend]\nuseDefault = true\n' + + +class Unverified(Exception): + """A fixed reason code, never a tool exception or repository content.""" + + +def git(repo: Path, *args: str) -> bytes: + env = {**os.environ, 'GIT_NO_REPLACE_OBJECTS': '1'} + result = subprocess.run(['git', '-C', str(repo), *args], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, env=env, timeout=20, check=False, + preexec_fn=functools.partial(child_limits, MAX_TOTAL_BYTES, os.getpid())) + if result.returncode: + raise Unverified('git-read-failed') + return result.stdout + + +def staged_objects(repo: Path) -> tuple[bytes, list[str]]: + snapshot = git(repo, 'ls-files', '--stage', '-z') + changed = set(git(repo, 'diff', '--cached', '--name-only', '-z', '--no-renames', + '--no-ext-diff', '--no-textconv', '--diff-filter=ACMRTUXB').split(b'\0')) - {b''} + objects: set[str] = set() + seen: set[bytes] = set() + for record in snapshot.split(b'\0'): + if not record: + continue + header, path = record.split(b'\t', 1) + if path not in changed: + continue + mode, oid, stage = header.split() + if stage != b'0' or mode not in (b'100644', b'100755', b'120000'): + raise Unverified('unsupported-index-entry') + if not re.fullmatch(rb'[a-f0-9]{40}|[a-f0-9]{64}', oid): + raise Unverified('invalid-object-id') + objects.add(oid.decode('ascii')) + seen.add(path) + if changed != seen: + raise Unverified('incomplete-index-enumeration') + if len(objects) > MAX_BLOBS: + raise Unverified('staged-blob-limit') + return snapshot, sorted(objects) + + +# Source bytes must never be copied onto persistent storage. The watchdog owns +# no source data; its pipe EOF detects parent death without a service or hook. +_CLEANUP = """ +import os, select, shutil, signal, stat, sys +path, dev, ino, uid, parent = sys.argv[1:] +root = os.open('/dev/shm', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) +if os.getppid() != int(parent): + sys.exit(4) +pid = os.pidfd_open(int(parent)) +if os.getppid() != int(parent): + sys.exit(4) +os.write(1, b'R') +ready, _, _ = select.select([0, pid], [], [], 180) +# Exceptional abort targets only the exact parent, never a process group. +# If kernel quiescence stalls, retain this observer/duty until death is proven. +if not ready: + try: + signal.pidfd_send_signal(pid, signal.SIGTERM) + except ProcessLookupError: + pass + dead, _, _ = select.select([pid], [], [], 5) + if not dead: + try: + signal.pidfd_send_signal(pid, signal.SIGKILL) + except ProcessLookupError: + pass + select.select([pid], [], []) +name = os.path.basename(path) +try: + info = os.stat(name, dir_fd=root, follow_symlinks=False) +except FileNotFoundError: + sys.exit(0) +if (info.st_dev, info.st_ino, info.st_uid) != (int(dev), int(ino), int(uid)): + sys.exit(3) +if not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o700: + sys.exit(3) +shutil.rmtree(name, dir_fd=root) +""" + + +def is_tmpfs(directory: Path) -> bool: + descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + result = ctypes.create_string_buffer(256) + if _LIBC.fstatfs(descriptor, ctypes.byref(result)): + raise OSError(ctypes.get_errno(), 'statfs failed') + return ctypes.c_long.from_buffer(result).value == 0x01021994 + finally: + os.close(descriptor) + + +def remove_storage(directory: Path, identity: os.stat_result) -> None: + root = os.open('/dev/shm', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + try: + current = os.stat(directory.name, dir_fd=root, follow_symlinks=False) + except FileNotFoundError: + return + if ((current.st_dev, current.st_ino, current.st_uid) != + (identity.st_dev, identity.st_ino, identity.st_uid) + or not stat.S_ISDIR(current.st_mode) + or stat.S_IMODE(current.st_mode) != 0o700): + raise Unverified('volatile-storage-identity-changed') + shutil.rmtree(directory.name, dir_fd=root) + finally: + os.close(root) + + +@contextlib.contextmanager +def private_storage(): + if not is_tmpfs(Path('/dev/shm')): + raise Unverified('volatile-storage-unavailable') + temporary = tempfile.mkdtemp(prefix='rmgr-secret-scan-', dir='/dev/shm') + directory = Path(temporary) + identity = directory.lstat() + watcher = None + try: + if not is_tmpfs(directory): + raise Unverified('volatile-storage-unavailable') + watcher = subprocess.Popen([sys.executable, '-I', '-c', _CLEANUP, temporary, + str(identity.st_dev), str(identity.st_ino), str(identity.st_uid), str(os.getpid())], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + env={'PATH': '/usr/bin:/bin', 'LANG': 'C'}, close_fds=True) + import select + ready, _, _ = select.select([watcher.stdout], [], [], 5) + if not ready or watcher.stdout.read(1) != b'R' or watcher.poll() is not None: + raise Unverified('cleanup-watchdog-unavailable') + watcher.stdout.close() + yield directory, watcher + if watcher.poll() is not None: + raise Unverified('cleanup-watchdog-unavailable') + finally: + pending_error = sys.exc_info()[0] is not None + # All synchronous Git/scanner children have exited before yield unwinds. + try: + remove_storage(directory, identity) + finally: + if watcher is not None: + watcher.stdin.close() + watcher.stdout.close() + try: + code = watcher.wait(timeout=5) + except subprocess.TimeoutExpired: + watcher.kill() + code = watcher.wait(timeout=5) + if code and not pending_error: + raise Unverified('cleanup-watchdog-failed') + + +def child_limits(file_bytes: int, parent_pid: int) -> None: + # Parent loss must quiesce every source-reading/writing child. Install the + # signal before any exec and close the fork/prctl race using the exact PID. + if _PRCTL(1, signal.SIGKILL, 0, 0, 0) or os.getppid() != parent_pid: + os._exit(127) + # Linux adapter already requires /proc/self/fd. Limits apply only to children. + resource.setrlimit(resource.RLIMIT_FSIZE, (file_bytes, file_bytes)) + resource.setrlimit(resource.RLIMIT_CPU, (20, 20)) + resource.setrlimit(resource.RLIMIT_AS, (8 * 1024**3, 8 * 1024**3)) + + +def object_batch(repo: Path, objects: list[str], directory: Path) -> list[tuple[str, bytes]]: + """Bound Git's output before capture, then validate the complete framing.""" + request = ''.join(oid + '\n' for oid in objects).encode('ascii') + env = {**os.environ, 'GIT_NO_REPLACE_OBJECTS': '1'} + with tempfile.TemporaryFile(dir=directory) as output: + result = subprocess.run(['git', '-C', str(repo), 'cat-file', '--batch'], + input=request, stdout=output, stderr=subprocess.DEVNULL, + env=env, timeout=20, check=False, + preexec_fn=functools.partial(child_limits, + MAX_BATCH_BYTES + len(objects) * 128, os.getpid())) + if result.returncode: + raise Unverified('git-read-failed') + output.seek(0) + blobs = [] + for oid in objects: + header = output.readline(128) + match = re.fullmatch(rb'([a-f0-9]{40}|[a-f0-9]{64}) blob ([0-9]+)\n', header) + if not match or match[1].decode() != oid: + raise Unverified('invalid-object-batch') + size = int(match[2]) + if size > MAX_BLOB_BYTES or size > MAX_BATCH_BYTES: + raise Unverified('staged-byte-limit') + content = output.read(size) + if len(content) != size or output.read(1) != b'\n': + raise Unverified('blob-size-mismatch') + algorithm = 'sha1' if len(oid) == 40 else 'sha256' + digest = hashlib.new(algorithm, b'blob ' + str(size).encode() + b'\0' + content) + if digest.hexdigest() != oid: + raise Unverified('blob-digest-mismatch') + blobs.append((oid, content)) + if output.read(1): + raise Unverified('invalid-object-batch') + return blobs + + +def scan_staged(repo: Path, binary: Path) -> dict: + """Return only fixed statuses/counts and Git blob IDs, never finding contents.""" + scanned = 0 + descriptor = None + executable = None + deadline = time.monotonic() + SCAN_SECONDS + try: + descriptor = os.open(binary, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + info = os.fstat(descriptor) + if (not stat.S_ISREG(info.st_mode) or info.st_mode & 0o022 + or not info.st_mode & 0o111 or info.st_uid not in (0, os.getuid()) + or info.st_size > 64 * 1024 * 1024): + raise Unverified('scanner-file-refused') + executable = os.memfd_create('rmgr-verified-scanner', os.MFD_ALLOW_SEALING) + digest = hashlib.sha256() + copied = 0 + while chunk := os.read(descriptor, 1024 * 1024): + copied += len(chunk) + if copied > 64 * 1024 * 1024: + raise Unverified('scanner-file-refused') + digest.update(chunk) + with memoryview(chunk) as view: + while view: + written = os.write(executable, view) + if written <= 0: + raise Unverified('scanner-file-refused') + view = view[written:] + if digest.hexdigest() != BINARY_SHA256: + raise Unverified('scanner-digest-mismatch') + fcntl.fcntl(executable, fcntl.F_ADD_SEALS, fcntl.F_SEAL_WRITE | fcntl.F_SEAL_GROW + | fcntl.F_SEAL_SHRINK | fcntl.F_SEAL_SEAL) + snapshot, objects = staged_objects(repo) + blocked = [] + # Neither source names nor repository config/ignore files enter this + # private directory. Reports contain ONLY neutral blob paths. + with private_storage() as (directory, watcher): + temporary = str(directory) + config = directory / 'scanner.toml'; config.write_text(CONFIG) + ignore = directory / '.gitleaksignore'; ignore.write_text('') + template = directory / 'report.tmpl' + template.write_text('{{range .}}{{.File}}\n{{end}}') + env = {'PATH': '/usr/bin:/bin', 'HOME': temporary, 'LANG': 'C', + 'TMPDIR': temporary, 'GOMAXPROCS': '2'} + # One bounded metadata query replaces a size subprocess per object. + request = ''.join(oid + '\n' for oid in objects).encode('ascii') + with tempfile.TemporaryFile(dir=directory) as output: + result = subprocess.run(['git', '-C', str(repo), 'cat-file', '--batch-check'], + input=request, stdout=output, stderr=subprocess.DEVNULL, + env={**os.environ, 'GIT_NO_REPLACE_OBJECTS': '1'}, timeout=20, + check=False, preexec_fn=functools.partial(child_limits, MAX_BLOBS * 128, os.getpid())) + if result.returncode: + raise Unverified('git-read-failed') + output.seek(0) + metadata = output.read(MAX_BLOBS * 128 + 1).splitlines() + if len(metadata) != len(objects): + raise Unverified('invalid-object-batch') + sizes = [] + for oid, line in zip(objects, metadata): + match = re.fullmatch(rb'([a-f0-9]{40}|[a-f0-9]{64}) blob ([0-9]+)', line) + if not match or match[1].decode() != oid: + raise Unverified('invalid-object-batch') + sizes.append(int(match[2])) + if any(size > MAX_BLOB_BYTES for size in sizes) or sum(sizes) > MAX_TOTAL_BYTES: + raise Unverified('staged-byte-limit') + + def invoke(mode: str, content: bytes | None = None) -> list[str]: + if watcher.poll() is not None: + raise Unverified('cleanup-watchdog-unavailable') + remaining = deadline - time.monotonic() + if remaining <= 0: + raise Unverified('scan-deadline-exceeded') + report = directory / 'report.paths' + report.unlink(missing_ok=True) + command = [str(binary), mode, '--config', str(config), + '--gitleaks-ignore-path', str(ignore), '--ignore-gitleaks-allow', + '--redact=100', '--no-banner', '--no-color', '--log-level', 'warn', + '--exit-code', '20', '--timeout', '15'] + if mode == 'dir': + command += ['--report-format', 'template', '--report-template', str(template), '--report-path', str(report), + '--max-target-megabytes', '0', 'blobs'] + with tempfile.TemporaryFile(dir=directory) as diagnostics: + result = subprocess.run(command, executable=f'/proc/self/fd/{executable}', + pass_fds=(executable,), input=content, cwd=directory, env=env, + stdout=subprocess.DEVNULL, stderr=diagnostics, + timeout=min(20, remaining), check=False, + preexec_fn=functools.partial(child_limits, MAX_REPORT_BYTES, os.getpid())) + # The scanner can warn and skip a file without an error exit. + # Refuse ALL warnings/errors, including partial traversal. + diagnostics.seek(0) + messages = diagnostics.read(MAX_REPORT_BYTES + 1) + # v8.30.1 emits exactly this summary for a successful blocked + # scan. Every other warning/error is a refusal, never output. + summary = re.fullmatch( + rb'[0-9]{1,2}:[0-9]{2}(?:AM|PM) WRN leaks found: [1-9][0-9]*\n', + messages) + if (result.returncode not in (0, 20) + or (messages and not (result.returncode == 20 and summary))): + raise Unverified('scanner-failed') + if mode == 'stdin': + return ['stdin'] if result.returncode == 20 else [] + if not report.exists() or report.stat().st_size > MAX_REPORT_BYTES: + raise Unverified('scanner-report-refused') + paths = report.read_bytes().splitlines() + known = {('blobs/' + oid).encode(): oid for oid in small} + if any(path not in known for path in paths): + raise Unverified('scanner-report-refused') + findings = sorted({known[path] for path in paths}) + if bool(findings) != (result.returncode == 20): + raise Unverified('scanner-report-refused') + return findings + + offset = 0 + while offset < len(objects): + batch = [] + batch_bytes = 0 + while offset < len(objects) and len(batch) < MAX_BATCH_BLOBS: + size = sizes[offset] + if batch and batch_bytes + size > MAX_BATCH_BYTES: + break + batch.append(objects[offset]); batch_bytes += size; offset += 1 + if watcher.poll() is not None: + raise Unverified('cleanup-watchdog-unavailable') + blobs = object_batch(repo, batch, directory) + # Check metadata again against exact bytes, including same-size + # substitution via corrupt object storage. Hashes are checked above. + expected = dict(zip(objects, sizes)) + if any(len(content) != expected[oid] for oid, content in blobs): + raise Unverified('blob-size-mismatch') + payload = directory / 'blobs'; payload.mkdir(mode=0o700) + small = [] + try: + for oid, content in blobs: + if len(content) > DIRECTORY_BLOB_BYTES: + if invoke('stdin', content): + blocked.append(oid) + scanned += 1 + else: + path = payload / oid + with path.open('xb') as target: + target.write(content) + path.chmod(0o600) + small.append(oid) + if small: + blocked.extend(invoke('dir')) + scanned += len(small) + finally: + for path in payload.iterdir(): + path.unlink() + payload.rmdir() + if git(repo, 'ls-files', '--stage', '-z') != snapshot: + raise Unverified('index-changed-during-scan') + if time.monotonic() > deadline: + raise Unverified('scan-deadline-exceeded') + return {'status': 'blocked' if blocked else 'clean', 'scope': 'changed-staged-blobs', + 'scanner_version': VERSION, 'scanner_sha256': BINARY_SHA256, + 'scanned_blobs': scanned, 'blocked_blob_ids': sorted(blocked)} + except Unverified as error: + reason = error.args[0] + except (OSError, ValueError, subprocess.SubprocessError): + reason = 'scan-unavailable' + finally: + if descriptor is not None: + os.close(descriptor) + if executable is not None: + os.close(executable) + return {'status': 'unverified', 'scope': 'changed-staged-blobs', + 'reason': reason, 'scanned_blobs': scanned} + + +def exit_code(result: dict) -> int: + return {'clean': 0, 'blocked': 1, 'unverified': 2}[result['status']] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--path', type=Path, default=Path('.')) + parser.add_argument('--binary', type=Path, required=True) + args = parser.parse_args(argv) + result = scan_staged(args.path, args.binary) + print(json.dumps(result, sort_keys=True)) + return exit_code(result) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/source_ref.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/source_ref.py new file mode 100644 index 00000000..7de1001e --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/source_ref.py @@ -0,0 +1,174 @@ +"""Controlled Forgejo source references for authoritative SBOM ingestion.""" + +from __future__ import annotations + +import re +import subprocess +from collections.abc import Callable +from pathlib import Path +from typing import Any +from urllib.parse import quote, urlparse + +import httpx + +from repo_manager.time import utc_now_text + +DEFAULT_FORGEJO_BASE_URL = "https://forgejo.coulomb.social" +SOURCE_REF_KIND = "forgejo-archive-v1" +_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") +_SHA_RE = re.compile(r"^[0-9a-f]{40}$") +_SCP_REMOTE_RE = re.compile(r"^(?:[^@]+@)?(?P[^:]+):(?P.+)$") +_ALLOWED_REMOTE_HOSTS = frozenset({"forgejo.coulomb.social", "forgejo-remote"}) + + +def canonical_coulomb_repository(remote_url: str, repo_slug: str) -> str | None: + """Normalize accepted Forgejo remotes to the strict public v1 identity.""" + if not _SLUG_RE.fullmatch(repo_slug): + return None + remote_url = remote_url.strip() + parsed = urlparse(remote_url) + if "://" in remote_url: + if parsed.scheme not in {"http", "https", "ssh"}: + return None + if ( + parsed.username not in {None, "git"} + or parsed.password + or parsed.query + or parsed.fragment + ): + return None + host = parsed.hostname + path = parsed.path + else: + match = _SCP_REMOTE_RE.fullmatch(remote_url) + if not match: + return None + host = match.group("host") + path = match.group("path") + if host not in _ALLOWED_REMOTE_HOSTS: + return None + parts = path.strip("/").removesuffix(".git").split("/") + if parts != ["coulomb", repo_slug]: + return None + return f"coulomb/{repo_slug}" + + +class ForgejoSourceResolver: + """Observe a public default-branch commit without using workstation Git state.""" + + def __init__( + self, + *, + base_url: str = DEFAULT_FORGEJO_BASE_URL, + timeout_seconds: float = 10.0, + transport: httpx.BaseTransport | None = None, + now: Callable[[], str] = utc_now_text, + ) -> None: + parsed = httpx.URL(base_url) + if parsed.scheme not in {"http", "https"} or not parsed.host: + raise ValueError("Forgejo base_url must be an absolute HTTP(S) URL") + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError("Forgejo base_url must not contain credentials, query, or fragment") + if not 0.1 <= timeout_seconds <= 60.0: + raise ValueError("Forgejo timeout_seconds must be between 0.1 and 60") + self.base_url = base_url.rstrip("/") + self.timeout_seconds = timeout_seconds + self._transport = transport + self._now = now + + def resolve( + self, + repo_root: Path, + *, + repo_slug: str | None = None, + remote_name: str = "origin", + ) -> dict[str, Any]: + repo_root = repo_root.expanduser().resolve() + slug = repo_slug or repo_root.name + if not _SLUG_RE.fullmatch(slug): + return self._unsupported(slug, "invalid-repo-slug") + + remote = subprocess.run( + ["git", "remote", "get-url", remote_name], + cwd=repo_root, + capture_output=True, + text=True, + check=False, + ) + if remote.returncode != 0: + return self._unsupported(slug, "missing-canonical-remote") + repository = canonical_coulomb_repository(remote.stdout, slug) + if repository is None: + return self._unsupported(slug, "non-coulomb-or-identity-mismatch") + + repo_path = f"/api/v1/repos/coulomb/{quote(slug, safe='')}" + metadata = self._get_object(repo_path) + if metadata is None: + return self._unsupported(slug, "source-unresolvable") + if metadata.get("private") is True: + return self._unsupported(slug, "private-repository") + if metadata.get("full_name") != repository: + return self._unsupported(slug, "forgejo-identity-mismatch") + branch = metadata.get("default_branch") + if not isinstance(branch, str) or not branch or branch.startswith("/"): + return self._unsupported(slug, "default-branch-unresolvable") + + observed_ref = f"refs/heads/{branch}" + ref_path = f"{repo_path}/git/refs/heads/{quote(branch, safe='')}" + refs = self._get_json(ref_path) + if isinstance(refs, dict): + refs = [refs] + if not isinstance(refs, list): + return self._unsupported(slug, "default-branch-unresolvable") + matching = next( + (item for item in refs if isinstance(item, dict) and item.get("ref") == observed_ref), + None, + ) + obj = matching.get("object") if matching else None + revision = obj.get("sha") if isinstance(obj, dict) else None + if not isinstance(revision, str) or not _SHA_RE.fullmatch(revision): + return self._unsupported(slug, "default-branch-unresolvable") + + return { + "ok": True, + "supported": True, + "repo_slug": slug, + "source_ref": { + "kind": SOURCE_REF_KIND, + "repository": repository, + "revision": revision, + "observed_ref": observed_ref, + "observed_at": self._now(), + }, + } + + def _get_object(self, path: str) -> dict[str, Any] | None: + payload = self._get_json(path) + return payload if isinstance(payload, dict) else None + + def _get_json(self, path: str) -> Any | None: + try: + with httpx.Client( + base_url=self.base_url, + timeout=self.timeout_seconds, + transport=self._transport, + ) as client: + response = client.get(path) + except httpx.RequestError: + return None + if response.status_code != 200: + return None + try: + return response.json() + except ValueError: + return None + + @staticmethod + def _unsupported(repo_slug: str, reason: str) -> dict[str, Any]: + return { + "ok": True, + "supported": False, + "repo_slug": repo_slug, + "reason": reason, + "source_ref": None, + } diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/standards.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/standards.py new file mode 100644 index 00000000..dc7db91c --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/standards.py @@ -0,0 +1,315 @@ +"""Repository standards as Repo Manager implements Custodian canon. + +Canon remains authoritative. This module does not fork the rules; it names +the signals, required files, and anti-patterns the canon already states. + +Sources: +- the-custodian/canon/standards/project-repository-flavor_v0.1.md +- the-custodian/canon/standards/repo-classification-standard_v1.0.md +- ADR-007 decision 1 (prefix from repo/project identity, never flavor) +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal + +from repo_manager.observe import load_classification +from repo_manager.parse.workplan import parse_frontmatter +from repo_manager.record_identity import scan_record_identities + +CANON_FLAVOR = "the-custodian/canon/standards/project-repository-flavor_v0.1.md" +CANON_CLASSIFICATION = "the-custodian/canon/standards/repo-classification-standard_v1.0.md" + +DURABLE_FLAVORS = ("experimental", "research", "tooling", "product", "business", "publication") +CATEGORIES = (*DURABLE_FLAVORS, "project") +FLAVOR_MARKER_PREFIX = "PRJ-WP" +WP_ID_RE = re.compile(r"^([A-Z][A-Z0-9]*)-WP-(\d{4})(?:-T\d{2})?$") +GOAL_SECTIONS = ("outcome", "invariants", "success gates", "project retirement") + +Severity = Literal["missing", "contradictory", "warning"] + + +@dataclass +class Finding: + code: str + severity: Severity + path: str + message: str + canon: str + + +@dataclass +class FlavorResolution: + flavor: str + prj_layout: bool + signals: dict[str, str | None] + disagreements: list[str] + + +@dataclass +class ConformanceReport: + slug: str + flavor: FlavorResolution + findings: list[Finding] = field(default_factory=list) + + @property + def ok(self) -> bool: + return not any(f.severity in {"missing", "contradictory"} for f in self.findings) + + def to_dict(self) -> dict[str, Any]: + return { + "ok": self.ok, + "slug": self.slug, + "flavor": self.flavor.flavor, + "prj_layout": self.flavor.prj_layout, + "signals": self.flavor.signals, + "disagreements": self.flavor.disagreements, + "findings": [f.__dict__ for f in self.findings], + "canon": [CANON_FLAVOR, CANON_CLASSIFICATION], + } + + +def _slug(repo_root: Path) -> str: + return re.sub(r"[^a-z0-9]+", "-", repo_root.name.lower()).strip("-") or "repo" + + +def _goal_flavor(repo_root: Path) -> str | None: + path = repo_root / "GOAL.md" + if not path.is_file(): + return None + fm, _ = parse_frontmatter(path.read_text(encoding="utf-8")) + raw = fm.get("repo_flavor") + return str(raw).strip().lower() if raw else None + + +def resolve_flavor(repo_root: Path, *, slug: str | None = None) -> FlavorResolution: + """Precedence: classification category, GOAL.md repo_flavor, slug prefix.""" + slug = slug or _slug(repo_root) + classification = load_classification(repo_root) or {} + category = classification.get("category") + category = str(category).strip().lower() if category else None + goal_flavor = _goal_flavor(repo_root) + slug_prj = slug.startswith("prj-") + signals = { + "classification.category": category, + "GOAL.md.repo_flavor": goal_flavor, + "slug_prefix": "prj-" if slug_prj else None, + } + + if category in CATEGORIES: + flavor = category + elif goal_flavor in CATEGORIES: + flavor = goal_flavor + elif slug_prj: + flavor = "project" + else: + flavor = "tooling" + + disagreements: list[str] = [] + if category and goal_flavor and category != goal_flavor: + disagreements.append( + f"classification.category={category!r} disagrees with GOAL.md repo_flavor={goal_flavor!r}" + ) + if slug_prj and flavor != "project": + disagreements.append(f"slug is prj-* but resolved flavor is {flavor!r}") + if category == "project" and not slug_prj and goal_flavor != "project": + disagreements.append( + "category is project but slug is not prj-*; not every category:project is a prj- repo" + ) + + prj_layout = slug_prj or goal_flavor == "project" + return FlavorResolution( + flavor=flavor, + prj_layout=prj_layout, + signals=signals, + disagreements=disagreements, + ) + + +def required_files(flavor: FlavorResolution) -> tuple[str, ...]: + common = ("README.md", "SCOPE.md", "AGENTS.md", ".repo-classification.yaml", "workplans") + if flavor.prj_layout: + return ("GOAL.md", *common) + return ("INTENT.md", *common) + + +def purpose_document(flavor: FlavorResolution) -> str: + return "GOAL.md" if flavor.prj_layout else "INTENT.md" + + +def expected_workplan_prefix(repo_root: Path, *, slug: str | None = None) -> str | None: + """Prefix must come from repo/project identity, never the flavor marker.""" + slug = slug or _slug(repo_root) + if slug.startswith("prj-"): + stem = slug.removeprefix("prj-") + letters = re.sub(r"[^a-z0-9]", "", stem) + if letters: + return letters[:8].upper() + "-WP" + return None + compact = re.sub(r"[^a-z0-9]", "", slug) + if not compact: + return None + return compact[:8].upper() + "-WP" + + +def workplan_ids(repo_root: Path) -> list[tuple[str, str, str]]: + """Return (path, prefix, number) for each PREFIX-WP-NNNN workplan id.""" + found: list[tuple[str, str, str]] = [] + wp_dir = repo_root / "workplans" + if not wp_dir.is_dir(): + return found + for path in sorted(wp_dir.rglob("*.md")): + if path.name.startswith("ADHOC-"): + continue + fm, _ = parse_frontmatter(path.read_text(encoding="utf-8")) + ident = fm.get("id") + if ident is None: + continue + match = WP_ID_RE.match(str(ident).strip()) + if not match: + continue + found.append((str(path.relative_to(repo_root)), match.group(1) + "-WP", match.group(2))) + return found + + +def check_repository(repo_root: Path, *, slug: str | None = None) -> ConformanceReport: + repo_root = repo_root.resolve() + slug = slug or _slug(repo_root) + flavor = resolve_flavor(repo_root, slug=slug) + report = ConformanceReport(slug=slug, flavor=flavor) + + for note in flavor.disagreements: + report.findings.append( + Finding( + code="flavor-signal-disagreement", + severity="warning", + path=".repo-classification.yaml", + message=note, + canon=CANON_CLASSIFICATION, + ) + ) + + for rel in required_files(flavor): + target = repo_root / rel + present = target.is_dir() if rel.endswith("/") or rel == "workplans" else target.is_file() + if not present: + report.findings.append( + Finding( + code="required-file-missing", + severity="missing", + path=rel, + message=f"{rel} is required for flavor {flavor.flavor}", + canon=CANON_FLAVOR if flavor.prj_layout else CANON_CLASSIFICATION, + ) + ) + + has_intent = (repo_root / "INTENT.md").is_file() + has_goal = (repo_root / "GOAL.md").is_file() + if flavor.prj_layout and has_intent and has_goal: + report.findings.append( + Finding( + code="intent-and-goal", + severity="contradictory", + path="INTENT.md", + message="prj- flavor forbids shipping both INTENT.md and GOAL.md", + canon=CANON_FLAVOR, + ) + ) + if flavor.prj_layout and has_intent and not has_goal: + report.findings.append( + Finding( + code="intent-instead-of-goal", + severity="contradictory", + path="INTENT.md", + message="prj- flavor requires GOAL.md and must not use INTENT.md as the purpose document", + canon=CANON_FLAVOR, + ) + ) + if not flavor.prj_layout and has_intent and has_goal: + report.findings.append( + Finding( + code="intent-and-goal", + severity="warning", + path="GOAL.md", + message="durable flavors use INTENT.md; GOAL.md is the project-flavor purpose document", + canon=CANON_FLAVOR, + ) + ) + + if flavor.prj_layout and has_goal: + body = (repo_root / "GOAL.md").read_text(encoding="utf-8").lower() + missing = [name for name in GOAL_SECTIONS if name not in body] + if missing: + report.findings.append( + Finding( + code="goal-sections-missing", + severity="missing", + path="GOAL.md", + message="GOAL.md must include " + ", ".join(GOAL_SECTIONS) + f"; missing {missing}", + canon=CANON_FLAVOR, + ) + ) + + ids = workplan_ids(repo_root) + prefixes = {prefix for _path, prefix, _num in ids} + if FLAVOR_MARKER_PREFIX in prefixes: + report.findings.append( + Finding( + code="flavor-derived-prefix", + severity="contradictory", + path="workplans/", + message="PRJ-WP- is derived from the flavor marker; prefixes must come from project identity", + canon=CANON_FLAVOR, + ) + ) + if len(prefixes) > 1: + report.findings.append( + Finding( + code="multiple-prefixes", + severity="warning", + path="workplans/", + message=f"repository uses more than one workplan prefix: {sorted(prefixes)}", + canon="ADR-007", + ) + ) + identity = scan_record_identities(repo_root) + for item in identity["invalid_identifiers"]: + report.findings.append( + Finding( + code="work-record-id-invalid", + severity="contradictory", + path=item["source"], + message=f"{item['kind']} id {item['id']!r} is not accepted by the canon kind registry", + canon=identity["canon"], + ) + ) + for item in identity["duplicate_source_occurrences"]: + report.findings.append( + Finding( + code="work-record-duplicate-source-occurrence", + severity="warning", + path=item["sources"][0], + message=( + f"{item['kind']} {item['id']!r} repeats with UUID {item['uuid']}; " + f"index once and clean up sources {item['sources']}" + ), + canon=identity["canon"], + ) + ) + for item in identity["identity_collisions"]: + report.findings.append( + Finding( + code="work-record-identity-collision", + severity="contradictory", + path=item["sources"][0], + message=( + f"{item['kind']} {item['id']!r} has conflicting UUID assignments " + f"at {item['sources']}" + ), + canon=identity["canon"], + ) + ) + return report diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/time.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/time.py new file mode 100644 index 00000000..e3e2ee9a --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/time.py @@ -0,0 +1,26 @@ +"""Canonical UTC time helpers (RMGR-ADR-002).""" + +from __future__ import annotations + +from datetime import UTC, date, datetime + + +def utc_now() -> datetime: + """Return the current timezone-aware UTC instant.""" + return datetime.now(UTC) + + +def format_utc(value: datetime) -> str: + """Serialize an aware instant as canonical RFC 3339 UTC with ``Z``.""" + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("timestamp must be timezone-aware") + return value.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def utc_now_text() -> str: + return format_utc(utc_now()) + + +def utc_today() -> date: + """Return the calendar date derived from the current UTC instant.""" + return utc_now().date() diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/workloads.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/workloads.py new file mode 100644 index 00000000..69b110ba --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/src/repo_manager/workloads.py @@ -0,0 +1,238 @@ +"""Read-only index and resolver for authoritative rapp workload declarations.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +import yaml + +REFERENCE_CONTRACT = "helixforge.workload-reference/v1" + + +@dataclass(frozen=True) +class WorkloadRecord: + rapp_id: str + name: str + declaration_repo: str + declaration_path: str + ownership_repo: str | None + readiness_state: str | None + data_classification: str | None + criticality: str | None + deployables: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + data = asdict(self) + data["deployables"] = list(self.deployables) + data["reference"] = {"rapp_id": self.rapp_id, "name": self.name} + return data + + +def _error(code: str, path: Path, message: str) -> dict[str, str]: + return {"code": code, "path": str(path), "message": message} + + +def _package_dirs(root: Path) -> list[Path]: + root = root.expanduser().resolve() + if root.name.startswith("rapp-"): + return [root] + return sorted(path for path in root.glob("rapp-*") if path.is_dir()) + + +def _text(value: Any) -> str | None: + if not isinstance(value, str): + return None + stripped = value.strip() + return stripped or None + + +def index_workloads(root: Path) -> dict[str, Any]: + """Index authoritative declarations without inventing missing identity.""" + root = root.expanduser().resolve() + records: list[WorkloadRecord] = [] + errors: list[dict[str, str]] = [] + + for package_dir in _package_dirs(root): + declaration = package_dir / "declarations" / "rapp.yaml" + if not declaration.is_file(): + errors.append( + _error( + "declaration_missing", + declaration, + "rapp repository has no authoritative declarations/rapp.yaml", + ) + ) + continue + try: + raw = yaml.safe_load(declaration.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError) as exc: + errors.append(_error("declaration_invalid", declaration, str(exc))) + continue + if not isinstance(raw, dict): + errors.append( + _error("declaration_invalid", declaration, "declaration root must be a mapping") + ) + continue + if raw.get("kind") != "managed-workload-package": + errors.append( + _error( + "declaration_kind_invalid", + declaration, + "kind must be managed-workload-package", + ) + ) + continue + + rapp_id = _text(raw.get("rapp_id")) + identity = raw.get("workload_identity") + name = _text(identity.get("name")) if isinstance(identity, dict) else None + if not rapp_id or not name: + errors.append( + _error( + "identity_missing", + declaration, + "rapp_id and workload_identity.name are required", + ) + ) + continue + if rapp_id != package_dir.name: + errors.append( + _error( + "rapp_repo_mismatch", + declaration, + f"rapp_id {rapp_id!r} does not match declaration repo {package_dir.name!r}", + ) + ) + continue + + deployables: list[str] = [] + composition = raw.get("composition") + members = composition.get("member_repos", []) if isinstance(composition, dict) else [] + if not isinstance(members, list): + errors.append( + _error( + "composition_invalid", + declaration, + "composition.member_repos must be a list", + ) + ) + continue + for member in members: + if not isinstance(member, dict) or not isinstance(member.get("deployables"), list): + errors.append( + _error( + "composition_invalid", + declaration, + "every composition member must declare a deployables list", + ) + ) + continue + deployables.extend( + value for value in (_text(item) for item in member["deployables"]) if value + ) + if not deployables: + errors.append( + _error( + "deployables_missing", + declaration, + "the workload declaration has no running deployables", + ) + ) + continue + + records.append( + WorkloadRecord( + rapp_id=rapp_id, + name=name, + declaration_repo=package_dir.name, + declaration_path=str(declaration.relative_to(root)), + ownership_repo=_text(raw.get("ownership_repo")), + readiness_state=_text(raw.get("readiness_state")), + data_classification=_text(raw.get("data_classification")), + criticality=_text(raw.get("criticality")), + deployables=tuple(sorted(set(deployables))), + ) + ) + + references: dict[tuple[str, str], list[WorkloadRecord]] = {} + deployable_owners: dict[str, list[WorkloadRecord]] = {} + for record in records: + references.setdefault((record.rapp_id, record.name), []).append(record) + for deployable in record.deployables: + deployable_owners.setdefault(deployable, []).append(record) + for reference, owners in sorted(references.items()): + if len(owners) > 1: + errors.append( + _error( + "reference_duplicate", + root, + f"workload reference {reference!r} resolves to {len(owners)} declarations", + ) + ) + for deployable, owners in sorted(deployable_owners.items()): + if len(owners) > 1: + errors.append( + _error( + "deployable_duplicate", + root, + f"deployable {deployable!r} belongs to more than one rapp: " + + ", ".join(sorted(owner.rapp_id for owner in owners)), + ) + ) + + return { + "ok": not errors, + "contract": REFERENCE_CONTRACT, + "root": str(root), + "declaration_count": len(records), + "workloads": [record.to_dict() for record in sorted(records, key=lambda r: r.rapp_id)], + "errors": errors, + } + + +def resolve_workload( + root: Path, + *, + rapp_id: str, + name: str, + deployable: str | None = None, +) -> dict[str, Any]: + """Resolve an explicit reference, returning unknown instead of guessing.""" + index = index_workloads(root) + matches = [ + record + for record in index["workloads"] + if record["rapp_id"] == rapp_id and record["name"] == name + ] + reference: dict[str, str] = {"rapp_id": rapp_id, "name": name} + if deployable: + reference["deployable"] = deployable + if len(matches) != 1: + return { + "ok": True, + "contract": REFERENCE_CONTRACT, + "status": "unknown", + "reference": reference, + "reason": "not_found" if not matches else "ambiguous", + "index_errors": index["errors"], + } + workload = matches[0] + if deployable and deployable not in workload["deployables"]: + return { + "ok": True, + "contract": REFERENCE_CONTRACT, + "status": "unknown", + "reference": reference, + "reason": "deployable_not_declared", + "index_errors": index["errors"], + } + return { + "ok": True, + "contract": REFERENCE_CONTRACT, + "status": "resolved", + "reference": reference, + "workload": workload, + "index_errors": index["errors"], + } diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md new file mode 100644 index 00000000..595cd5c1 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md @@ -0,0 +1,177 @@ +--- +id: RAIL-BS-WP-0002 +type: workplan +title: "k3s and Kubernetes Platform Baseline" +domain: financials +repo: railiance-cluster +status: finished +owner: railiance +topic_slug: railiance +repo_goal_id: "70ab2379-fb9d-4fec-a09d-b2a717e4ace8" +state_hub_workstream_id: "4c63dfc6-9eac-4e79-9f77-8f644ad7147d" +created: "2026-03-09" +updated: "2026-03-10" +completed: "2026-03-10" +--- + +> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`). + +# k3s and Kubernetes Platform Baseline + +## Goal + +Install k3s, Helm, and the baseline Kubernetes services on the converged +HostEurope node. This workplan picks up exactly where `railiance-hosts` +leaves off: a hardened, verified OS node that is ready for Kubernetes. + +## Pre-condition + +`railiance-infra` converge + Goss verify must pass before any task here +is executed: + +```bash +# In railiance-infra/ +make converge +make verify # must exit 0 +``` + +## Boundary + +This repo owns everything from k3s upward. It must not re-configure items +defined in `railiance-infra/spec/server-baseline.yaml`. See ADR-003: +`railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`. + +**Out of scope here:** platform services (PostgreSQL, storage, identity) +→ `railiance-platform`. Application deployments (Gitea, coulomb services) +→ `railiance-apps`. + +--- + +## Tasks + +### T01 — Ansible playbook: install k3s (server mode) + +```task +id: T01 +status: done +priority: high +state_hub_task_id: "3f042630-eab0-4c6a-9167-e2b28ff20e40" +completed: "2026-03-10" +``` + +Harden `ansible/bootstrap.yml` to a production-ready k3s install: + +- Use the official k3s install script pinned to a specific version + (`INSTALL_K3S_VERSION=v1.35.1+k3s1`) +- `INSTALL_K3S_EXEC="server --cluster-init --write-kubeconfig-mode=644"` + (cluster-init enables embedded etcd for future HA expansion) +- Wait for node `Ready` before proceeding: + ```bash + k3s kubectl wait node --all --for=condition=Ready --timeout=120s + ``` +- Fetch kubeconfig to the control node as `~/.kube/config-hosteurope` + +**Done when:** `k3s kubectl get nodes` returns `Ready` from both the server +and the control node (via kubeconfig). + +--- + +### T02 — Helm installation + +```task +id: T02 +status: done +priority: high +state_hub_task_id: "e8510646-46ed-4697-a345-f3d3009eea78" +completed: "2026-03-10" +``` + +Add a task (or a role `roles/helm/`) that: + +1. Downloads the Helm binary (pinned version) to `/usr/local/bin/helm` +2. Verifies the checksum +3. Confirms `helm version` succeeds + +**Done when:** `helm version` succeeds on the HostEurope node. + +--- + +### T03 — Smoke test: k3s + Helm + +```task +id: T03 +status: done +priority: high +state_hub_task_id: "dab2c07f-8aa0-4635-8df6-857e87e93fc5" +completed: "2026-03-10" +``` + +Extend `tests/smoke_kube.sh` to assert: + +- `k3s kubectl get nodes` → node in Ready state +- `helm version` exits 0 +- CoreDNS pod running in `kube-system` +- Traefik ingress controller pod running (default in k3s) + +Run via: +```bash +ansible-playbook -i ansible/hosts.ini ansible/smoke.yml +``` +or directly over SSH if the kubeconfig is available locally. + +**Done when:** all assertions pass and the script exits 0. + +--- + +### T04 — Commit kubeconfig management notes + +```task +id: T04 +status: done +priority: medium +state_hub_task_id: "5c3d40e4-239b-488e-9519-6f7a38d2325f" +completed: "2026-03-10" +``` + +Document in `docs/kubeconfig.md`: + +- Where the kubeconfig is fetched to (`~/.kube/config-hosteurope`) +- How to merge it into `~/.kube/config` +- How to switch context: `kubectl config use-context default` +- Security note: kubeconfig is gitignored (contains cluster CA + client cert) + +**Done when:** doc written and committed. + +--- + +### T05 — Add `make k3s-install` and `make smoke` targets + +```task +id: T05 +status: done +priority: medium +state_hub_task_id: "7f9e0e58-a130-467a-a2d0-b3f2564e496f" +completed: "2026-03-10" +``` + +Add to Makefile (create one if none exists): + +```makefile +k3s-install: ## Install k3s and Helm on all inventory hosts + ansible-playbook -i ansible/hosts.ini ansible/bootstrap.yml + +smoke: ## Run Kubernetes smoke tests + bash tests/smoke_kube.sh +``` + +**Done when:** both targets work and are listed in `make help`. + +--- + +## References + +- Repo goal: `70ab2379-fb9d-4fec-a09d-b2a717e4ace8` (Install k3s and Kubernetes Baseline) +- Domain goal: `6f96c712-60e6-4ea9-ab06-168878eafbce` (Three-Phoenix Secure Kubernetes Infrastructure) +- Pre-condition: railiance-infra WP-0001 (Secure Single-Server Bootstrap) — completed 2026-03-09 +- Boundary ADR: `railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md` +- k3s releases: https://github.com/k3s-io/k3s/releases diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md new file mode 100644 index 00000000..fe7b21da --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md @@ -0,0 +1,196 @@ +--- +id: RAIL-BS-WP-0003 +type: workplan +title: "pgpool CrashLoopBackOff on PostgreSQL HA failover — missing secret key" +domain: financials +repo: railiance-cluster +status: finished +owner: tegwick +created: "2026-03-10" +updated: "2026-03-10" +state_hub_workstream_id: "7ee9ee22-1fae-4567-9194-8d70a9e0f45b" +--- + +> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`). + +# Bug Report: pgpool CrashLoopBackOff on PostgreSQL HA failover + +## Summary + +On 2026-03-10 a PostgreSQL HA failover caused all three postgresql pods to +restart. pgpool — the connection pooler between Gitea and PostgreSQL — then +entered CrashLoopBackOff and produced no logs. As a result Gitea's login +and all write operations hung indefinitely. The root page was still served +(from Valkey cache) which masked the failure. + +The fix was to patch a missing key in a Kubernetes secret. The root cause is +that the `gitea-12.2.0` Helm chart (postgresql-ha subchart v16.2.2) does not +populate the `pgpool-password` key in the `gitea-postgresql-ha-postgresql` +secret, even though the pgpool pod requires it at startup. + +--- + +## Timeline + +| Time (UTC) | Event | +|---|---| +| ~09:45 | postgresql-0, postgresql-2 pods restarted (repmgr failover) | +| ~09:45 | pgpool pod restarted and entered CrashLoopBackOff | +| ~11:00 | User noticed Gitea login hanging; home page still loading | +| ~13:00 | Root cause identified: missing `pgpool-password` secret key | +| ~13:10 | Secret patched; pgpool pod deleted and restarted cleanly | +| ~13:15 | Gitea fully operational | + +--- + +## Root Cause + +The Bitnami `pgpool` container startup script reads the file +`/opt/bitnami/pgpool/secrets/pgpool-password`, which is mounted from the +`gitea-postgresql-ha-postgresql` Kubernetes Secret via a `subPath` volume +mount. That secret key was never created by the Helm chart, so the file did +not exist. The container exited immediately with no logs. + +The pod had been running for 20 days without a restart, so this gap was +never discovered during initial deployment. + +--- + +## Evidence + +```bash +# Secret was missing the pgpool-password key +sudo k3s kubectl get secret -n default gitea-postgresql-ha-postgresql -o yaml +# data: keys were password, postgres-password, repmgr-password only +# pgpool-password was absent + +# pgpool pod describe showed 824 back-off restarts over 173 minutes +# No logs in either current or --previous output +sudo k3s kubectl logs -n default --previous +# (empty) + +# Gitea process had zero TCP connections to PostgreSQL port 5432 +# but many connections to Valkey port 6379 +cat /proc//net/tcp | grep 1538 # 1538 = 5432 hex — no results +``` + +--- + +## Immediate Fix Applied + +```bash +# Add the missing key (value = sr-check-password = changeme4 = base64: Y2hhbmdlbWU0) +sudo k3s kubectl patch secret -n default gitea-postgresql-ha-postgresql \ + --type='json' \ + -p='[{"op":"add","path":"/data/pgpool-password","value":"Y2hhbmdlbWU0"}]' + +# Restart pgpool +sudo k3s kubectl delete pod -n default +``` + +--- + +## Risk: Fix Will Be Lost on helm upgrade + +The patched secret is managed by Helm (annotation: +`meta.helm.sh/release-name: gitea`). A `helm upgrade` will regenerate the +secret from the chart template, which does not include `pgpool-password`, +and the bug will recur. + +--- + +## Tasks + +### T01 — Add pgpool-password to Helm values + +```task +id: T01 +status: done +priority: high +state_hub_task_id: "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc" +``` + +Create or update `helm/gitea-values.yaml` (or equivalent) to permanently +include the pgpool-password so it survives `helm upgrade`: + +```yaml +postgresql-ha: + postgresql: + pgpoolPassword: +``` + +**Done when:** `helm upgrade gitea` completes and pgpool starts cleanly +without manual secret patching. + +--- + +### T02 — Add pgpool health check to smoke test + +```task +id: T02 +status: done +priority: high +state_hub_task_id: "ab166073-30a7-4702-a037-4091e8706e20" +``` + +Extend `tests/smoke_kube.sh` to assert: + +```bash +# All postgresql-ha pods Running +kubectl get pods -n default | grep gitea-postgresql-ha | grep -v Running && exit 1 + +# pgpool specifically not in CrashLoopBackOff +kubectl get pod -n default -l app.kubernetes.io/component=pgpool \ + -o jsonpath='{.items[0].status.containerStatuses[0].state}' | grep -v crash +``` + +**Done when:** the smoke test catches a pgpool failure within 5 minutes. + +--- + +### T03 — Add HA failover test + +```task +id: T03 +status: done +priority: high +state_hub_task_id: "140da396-8e30-4f4d-b88c-c42c0cd46c01" +``` + +Create `tests/test_ha_failover.sh` that: + +1. Records Gitea login response time (baseline) +2. Kills the primary PostgreSQL pod: `kubectl delete pod gitea-postgresql-ha-postgresql-0 -n default` +3. Waits for repmgr to promote a replica (max 60s) +4. Asserts Gitea login POST still succeeds within 10s +5. Asserts pgpool pod is Running (not CrashLoopBackOff) +6. Asserts all postgresql pods return to Running + +This test must pass before any PostgreSQL HA deployment is considered done. + +**Done when:** script exits 0 against a live cluster. + +--- + +### T04 — Document the incident in docs/ + +```task +id: T04 +status: done +priority: medium +state_hub_task_id: "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d" +``` + +Add `docs/incidents/2026-03-10-pgpool-missing-secret.md` with the full +timeline, root cause, and fix, so future operators understand what happened +and how to recover. + +**Done when:** doc committed and linked from `docs/README.md`. + +--- + +## References + +- Bitnami postgresql-ha chart v16.2.2 +- Gitea Helm chart v12.2.0 +- Related decision: D3 (HA testing policy) in `DECISIONS.md` diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md new file mode 100644 index 00000000..89d14fff --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md @@ -0,0 +1,275 @@ +--- +id: RAIL-BS-WP-0004 +type: workplan +title: "Integrated Backup — S2 Kubernetes Runtime Layer" +domain: financials +repo: railiance-cluster +status: finished +owner: tegwick +topic_slug: railiance +state_hub_workstream_id: "7e8b0c20-51eb-40c9-9e3b-85dd380d7625" +created: "2026-02-25" +updated: "2026-03-26" +--- + +> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`). + +# Integrated Backup — S2 Kubernetes Runtime Layer + +## Goal + +Implement the Q3 (Operability & Resilience) integrated backup for +railiance-cluster (S2). Backs up what S2 owns — the Kubernetes runtime state — +encrypted with age, written to a local directory on the server. No external +dependencies required. + +## Architecture (Decision D4) + +Each railiance repo implements its own backup for what it owns. No central +backup service. See `DECISIONS.md` D4 for full rationale. + +**Standard interface every railiance repo must provide:** + +```bash +make backup # encrypt + write to /opt/backup/railiance// +make restore # restore from most recent local backup +``` + +Encryption: age, same key pair as SOPS secrets (`.sops.yaml` public key). +Output: `/opt/backup/railiance/cluster/` on the server. + +## What S2 (railiance-cluster) owns and must back up + +| Asset | Why it matters | +|---|---| +| k3s etcd snapshots | Full cluster state — all workloads, configs, secrets | +| Helm release values | Runtime values not in git (any manually applied overrides) | +| kubeconfig | Admin access to the cluster | + +**Not S2's responsibility:** +- Custodian State Hub DB → the-custodian owns this +- Operator workstation config (`.claude/`, `.gitconfig`) → operator's own concern +- Application data (Gitea repos, uploads) → S5 (railiance-apps) owns this +- PostgreSQL data volumes → S3 (railiance-platform) owns this + +## Encryption + +Reuse the age public key from `.sops.yaml`: + +```bash +AGE_PUBLIC_KEY=$(grep 'age:' .sops.yaml | awk '{print $2}') +tar -czf - | age -r "${AGE_PUBLIC_KEY}" -o backup.tar.gz.age +``` + +Decryption requires the private key at `~/.config/sops/age/keys.txt` +(same key used for `sops -d`). No additional key management needed. + +## Extension Point EP-RAIL-005 + +Once all five OAS layers implement this interface, the custodian can +orchestrate a full-stack backup with: + +```bash +for repo in railiance-infra railiance-cluster railiance-platform \ + railiance-enablement railiance-apps; do + make -C ~/$repo backup +done +``` + +No special protocol needed — just the standard interface. + +--- + +## Tasks + +### T01 — Define backup directory and encryption wrapper + +```task +id: T01 +status: done +priority: high +state_hub_task_id: "4526a842-ea31-4874-9231-92ab556cfe7b" +``` + +Create `tools/cmd/railiance-backup-s2` (replacing the old `railiance-backup`): + +- Backup dir: `/opt/backup/railiance/cluster/` (create with `mkdir -p`) +- Encrypt each artifact with age using public key from `.sops.yaml` +- Write timestamp-named files: `etcd-.snap.age`, `helm-values-.tar.gz.age`, `kubeconfig-.yaml.age` +- Keep last 7 of each type +- Write `.last-backup` stamp +- Exit 0 on success, non-zero on any failure +- No network required + +Also remove the old `tools/cmd/railiance-backup` (backed up Docker-based +custodian DB — wrong scope, not applicable to this server). + +**Done when:** `make backup` runs on COULOMBCORE without error and files +appear in `/opt/backup/railiance/cluster/`. + +--- + +### T02 — Back up k3s state (SQLite hot backup) + +```task +id: T02 +status: done +priority: high +state_hub_task_id: "a6313e06-1976-46a7-8e31-df4eb2eca880" +``` + +k3s has built-in etcd snapshot support: + +```bash +sudo k3s etcd-snapshot save --name railiance-$(date -u +%Y%m%dT%H%M%SZ) +# Default location: /var/lib/rancher/k3s/server/db/snapshots/ +``` + +Add to the backup script: take a fresh snapshot, encrypt with age, +copy to `/opt/backup/railiance/cluster/`. + +> **Note — verify etcd is in use before implementing:** +> `k3s etcd-snapshot` only works if k3s was started with `--cluster-init`. +> Without it, k3s uses SQLite and this command will fail. +> Verify first: `sudo k3s etcd-snapshot ls 2>&1` + +> **Note — sudo required:** etcd snapshot requires root. See T06 for how +> this is resolved (backup runs under root's crontab). + +**Done when:** backup includes a current etcd snapshot. + +--- + +### T03 — Back up Helm release values + +```task +id: T03 +status: done +priority: medium +state_hub_task_id: "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3" +``` + +Capture current runtime Helm values for all releases: + +```bash +KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm list -A -o json | \ + jq -r '.[].name + " " + .namespace' | \ + while read name ns; do + KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm get values "$name" -n "$ns" -o yaml + done +``` + +Tar and age-encrypt into `helm-values-.tar.gz.age`. + +> **Note — kubeconfig permissions:** `/etc/rancher/k3s/k3s.yaml` is root-readable +> only by default. The backup script must either run as root (see T06) or k3s +> must be configured with `--write-kubeconfig-mode=644`. Running as root +> (via root crontab) is the chosen approach — no config change needed. + +**Done when:** backup includes a snapshot of all Helm release values. + +--- + +### T04 — Back up kubeconfig + +```task +id: T04 +status: done +priority: medium +state_hub_task_id: "08233868-d522-4117-bc4e-6c0f52545665" +``` + +Age-encrypt `~/.kube/config-hosteurope` (or `/etc/rancher/k3s/k3s.yaml`) +into `kubeconfig-.yaml.age` in the backup directory. + +**Done when:** backup includes the encrypted kubeconfig. + +--- + +### T05 — make restore target + +```task +id: T05 +status: done +priority: medium +state_hub_task_id: "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8" +``` + +Add `tools/cmd/railiance-restore-s2` that decrypts and lists available +backups, with guided restore for the etcd snapshot case. + +Restore of etcd from snapshot: +```bash +sudo k3s server --cluster-reset \ + --cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/ +``` + +**Done when:** `make restore` prints available backups and a restore guide. + +--- + +### T06 — Install cron job and run restore drill + +```task +id: T06 +status: done +priority: medium +state_hub_task_id: "f8e4a094-c367-40eb-b895-da17bc144b07" +``` + +#### Solving the sudo problem + +The backup script needs root for two reasons: +- `k3s etcd-snapshot save` requires root +- `/etc/rancher/k3s/k3s.yaml` (kubeconfig) is root-readable only + +**Solution: run the cron under root's crontab.** + +This is the correct pattern for system-level backup jobs. It avoids a +proliferating sudoers whitelist (one entry per command, brittle to maintain) +and matches how tools like `rsnapshot`, `bacula`, and `borgbackup` work in +production. The backup writes to `/opt/backup/` which is root-owned anyway. + +Install the cron as root: + +```bash +sudo crontab -e +# Add: +0 2 * * * make -C /home/tegwick/railiance-cluster backup >> /opt/backup/railiance/cluster/backup.log 2>&1 +``` + +Note: use the absolute path to the repo — `~` does not expand reliably in +root's crontab unless HOME is set. + +Verify it is installed: +```bash +sudo crontab -l | grep railiance +``` + +#### Restore drill + +Once T01–T04 are done, run a decrypt-and-verify drill: + +```bash +# Decrypt the etcd snapshot and verify it is a valid snapshot file +sudo age -d -i ~/.config/sops/age/keys.txt \ + /opt/backup/railiance/cluster/etcd-$(ls /opt/backup/railiance/cluster/etcd-*.snap.age | sort -r | head -1 | xargs basename | sed 's/etcd-//;s/.snap.age//').snap.age \ + | file - + +# Record the drill +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) restore drill OK" \ + >> /opt/backup/railiance/cluster/restore-drill.log +``` + +**Done when:** cron installed under root, drill completes without error, +log entry written. + +--- + +## References + +- Decision D4: Integrated backup per capability (`DECISIONS.md`) +- Decision D2: Nextcloud as optional offsite extension (still valid, not a requirement) +- OAS Q3: Operability & Resilience +- Extension point EP-RAIL-005: Custodian full-stack backup orchestration +- k3s etcd snapshots: https://docs.k3s.io/datastore/backup-restore diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/test_cluster_frozen_history.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/test_cluster_frozen_history.py new file mode 100644 index 00000000..b5d94988 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/test_cluster_frozen_history.py @@ -0,0 +1,52 @@ +import shutil +from pathlib import Path +import pytest +from repo_manager.identifiers import ensure_missing_work_record_identifiers as ensure +from repo_manager.record_identity import scan_record_identities + +FIXTURE=Path(__file__).parent/'fixtures/railiance-cluster' +@pytest.fixture +def repo(tmp_path): + root=tmp_path/'railiance-cluster';shutil.copytree(FIXTURE,root) + live=root/'workplans/RCLUSTER-WP-0018.md';live.write_text('---\nid: RCLUSTER-WP-0018\nstatus: ready\n---\n\n```task\nid: RCLUSTER-WP-0018-T01\nstatus: todo\n```\n') + return root,live + +def test_exact_three_histories_preserve_15_bindings_and_strict_findings(repo): + root,live=repo;archive={p:p.read_bytes() for p in (root/'workplans/archived').glob('*.md')};before=scan_record_identities(root) + assert len(before['invalid_identifiers'])==15 and len(before['identity_collisions'])==5 + preview=ensure(root,execute=False);assert len(preview['assignments'])==2 + assert preview['historical_invalid_identifiers']==before['invalid_identifiers'] + assert preview['historical_identity_collisions']==before['identity_collisions'] + result=ensure(root,execute=True);assert len(result['assignments'])==2 + assert all(p.read_bytes()==raw for p,raw in archive.items()) + assert not scan_record_identities(root)['ok'] + assert ensure(root,execute=False)['assignments']==[] + +@pytest.mark.parametrize('change',['bytes','missing','symlink','live','nonterminal','uuid']) +def test_unknown_history_change_refuses_before_any_allocation(repo,change): + root,live=repo;p=next((root/'workplans/archived').glob('*.md'));before=live.read_bytes() + raw=p.read_bytes() + if change=='missing':p.unlink() + elif change=='symlink':p.unlink();target=root/'saved';target.write_bytes(raw);p.symlink_to(target) + elif change=='live':p.write_bytes(raw.replace(b'status: finished',b'status: active',1)) + elif change=='nonterminal':p.write_bytes(raw.replace(b'status: done',b'status: todo',1)) + elif change=='uuid':p.write_bytes(raw.replace(b'state_hub_task_id:',b'old_task_id:',1)) + else:p.write_bytes(raw+b'\nunreviewed change\n') + with pytest.raises(ValueError):ensure(root) + assert live.read_bytes()==before + +def test_any_unknown_or_live_source_in_collision_still_refuses(repo): + root,live=repo;before=live.read_bytes();other=root/'workplans/unknown.md' + other.write_text('---\nid: RCLUSTER-WP-0099\nstatus: active\n---\n\n```task\nid: T01\nstatus: todo\nstate_hub_task_id: 11111111-1111-4111-8111-111111111111\n```\n') + with pytest.raises(ValueError,match='identities are not safe'):ensure(root) + assert live.read_bytes()==before + +def test_other_repo_does_not_inherit_exact_frozen_exception(repo): + root,live=repo;other=root.with_name('other');root.rename(other) + with pytest.raises(ValueError,match='identities are not safe'):ensure(other) + +def test_qualified_collision_stays_global(repo): + root,live=repo;before=live.read_bytes() + (root/'workplans/other.md').write_text('---\nid: RCLUSTER-WP-0019\nstatus: active\n---\n\n```task\nid: RCLUSTER-WP-0018-T01\nstatus: done\nstate_hub_task_id: 11111111-1111-4111-8111-111111111111\n```\n') + with pytest.raises(ValueError,match='collisions='):ensure(root) + assert live.read_bytes()==before diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/test_custodian_legacy_identity.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/test_custodian_legacy_identity.py new file mode 100644 index 00000000..dff7a919 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/test_custodian_legacy_identity.py @@ -0,0 +1,130 @@ +from pathlib import Path + +from repo_manager.identifiers import ensure_missing_work_record_identifiers +from repo_manager.record_identity import scan_record_identities + + +def test_custodian_scoped_legacy_ids_are_grandfathered_and_parent_scoped(tmp_path: Path) -> None: + root = tmp_path + for number, workplan_id in enumerate( + ("CUST-WP-0000c", "CUST-WP-0010b", "CUST-WP-0017-LEGACY", "CUST-WP-0053-LEGACY"), start=1 + ): + _write_workplan( + root / "workplans" / f"{number}.md", + workplan_id, + f"{workplan_id}-T01", + f"{number:08d}-1111-4111-8111-111111111111", + ) + report = scan_record_identities(root) + assert report["invalid_identifiers"] == [] + assert report["identity_collisions"] == [] + assert report["duplicate_source_occurrences"] == [] + + +def test_full_legacy_bindings_dry_run_has_no_assignments(tmp_path: Path) -> None: + for number, workplan_id in enumerate( + ("CUST-WP-0000c", "CUST-WP-0010b", "CUST-WP-0017-LEGACY", "CUST-WP-0053-LEGACY"), start=1 + ): + _write_workplan( + tmp_path / "workplans" / f"{number}.md", + workplan_id, + f"{workplan_id}-T01", + f"{number:08d}-1111-4111-8111-111111111111", + ) + result = ensure_missing_work_record_identifiers(tmp_path, execute=False) + assert result["assignments"] == [] + assert result["files_changed"] == [] + + +def test_unqualified_task_remains_invalid_outside_allowlisted_parents(tmp_path: Path) -> None: + workplan = tmp_path / "workplans" / "CUST-WP-0099.md" + workplan.parent.mkdir() + workplan.write_text( + '---\nid: CUST-WP-0099\nstate_hub_workstream_id: "11111111-1111-4111-8111-111111111111"\n---\n' + '\n```task\nid: T01\nstatus: done\nstate_hub_task_id: "22222222-2222-4222-8222-222222222222"\n```\n', + encoding="utf-8", + ) + report = scan_record_identities(tmp_path) + assert report["invalid_identifiers"][0]["id"] == "T01" + + +def _write_workplan(path: Path, workplan_id: str, task_id: str, task_uuid: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + f'---\nid: {workplan_id}\nstate_hub_workstream_id: "11111111-1111-4111-8111-111111111111"\n---\n' + f'\n```task\nid: {task_id}\nstatus: done\nstate_hub_task_id: "{task_uuid}"\n```\n', + encoding="utf-8", + ) + + +def test_canonical_qualified_task_collision_remains_global(tmp_path: Path) -> None: + _write_workplan( + tmp_path / "workplans" / "a.md", + "CUST-WP-0099", + "CUST-WP-0099-T01", + "22222222-2222-4222-8222-222222222222", + ) + _write_workplan( + tmp_path / "workplans" / "b.md", + "CUST-WP-0100", + "CUST-WP-0099-T01", + "33333333-3333-4333-8333-333333333333", + ) + report = scan_record_identities(tmp_path) + assert report["identity_collisions"][0]["id"] == "CUST-WP-0099-T01" + + +def test_same_parent_local_alias_collision_fails_closed(tmp_path: Path) -> None: + path = tmp_path / "workplans" / "CUST-WP-0006.md" + path.parent.mkdir() + path.write_text( + '---\nid: CUST-WP-0006\nstate_hub_workstream_id: "11111111-1111-4111-8111-111111111111"\n---\n' + '\n```task\nid: T01\nstatus: done\nstate_hub_task_id: "22222222-2222-4222-8222-222222222222"\n```\n' + '\n```task\nid: T01\nstatus: done\nstate_hub_task_id: "33333333-3333-4333-8333-333333333333"\n```\n', + encoding="utf-8", + ) + report = scan_record_identities(tmp_path) + assert report["identity_collisions"][0]["id"] == "T01" + + +def test_unknown_alias_and_missing_managed_uuid_are_invalid(tmp_path: Path) -> None: + _write_workplan( + tmp_path / "workplans" / "unknown.md", + "CUST-WP-0006", + "T99", + "22222222-2222-4222-8222-222222222222", + ) + _write_workplan(tmp_path / "workplans" / "missing.md", "CUST-WP-0006", "T01", "") + report = scan_record_identities(tmp_path) + assert {item["id"] for item in report["invalid_identifiers"]} == {"T99", "T01"} + + +def test_all_38_local_aliases_preserve_binding_and_source_bytes(tmp_path): + before = {} + for parent, count in ( + ("CUST-WP-0006", 5), + ("CUST-WP-0007", 17), + ("CUST-WP-0030", 9), + ("CUST-WP-0031", 7), + ): + path = tmp_path / "workplans" / (parent + ".md") + path.parent.mkdir(exist_ok=True) + body = f'---\nid: {parent}\nstate_hub_workstream_id: "11111111-1111-4111-8111-111111111111"\n---\n' + for n in range(1, count + 1): + body += f'\n```task\nid: T{n:02d}\nstatus: done\nstate_hub_task_id: "{int(parent[-4:]):08d}-1111-4111-8111-{n:012d}"\n```\n' + path.write_text(body) + before[path] = path.read_bytes() + assert scan_record_identities(tmp_path)["ok"] + result = ensure_missing_work_record_identifiers(tmp_path, execute=True) + assert result["assignments"] == [] and result["files_changed"] == [] + assert all(path.read_bytes() == body for path, body in before.items()) + + +def test_new_suffix_task_is_not_grandfathered(tmp_path): + _write_workplan( + tmp_path / "workplans" / "new.md", + "CUST-WP-0000c", + "CUST-WP-0000c-T99", + "22222222-2222-4222-8222-222222222222", + ) + assert not scan_record_identities(tmp_path)["ok"] diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/test_historical_sources.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/test_historical_sources.py new file mode 100644 index 00000000..4fea5791 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/test_historical_sources.py @@ -0,0 +1,56 @@ +"""Exact frozen sources retain bytes and bindings while new work stays canonical.""" +import hashlib +from pathlib import Path + +import pytest + +from repo_manager import historical_sources as hs +from repo_manager.identifiers import ensure_missing_work_record_identifiers +from repo_manager.record_identity import scan_record_identities + + +def _fixture(tmp_path,monkeypatch,repo='activity-core'): + root=tmp_path/repo + path=root/'workplans/old.md';path.parent.mkdir(parents=True) + rid='custodian-WP-0001' if repo=='activity-core' else 'CUST-WP-ADHOC-2026-08-25-RETIRED' + data=(f'---\nid: {rid}\ntype: workplan\nrepo: {repo}\nstatus: finished\n' + 'state_hub_workstream_id: e22516cc-4fdf-4dad-a928-a9e04a9fa7c7\n---\n').encode() + path.write_bytes(data) + monkeypatch.setattr(hs,'BINDINGS',[dict(repo=repo,path='workplans/old.md',sha256=hashlib.sha256(data).hexdigest())]) + prefix='ACTIVITY' if repo=='activity-core' else 'CUST' + current=root/'workplans/current.md' + current.write_text(f'---\nid: {prefix}-WP-9999\ntype: workplan\nstatus: active\n---\n') + return root,path,current,data + + +@pytest.mark.parametrize('repo',['activity-core','the-custodian']) +def test_frozen_source_retains_bytes_and_existing_uuid(tmp_path,monkeypatch,repo): + root,path,current,data=_fixture(tmp_path,monkeypatch,repo) + report=ensure_missing_work_record_identifiers(root) + assert path.read_bytes()==data + assert len(report['historical_invalid_identifiers'])==1 + assert len(report['assignments'])==1 and report['assignments'][0]['path']=='workplans/current.md' + assert not scan_record_identities(root)['ok'] + assert ensure_missing_work_record_identifiers(root)['assignments']==[] + + +@pytest.mark.parametrize('change',['id','uuid','status','missing','symlink','extra']) +def test_unreviewed_history_refuses_before_any_allocation(tmp_path,monkeypatch,change): + root,path,current,data=_fixture(tmp_path,monkeypatch) + original=current.read_bytes() + if change=='id':path.write_bytes(data.replace(b'custodian-WP-0001',b'custodian-WP-0002')) + elif change=='uuid':path.write_bytes(data.replace(b'e22516cc',b'f22516cc')) + elif change=='status':path.write_bytes(data.replace(b'finished',b'active')) + elif change=='missing':path.unlink() + elif change=='symlink': + path.unlink();target=root/'other.md';target.write_bytes(data);path.symlink_to(target) + else:(root/'workplans/unreviewed.md').write_bytes(data.replace(b'custodian-WP-0001',b'custodian-WP-0099')) + with pytest.raises(ValueError):ensure_missing_work_record_identifiers(root) + assert current.read_bytes()==original + + +def test_other_repo_cannot_inherit_frozen_exception(tmp_path,monkeypatch): + root,path,current,data=_fixture(tmp_path,monkeypatch) + other=root.with_name('other-repo');root.rename(other) + with pytest.raises(ValueError,match='identities are not safe'): + ensure_missing_work_record_identifiers(other) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/test_historical_sync_validation.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/test_historical_sync_validation.py new file mode 100644 index 00000000..7e23077a --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/tests/test_historical_sync_validation.py @@ -0,0 +1,102 @@ +from pathlib import Path + +import pytest + +from repo_manager.identifiers import ensure_missing_work_record_identifiers +from repo_manager.record_identity import scan_record_identities + + +def _sources(root: Path, *, status='completed', task_status='done', + task_id='NK-WP-0003-T08a', binding='22222222-2222-4222-8222-222222222222'): + archive = root / 'workplans' / 'archived' / 'NK-WP-0003.md' + archive.parent.mkdir(parents=True) + archive.write_text( + f'---\nid: NK-WP-0003\nstatus: {status}\n' + 'state_hub_workstream_id: "11111111-1111-4111-8111-111111111111"\n---\n' + f'\n```task\nid: {task_id}\nstatus: {task_status}\n' + f'state_hub_task_id: "{binding}"\n```\n' + ) + live = root / 'workplans' / 'NK-WP-0099.md' + live.write_text('---\nid: NK-WP-0099\nstatus: active\n---\n' + '\n```task\nid: NK-WP-0099-T01\nstatus: todo\n```\n') + return archive, live + + +@pytest.mark.parametrize('status', ['finished', 'archived', 'completed']) +def test_historical_warning_preserves_source_and_allows_allocation(tmp_path, status): + archive, live = _sources(tmp_path, status=status) + before = {p: p.read_bytes() for p in (archive, live)} + preview = ensure_missing_work_record_identifiers(tmp_path, execute=False) + assert all(p.read_bytes() == content for p, content in before.items()) + result = ensure_missing_work_record_identifiers(tmp_path) + assert result['assignments'] == preview['assignments'] + assert {x['record_id'] for x in result['assignments']} == {'NK-WP-0099', 'NK-WP-0099-T01'} + assert archive.read_bytes() == before[archive] + assert result['historical_invalid_identifiers'] == scan_record_identities(tmp_path)['invalid_identifiers'] + assert result['historical_invalid_identifiers'][0]['id'] == 'NK-WP-0003-T08a' + assert not scan_record_identities(tmp_path)['ok'] + assert ensure_missing_work_record_identifiers(tmp_path)['assignments'] == [] + + +@pytest.mark.parametrize('kwargs', [ + {'status': 'active'}, {'status': 'unknown'}, {'task_status': 'todo'}, + {'binding': ''}, {'binding': 'not-a-uuid'}, {'task_id': 'T08a'}, + {'task_id': 'T01'}, {'task_id': 'NK-WP-0099-T08a'}, +]) +def test_unsafe_history_refuses_before_any_allocation(tmp_path, kwargs): + paths = _sources(tmp_path, **kwargs) + before = {p: p.read_bytes() for p in paths} + with pytest.raises(ValueError, match='identities are not safe'): + ensure_missing_work_record_identifiers(tmp_path) + assert all(p.read_bytes() == content for p, content in before.items()) + + +def test_historical_collision_still_blocks(tmp_path): + archive, live = _sources(tmp_path) + duplicate = archive.with_name('duplicate.md') + duplicate.write_text(archive.read_text().replace( + '22222222-2222-4222-8222-222222222222', + '33333333-3333-4333-8333-333333333333')) + before = live.read_bytes() + with pytest.raises(ValueError, match='collisions='): + ensure_missing_work_record_identifiers(tmp_path) + assert live.read_bytes() == before + + +def _railiance_item(root: Path, *, parent="RAILIANCE-WP-0004", status="finished", + task_status="done", task_id="RAILIANCE-WP-0004-I01", + binding="44444444-4444-4444-8444-444444444444") -> Path: + path = root / "workplans" / "archived" / "railiance.md" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + f'---\nid: {parent}\nstatus: {status}\n' + 'state_hub_workstream_id: "33333333-3333-4333-8333-333333333333"\n---\n' + f'\n```task\nid: {task_id}\nstatus: {task_status}\n' + f'state_hub_task_id: "{binding}"\n```\n', + encoding="utf-8", + ) + return path + + +def test_railiance_historical_item_labels_preserve_uuid(tmp_path: Path) -> None: + path = _railiance_item(tmp_path) + before = path.read_bytes() + result = ensure_missing_work_record_identifiers(tmp_path, execute=False) + assert result["assignments"] == [] + assert result["files_changed"] == [] + assert path.read_bytes() == before + + +@pytest.mark.parametrize("kwargs", [ + {"status": "active"}, + {"task_status": "todo"}, + {"parent": "RAILIANCE-WP-0005", "task_id": "RAILIANCE-WP-0005-I01"}, + {"binding": ""}, + {"binding": "not-a-uuid"}, + {"task_id": "RAILIANCE-WP-0004-I08"}, + {"task_id": "OTHER-WP-0004-I01"}, +]) +def test_railiance_item_label_preservation_is_bounded(tmp_path: Path, kwargs) -> None: + _railiance_item(tmp_path, **kwargs) + with pytest.raises(ValueError, match="identities are not safe"): + ensure_missing_work_record_identifiers(tmp_path) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/workplans/RMGR-WP-0019-historical-source-admission.md b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/workplans/RMGR-WP-0019-historical-source-admission.md new file mode 100644 index 00000000..fcfa4b3f --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/repo-manager/workplans/RMGR-WP-0019-historical-source-admission.md @@ -0,0 +1,207 @@ +--- +id: RMGR-WP-0019 +type: workplan +title: "Admit exact historical source views after receiving preservation support" +domain: infotech +repo: repo-manager +status: active +owner: codex +created: "2026-10-04" +updated: "2026-10-17" +flavor: implementation +depends_on: [] +state_hub_workstream_id: "efe0ecb2-2f3f-5f49-baee-b90ad1d353ab" +--- + +## Goal + +Unblock normal authoritative registration of the exact closed Activity legacy +source views and Custodian retired ADHOC source after receiving preserves their +existing records. Preserve every legacy source byte, label and UUID; do not +expand canonical syntax or the general grandfathering rules. CUST-WP-0077 owns +cross-owner coordination with STATE-WP-0095. + +## Review exact historical receiving contract + +```task +id: RMGR-WP-0019-T01 +status: done +priority: high +state_hub_task_id: "d78a622e-39f9-5923-b4e1-f70fdc17fb42" +``` + +Seven Activity sources yield four existing parents/64 closed bound tasks; +three split views repeat the archived parent. One Custodian retired source keeps +its existing parent/task IDs. Earlier /tmp/activity-core-bound-history-write.py +is superseded and must not be applied: registrar bypass alone would admit an +unsupported receiving projection. RMGR-WP-0014/0015 are finished and do not own +this new implementation demand; this workplan is not residual. + +## Enable exact frozen registrar exceptions after receiving deployment + +```task +id: RMGR-WP-0019-T02 +status: done +priority: high +depends_on: [STATE-WP-0095-T03] +state_hub_task_id: "95240c81-1805-5365-9b35-d238621a16bf" +``` + +Prepared code permits only source paths with reviewed full SHA256 digests in +repo_manager/historical_sources.py. Strict conformance scans retain their legacy +findings. Unknown/missing/altered source refuses before allocation. Source UUIDs +are neither changed nor allocated for these histories. Receiving and registrar +manifests must remain identical. Apply helper requires a deployment receipt. + +## Run normal owner sync and verify preserved identities + +```task +id: RMGR-WP-0019-T03 +status: done +priority: high +depends_on: [RMGR-WP-0019-T02] +state_hub_task_id: "682eea14-a4a8-5f33-9a9f-8f2f1ac8288d" +``` + +Use rmgr sync --path --push. No deep-check repair or direct hub UUID +patch substitutes for receiving. Historical unrelated stale projections are not +retirement-authorized by this workplan. Feed receipts back to STATE-WP-0095-T04. + +## Offline evidence, 2026-10-04 + +Prepared registrar suite: 38 focused cases passed (identifiers, registrar, +Custodian legacy identity and exact frozen source cases). Read-only execute=False +against both current owning repos accepted exact histories without source writes. +Implementation is staged; deployment gate is unresolved. + +## Approved deployment and actual registrar verification, 2026-10-04 + +STATE-WP-0095-T03 returned verified main-5dd3df9 / Helm revision72. +Global task-holder refusal passed inside the delivered API against the retired +Custodian parent, with transaction rollback and no committed probe. Five +historical parents/65 tasks retained their entire row snapshots. The gated +three-file registrar patch committed as c41d36c2ef8930628221ef36ec81d8b5d2145ed6; +38 actual owning-source cases passed and Repo Manager primary sync applied. + +Normal owner sync admitted all exact frozen histories without changing their +source bytes, UUIDs, parent ownership or row content. Custodian receiving +applied. Activity receiving refused only three empty duplicate projections +that would be retired; no retirement acknowledgement was issued. Registrar +allocated four missing IDs in ordinary Activity ad-hoc sources through its +normal path; frozen historical source files stayed unchanged. T03 remains wait +for explicit disposition and bounded receiving support under STATE-WP-0095-T04 +(decision91b7c633-a66f-5d43-9f4b-6565aefb9356). +Evidence: docs/evidence/2026-10-04-historical-registrar-verification.json. + +## Ungated T03 preparation, 2026-10-06 + +Re-ran the registrar suite at 8a47a5c (registrar source unchanged since +c41d36c2): 89 focused registrar/identity/historical cases and the full suite +(226) passed. Read-only hub check: the three refused projections are +683724d9 (custodian-wp-0003), 644017c7 (custodian-wp-0003a) and a5806df9 +(custodian-wp-0003b), each with zero tasks; every custodian-WP-0003* source +file binds parent b4eb45a9, which holds the tasks and must not be retired. +Added repo_manager.retirement_disposition, a read-only check that an Activity +expected_retirement_uuids set equals exactly those three empty projections and +excludes the preserved parent. It authorizes nothing; retirement stays with +Activity receiving under STATE-WP-0095-T04. + +## Activity disposition received, 2026-10-06 + +activity-core returned the exact expected_retirement_uuids (message 17ba4201, +evidence activity-core 941fac0, ACTIVITY-WP-0045): 683724d9, 644017c7, +a5806df9. repo_manager.retirement_disposition accepted the set: it equals the +three projections the 2026-10-04 run refused, each holds zero tasks, and it +excludes preserved parent b4eb45a9. Nothing was applied. T03 stays wait: its +acceptance needs a successful Activity owner sync with receipts, which can run +only after the founder decides 91b7c633 and STATE-WP-0095-T04 retires exactly +this set in a reviewed live rollout. Then re-run rmgr sync --path +~/activity-core --push and feed the receipt back to STATE-WP-0095-T04. + +## Post-retirement owner sync attempt, 2026-10-06 + +State Hub reported decision 91b7c633 resolved, main-968a811 / Helm rev 74 +deployed, and Activity reconcile at 67a92d2 retiring exactly the three empty +projections (3 retired, 0 created, 0 refused; 4 historical parents / 64 tasks +unchanged). Evidence: state-hub docs/evidence/2026-10-06-activity-stale-retirement-rollout.json. +`rmgr sync --path ~/activity-core --push` stopped before reconciliation with +status pending_commit: uncommitted `state_hub_workstream_id` / `state_hub_task_id` +writeback in activity-core workplans/ACTIVITY-WP-0045-stale-historical-projection-disposition.md +(plus WORK-RECORDS.md). No hub request was made. Repo Manager does not commit in +activity-core; T03 stays wait until Activity Core commits and the sync re-runs. + +Re-run 2026-10-06 08:42Z after Activity Core committed the writeback (e809fb2): +status applied, created 0 / updated 45 (ACTIVITY-WP only) / retired 0 / refused 0, +no task changes, notes preserve custodian-WP-0001..0004 UUIDs; hub shows 4 parents, +64 tasks. Not closed: historical parent b4eb45a9 (custodian-WP-0003, source +`status: archived`, evidence snapshot archived) now reads `finished` in the hub, +updated 08:42:03Z during this sync. Needs State Hub review before T03 closes. + +Closed 2026-10-06: State Hub traced the b4eb45a9 status change to its own C-04 +check, which ran once per file. The custodian-WP-0003a/b/c split views share the +parent UUID, so whichever view ran last set the row. The Forge reconcile did not +cause it. Fixed in state-hub 2b5d8f9 (STATE-WP-ADHOC-2026-10-06-T01); b4eb45a9 is +back to `archived`. Re-check: 4 parents (b4eb45a9 archived; 3a4f47d9, 759b1255, +e22516cc finished) holding 64 tasks (24+12+7+21). Receipt: Activity sync at e809fb2, +primary/railiance01, applied, 0 created / 45 updated (ACTIVITY-WP only) / 0 retired +/ 0 refused, 0 task changes, historical UUIDs preserved. Evidence: +state-hub docs/evidence/2026-10-06-activity-stale-retirement-rollout.json. + + +## Cluster exact historical source successor, 2026-10-17 + +This scoped extension preserves the completed Activity/Custodian work above. +The cluster publisher now refuses three closed RAIL-BS source files containing +15 existing UUID-bound bare task labels. Preserve all three source bytes, IDs, +and UUIDs. No new alias or general syntax exemption is introduced. Evidence and +exact receiver-first proposal: cluster-historical-publication-r1 in the Custodian +NetKingdom change packet. The existing SHR-WP-0001 project driver owns the +Repo Manager / State Hub cross-owner wait; NK-WP-0049-T35 benefits from the +blocked cluster publication. No worker or human gate is woken here. + +## Prepare exact cluster registrar successor + +```task +id: RMGR-WP-0019-T04 +status: done +priority: high +``` + +Prepared full-file SHA256 bindings for three archives and 15 exact task +occurrences; strict scans still return 15 invalid labels and five collisions. +Unknown/live sources, changed/missing/symlinked histories, UUID drift and +qualified/global collisions refuse before allocation. Candidate tests pass 47; +canonical application remains a separate reviewed step. The read-only current +cluster preview proposes seven ordinary UUID bindings in two files and changes +no bytes. One pre-existing ADHOC field is included explicitly: task +RCLUSTER-WP-ADHOC-2026-07-27-T01 in workplans/ADHOC-2026-07-27.md receives +3e3f39d3-5070-5ba3-97db-19f5f361f264 through managed allocation only. + +## Admit cluster registrar successor only after receiving verification + +```task +id: RMGR-WP-0019-T05 +status: wait +priority: high +depends_on: [STATE-WP-0095-T06] +``` + +Retain the current guard until exact reviewed receiving source has been delivered +and verified through State Hub's normal owner deployment. Match both manifests, +then integrate reviewed registrar source and verify ordinary allocator preview. +No direct API patch, history rename or bypass substitutes for this gate. + +## Run ordinary cluster owner publication and verify preservation + +```task +id: RMGR-WP-0019-T06 +status: wait +priority: high +depends_on: [RMGR-WP-0019-T05] +``` + +Only then run normal owner consistency/publication with the reviewed expected +file set: RCLUSTER-WP-0018 and the one-field ADHOC managed backfill, plus ordinary +generated indexes. Preserve all three archived files and all 18 existing +parent/task rows. Return actual receiving/publication receipts to STATE-WP-0095-T07. +Do not infer TLS recovery, credential custody or native acceptance from sync. diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/actual-readonly-preflight.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/actual-readonly-preflight.json new file mode 100644 index 00000000..da0ff838 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/actual-readonly-preflight.json @@ -0,0 +1,26 @@ +{ + "schema": "receiver-readonly-preflight.v1", + "observed_at": "2026-10-16T23:32:16.582701+00:00", + "source_revision": "51bc70d1ff8f25450a8650d8e50caf053093a02e", + "image_tag": "main-51bc70d", + "production_mutations": false, + "secret_reads": false, + "limitations": [ + "Chart/template server admission is not an exact Helm reuse-values upgrade diff.", + "Headroom is dated observation, not a scheduling guarantee or rollout approval.", + "No migration, pod restart, registry pull, live probe transaction or Helm command executes." + ], + "target_node_confirmed": true, + "headroom_exit": 0, + "headroom_ok": true, + "headroom_summary": { + "ok": true, + "reasons": [], + "notes": [ + "Aggregate remaining millicores is not a kube-scheduler guarantee.", + "Preflight does not lower requests and does not start Helm." + ] + }, + "server_dryrun_exit": 2, + "status": "server_dryrun_refused_or_unverified" +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/actual-readonly-preflight.stderr.txt b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/actual-readonly-preflight.stderr.txt new file mode 100644 index 00000000..e69de29b diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/bounded-preflight-commands.md b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/bounded-preflight-commands.md new file mode 100644 index 00000000..35aaa60a --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/bounded-preflight-commands.md @@ -0,0 +1,55 @@ +# Source-only publication and later read-only preflight + +These helpers contain no Helm upgrade/rollback, kubectl mutation without dry-run, +Repo Manager admission or work-record sync. No credential input is accepted. +Do not call execution switches before root's independent source review and +canonical integration. Normal hooks/scanner remain enabled. + +From this r2 directory, after reviewed patch application: + +```sh +python3 publish_receiver_source.py +python3 publish_receiver_source.py --execute-source-commit --push-source +``` + +The second command changes only the explicitly hashed State Hub source paths, +commits and pushes normally. It preserves unrelated index entries and working +brief/index/evidence bytes, checks exact committed path set, and verifies the +remote main ref. A failure never retries or deploys. Retain its sanitized JSON +receipt. It deliberately fails if HEAD or reviewed source bytes drift; prepare +a new reviewed revision instead of changing its expected hashes silently. + +After successful commit/push, use the returned full40character source revision: + +```sh +python3 lookup_ci_metadata.py --source-revision FULL40CHAR_SOURCE_REVISION +python3 preflight_receiver.py --source-revision FULL40CHAR_SOURCE_REVISION +``` + +The CI lookup uses State Hub's existing public actions/tasks endpoint and public +registry manifest HEAD. It requires exact head_sha build-and-push success plus +valid sha256 image manifest digest. A chart/docs-only commit can have no image; +unavailable/public-auth refusal stays unverified without a login fallback. +Task results and manifest digest alone do not establish image internal files or +floating hub-core provenance. Capture normal image metadata and compare actual +receiver module SHA, migration source/head, hub-core package/source SHA and +installed package inventory before native approval. Image fetch/inspection +must use the existing approved owner path and never execute the image or +surface bundled credentials. This packet introduces no new credential need. + +The preflight pins the existing kubeconfig, confirms railiance01 node IP before +invocation, runs the owner's headroom script (45second timeout) and server dry-run +(60second timeout), and emits fixed metadata only. It retains no raw stderr or +manifest output. It must be run close to a later rollout: dated headroom must +not be reused after workload changes. The owner Make dry-run also admits public +Secret templates and platform resources with dry-run only, never existing Secret +contents. Its values file is a public intended baseline, while real deployment +uses Helm --reuse-values. Therefore exact upgrade diff still needs a separately +reviewed values-preserving Helm dry-run; these checks do not prove equivalence. +No Helm values/manifest dump should be pasted or retained as public evidence. + +The image change also includes migration c4d5e6f7a8b9. Require disposable real +PostgreSQL migration and old-image-on-new-schema compatibility tests, migration +lock/time estimates and a verified recovery plan before approval. A Helm atomic +rollback does not downgrade schema, and blindly executing Alembic downgrade +would discard feed data. No migration or new native trial was executed here. diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/current-base-public-ci-metadata.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/current-base-public-ci-metadata.json new file mode 100644 index 00000000..5b0bc53e --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/current-base-public-ci-metadata.json @@ -0,0 +1,23 @@ +{ + "schema": "receiver-ci-metadata.v1", + "source_revision": "4ab66e363ed00a67ad1ea0434854bc4284c7a26e", + "image_tag": "main-4ab66e3", + "no_credentials": true, + "no_logs": true, + "ci_lookup_status": "ok", + "exact_revision_build_tasks": [ + { + "id": 12477, + "name": "build-and-push", + "status": "success", + "head_sha": "4ab66e363ed00a67ad1ea0434854bc4284c7a26e", + "started_at": null, + "completed_at": null + } + ], + "image_lookup_status": "public_access_refused", + "status": "artifact_unverified", + "limitations": [ + "Successful task+registry digest does not prove floating dependency closure or delivered image file hashes." + ] +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/lookup_ci_metadata.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/lookup_ci_metadata.py new file mode 100644 index 00000000..5b20ff98 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/lookup_ci_metadata.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Public-only bounded CI/image metadata. No credential lookup/login/logs.""" +import argparse +import json +import re +import urllib.error +import urllib.request + +LIMIT = 512 * 1024 +TASKS = 'https://forgejo.coulomb.social/api/v1/repos/coulomb/state-hub/actions/tasks?limit=50' +REGISTRY = 'https://forgejo.coulomb.social/v2/coulomb/state-hub/manifests/' + +def request(url, method='GET', headers=None): + try: + with urllib.request.urlopen(urllib.request.Request(url, method=method, headers=headers or {}), timeout=30) as response: + raw = response.read(LIMIT+1) if method == 'GET' else b'' + if len(raw) > LIMIT: + return {'status': 'oversized_refused'} + return {'status': 'ok', 'http_status': response.status, 'body': raw, + 'digest': response.headers.get('Docker-Content-Digest'), + 'content_type': response.headers.get('Content-Type')} + except urllib.error.HTTPError as exc: + return {'status': 'public_access_refused' if exc.code in (401,403) else 'http_error', 'http_status': exc.code} + except (urllib.error.URLError, TimeoutError): + return {'status': 'metadata_unavailable'} + +def lookup(revision): + if not re.fullmatch(r'[0-9a-f]{40}',revision): + raise ValueError('exact40char_revision_required') + result={'schema':'receiver-ci-metadata.v1','source_revision':revision, + 'image_tag':'main-'+revision[:7], 'no_credentials':True,'no_logs':True} + tasks=request(TASKS) + result['ci_lookup_status']=tasks['status'] + result['exact_revision_build_tasks']=[] + if tasks['status']=='ok': + try: + parsed=json.loads(tasks['body']) + for task in parsed.get('workflow_runs',[]): + if task.get('head_sha')==revision and task.get('name')=='build-and-push': + result['exact_revision_build_tasks'].append({k:task.get(k) for k in ('id','name','status','head_sha','started_at','completed_at')}) + except (ValueError,TypeError,AttributeError): + result['ci_lookup_status']='malformed_refused' + image=request(REGISTRY+result['image_tag'],'HEAD',{'Accept':'application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'}) + result['image_lookup_status']=image['status'] + digest=image.get('digest') + if image['status']=='ok' and isinstance(digest,str) and re.fullmatch(r'sha256:[0-9a-f]{64}',digest): + result['image_digest']=digest + result['status']='artifact_metadata_verified' if any(t.get('status')=='success' for t in result['exact_revision_build_tasks']) and 'image_digest' in result else 'artifact_unverified' + result['limitations']=['Successful task+registry digest does not prove floating dependency closure or delivered image file hashes.'] + return result + +if __name__=='__main__': + ap=argparse.ArgumentParser();ap.add_argument('--source-revision',required=True);args=ap.parse_args() + result=lookup(args.source_revision);print(json.dumps(result,indent=2));raise SystemExit(0 if result['status']=='artifact_metadata_verified' else 1) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/managed-statehub-uuid-assignment.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/managed-statehub-uuid-assignment.json new file mode 100644 index 00000000..a725e7ad --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/managed-statehub-uuid-assignment.json @@ -0,0 +1,31 @@ +{ + "schema": "repo-manager.identifier-assignment.v1", + "ok": true, + "executed": true, + "repo": "state-hub", + "namespace": "helixforge", + "assignments": [ + { + "kind": "task", + "record_id": "STATE-WP-0095-T05", + "uuid": "339b9a37-181c-5599-9a89-0393e199bd4b", + "path": "workplans/STATE-WP-0095-historical-source-receiving.md" + }, + { + "kind": "task", + "record_id": "STATE-WP-0095-T06", + "uuid": "6af973b4-1964-5d4d-a2f8-8dbd4c01fda4", + "path": "workplans/STATE-WP-0095-historical-source-receiving.md" + }, + { + "kind": "task", + "record_id": "STATE-WP-0095-T07", + "uuid": "cfd1ea99-e75b-542e-a434-acefb62f3c7d", + "path": "workplans/STATE-WP-0095-historical-source-receiving.md" + } + ], + "historical_invalid_identifiers": [], + "files_changed": [ + "workplans/STATE-WP-0095-historical-source-receiving.md" + ] +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/owner-ssh-image-metadata.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/owner-ssh-image-metadata.json new file mode 100644 index 00000000..a86489ae --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/owner-ssh-image-metadata.json @@ -0,0 +1,10 @@ +{ + "schema": "receiver-owner-ssh-image-metadata.v1", + "target": "railiance01", + "image": "forgejo.coulomb.social/coulomb/state-hub:main-51bc70d", + "read_only": true, + "pulled": false, + "exit_code": 127, + "status": "metadata_unverified", + "classification": "docker_cli_unavailable" +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/preflight_receiver.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/preflight_receiver.py new file mode 100644 index 00000000..8d2107d8 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/preflight_receiver.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Bounded read-only owner headroom + chart server admission. Never deploys.""" +import argparse +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import re +import subprocess +OWNER=Path('/home/worsch/state-hub') +KUBECONFIG='/home/worsch/.kube/config-railiance01' + +def command(argv,env,timeout): + try: + p=subprocess.run(argv,cwd=OWNER,env=env,capture_output=True,timeout=timeout) + return p.returncode,p.stdout + except subprocess.TimeoutExpired: + return 124,b'' + +def main(): + ap=argparse.ArgumentParser();ap.add_argument('--source-revision',required=True);args=ap.parse_args() + if not re.fullmatch('[0-9a-f]{40}',args.source_revision):ap.error('exact40char_revision_required') + env=dict(os.environ);env['KUBECONFIG']=KUBECONFIG + result={'schema':'receiver-readonly-preflight.v1','observed_at':datetime.now(timezone.utc).isoformat(), + 'source_revision':args.source_revision,'image_tag':'main-'+args.source_revision[:7], + 'production_mutations':False,'secret_reads':False, + 'limitations':['Chart/template server admission is not an exact Helm reuse-values upgrade diff.', + 'Headroom is dated observation, not a scheduling guarantee or rollout approval.', + 'No migration, pod restart, registry pull, live probe transaction or Helm command executes.']} + code,raw=command(['kubectl','--kubeconfig',KUBECONFIG,'get','nodes','-o','json'],env,20) + matched=False + if code==0: + try:matched=any(a.get('address')=='92.205.62.239' for n in json.loads(raw).get('items',[]) for a in n.get('status',{}).get('addresses',[])) + except (ValueError,AttributeError,TypeError):pass + result['target_node_confirmed']=matched + if not matched: + result['status']='target_identity_refused';print(json.dumps(result,indent=2));return 1 + code,raw=command(['make','--no-print-directory','railiance-state-hub-headroom'],env,45) + result['headroom_exit']=code + try: + headroom=json.loads(raw);result['headroom_ok']=headroom.get('ok') is True + result['headroom_summary']={k:headroom[k] for k in ('ok','reasons','notes') if k in headroom} + except (ValueError,AttributeError):result['headroom_ok']=False + if code or not result['headroom_ok']: + result['status']='headroom_refused_or_unverified';print(json.dumps(result,indent=2));return 1 + code,_=command(['make','--no-print-directory','railiance-state-hub-server-dry-run', + 'RAILIANCE_STATE_HUB_IMAGE_TAG='+result['image_tag']],env,60) + result['server_dryrun_exit']=code + result['status']='readonly_checks_passed' if code==0 else 'server_dryrun_refused_or_unverified' + print(json.dumps(result,indent=2));return 0 if code==0 else 1 +if __name__=='__main__':raise SystemExit(main()) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/publish_receiver_source.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/publish_receiver_source.py new file mode 100644 index 00000000..b615b420 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/publish_receiver_source.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Exact-path source publication only; no deployment or work-record sync.""" +from __future__ import annotations +import argparse +import hashlib +import json +from pathlib import Path +import subprocess + +BASE = Path(__file__).resolve().parent +OWNER = Path('/home/worsch/state-hub') +SCANNER = ['/usr/bin/python3', '/home/worsch/repo-manager/src/repo_manager/secret_scan.py', + '--path', str(OWNER), '--binary', '/home/worsch/.local/lib/repo-manager/gitleaks/8.30.1/gitleaks'] + +class Refusal(RuntimeError): + pass + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + +def git(owner, *args, check=True): + p = subprocess.run(['git', '-C', str(owner), *args], capture_output=True, timeout=150) + if check and p.returncode: + raise Refusal('git_' + args[0] + '_refused_' + str(p.returncode)) + return p + +def index_entries(owner): + # Stage bytes remain private to this process. Receipts expose hashes only. + return dict((entry.split(b'\t', 1)[1].decode(), entry.split(b'\t', 1)[0]) + for entry in git(owner, 'ls-files', '--stage', '-z').stdout.split(b'\0') if entry) + +def dirty_files(owner): + tracked = git(owner, 'diff', '--name-only', 'HEAD', '-z').stdout.split(b'\0') + untracked = git(owner, 'ls-files', '--others', '--exclude-standard', '-z').stdout.split(b'\0') + return sorted({p.decode() for p in tracked + untracked if p}) + +def preserved_state(owner, owned): + indices = {p: sha(v) for p, v in index_entries(owner).items() if p not in owned} + worktree = {} + for p in dirty_files(owner): + if p in owned: + continue + file = owner / p + if file.is_symlink(): + raise Refusal('unrelated_symlink_requires_review') + worktree[p] = sha(file.read_bytes()) if file.exists() else None + return {'index_entries': indices, 'dirty_worktree': worktree} + +def validate(owner, expected): + if git(owner, 'rev-parse', 'HEAD').stdout.decode().strip() != expected['base_revision']: + raise Refusal('source_base_drift') + files = expected['files'] + names = {f['path'] for f in files} + if len(names) != len(files): + raise Refusal('duplicate_expected_path') + for f in files: + p = owner / f['path'] + if p.is_symlink() or not p.is_file() or sha(p.read_bytes()) != f['candidate_sha256']: + raise Refusal('reviewed_source_drift') + # Every unrelated worktree/index entry is retained. No stash/reset/clean. + return names, preserved_state(owner, names) + +def commit_source(owner, expected, scanner, execute=False, push=False): + names, before = validate(owner, expected) + result = {'schema': 'receiver-source-publication.v1', 'scope': 'source_only', + 'deployed': False, 'repo_manager_enabled': False, + 'base_revision': expected['base_revision'], 'files': sorted(names), + 'preserved_before': before, 'status': 'validated_not_executed'} + if not execute: + return result + # Stage exact reviewed paths first so the normal scanner inspects them. + # A refusal retains reviewed source/staging for inspection; unrelated state stays intact. + git(owner, 'add', '--', *sorted(names)) + scan = subprocess.run(scanner, cwd=owner, capture_output=True, timeout=150) + result['scanner_exit'] = scan.returncode + if scan.returncode: + raise Refusal('normal_source_scanner_refused') + # --only preserves unrelated staged content while committing exact paths. + git(owner, 'commit', '--only', '-m', + 'fix(projection): preserve exact frozen cluster historical bindings', '--', *sorted(names)) + commit = git(owner, 'rev-parse', 'HEAD').stdout.decode().strip() + changed = set(git(owner, 'diff-tree', '--no-commit-id', '--name-only', '-r', commit).stdout.decode().splitlines()) + if changed != names: + raise Refusal('committed_path_set_mismatch_stop') + if preserved_state(owner, names) != before: + raise Refusal('unrelated_state_changed_stop') + result.update(status='committed_not_pushed', source_revision=commit, + preserved_after=before) + if push: + # Explicit ref and bounded push. A nonzero or timeout never triggers retry. + try: + p = git(owner, 'push', 'origin', 'HEAD:refs/heads/main', check=False) + result['push_exit'] = p.returncode + except subprocess.TimeoutExpired: + result.update(status='push_timeout_remote_unverified') + return result + remote = git(owner, 'ls-remote', '--exit-code', 'origin', 'refs/heads/main', check=False) + actual = remote.stdout.split()[0].decode() if remote.returncode == 0 and remote.stdout else None + result['observed_remote_revision'] = actual + result['status'] = 'source_published_verified' if actual == commit else 'push_remote_unverified' + return result + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--execute-source-commit', action='store_true') + ap.add_argument('--push-source', action='store_true') + args = ap.parse_args() + if args.push_source and not args.execute_source_commit: + ap.error('--push-source requires --execute-source-commit') + expected = json.loads((BASE/'source-patch-manifest.json').read_text())['repos']['state-hub'] + try: + result = commit_source(OWNER, expected, SCANNER, args.execute_source_commit, args.push_source) + except (Refusal, subprocess.TimeoutExpired) as exc: + result = {'schema': 'receiver-source-publication.v1', 'status': 'refused_or_unverified', + 'reason': str(exc) if isinstance(exc, Refusal) else 'bounded_command_timeout', + 'deployed': False, 'repo_manager_enabled': False} + print(json.dumps(result, indent=2)) + return 0 if result['status'] in {'validated_not_executed','committed_not_pushed','source_published_verified'} else 1 + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/published-source-public-ci-metadata-2.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/published-source-public-ci-metadata-2.json new file mode 100644 index 00000000..62407d50 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/published-source-public-ci-metadata-2.json @@ -0,0 +1,23 @@ +{ + "schema": "receiver-ci-metadata.v1", + "source_revision": "51bc70d1ff8f25450a8650d8e50caf053093a02e", + "image_tag": "main-51bc70d", + "no_credentials": true, + "no_logs": true, + "ci_lookup_status": "ok", + "exact_revision_build_tasks": [ + { + "id": 12624, + "name": "build-and-push", + "status": "running", + "head_sha": "51bc70d1ff8f25450a8650d8e50caf053093a02e", + "started_at": null, + "completed_at": null + } + ], + "image_lookup_status": "public_access_refused", + "status": "artifact_unverified", + "limitations": [ + "Successful task+registry digest does not prove floating dependency closure or delivered image file hashes." + ] +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/published-source-public-ci-metadata-3.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/published-source-public-ci-metadata-3.json new file mode 100644 index 00000000..b8347449 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/published-source-public-ci-metadata-3.json @@ -0,0 +1,25 @@ +{ + "schema": "receiver-ci-metadata.v1", + "source_revision": "51bc70d1ff8f25450a8650d8e50caf053093a02e", + "image_tag": "main-51bc70d", + "no_credentials": true, + "no_logs": true, + "ci_lookup_status": "ok", + "exact_revision_build_tasks": [ + { + "id": 12624, + "name": "build-and-push", + "status": "success", + "head_sha": "51bc70d1ff8f25450a8650d8e50caf053093a02e", + "started_at": null, + "completed_at": null + } + ], + "image_lookup_status": "public_access_refused", + "status": "artifact_unverified", + "limitations": [ + "Successful task+registry digest does not prove floating dependency closure or delivered image file hashes." + ], + "registry_head_status_interpretation": "Anonymous OCI challenge; root verified manifest through ephemeral anonymous-token flow, not a private credential need.", + "root_verified_image_digest": "sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015" +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/published-source-public-ci-metadata.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/published-source-public-ci-metadata.json new file mode 100644 index 00000000..28827d96 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/published-source-public-ci-metadata.json @@ -0,0 +1,14 @@ +{ + "schema": "receiver-ci-metadata.v1", + "source_revision": "51bc70d1ff8f25450a8650d8e50caf053093a02e", + "image_tag": "main-51bc70d", + "no_credentials": true, + "no_logs": true, + "ci_lookup_status": "ok", + "exact_revision_build_tasks": [], + "image_lookup_status": "public_access_refused", + "status": "artifact_unverified", + "limitations": [ + "Successful task+registry digest does not prove floating dependency closure or delivered image file hashes." + ] +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/repo-manager.patch b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/repo-manager.patch new file mode 100644 index 00000000..e78fc5b6 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/repo-manager.patch @@ -0,0 +1,972 @@ +--- a/src/repo_manager/historical_sources.py ++++ b/src/repo_manager/historical_sources.py +@@ -70,6 +70,111 @@ + "record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED", + "uuid": "1c53d3db-6633-5e2d-987f-10706d73be9e", + "parent_record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED" ++ }, ++ { ++ "repo": "railiance-cluster", ++ "path": "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md", ++ "sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9", ++ "record_id": "RAIL-BS-WP-0002", ++ "uuid": "4c63dfc6-9eac-4e79-9f77-8f644ad7147d", ++ "parent_record_id": "RAIL-BS-WP-0002", ++ "tasks": [ ++ { ++ "label": "T01", ++ "uuid": "3f042630-eab0-4c6a-9167-e2b28ff20e40", ++ "status": "done" ++ }, ++ { ++ "label": "T02", ++ "uuid": "e8510646-46ed-4697-a345-f3d3009eea78", ++ "status": "done" ++ }, ++ { ++ "label": "T03", ++ "uuid": "dab2c07f-8aa0-4635-8df6-857e87e93fc5", ++ "status": "done" ++ }, ++ { ++ "label": "T04", ++ "uuid": "5c3d40e4-239b-488e-9519-6f7a38d2325f", ++ "status": "done" ++ }, ++ { ++ "label": "T05", ++ "uuid": "7f9e0e58-a130-467a-a2d0-b3f2564e496f", ++ "status": "done" ++ } ++ ] ++ }, ++ { ++ "repo": "railiance-cluster", ++ "path": "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md", ++ "sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a", ++ "record_id": "RAIL-BS-WP-0003", ++ "uuid": "7ee9ee22-1fae-4567-9194-8d70a9e0f45b", ++ "parent_record_id": "RAIL-BS-WP-0003", ++ "tasks": [ ++ { ++ "label": "T01", ++ "uuid": "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc", ++ "status": "done" ++ }, ++ { ++ "label": "T02", ++ "uuid": "ab166073-30a7-4702-a037-4091e8706e20", ++ "status": "done" ++ }, ++ { ++ "label": "T03", ++ "uuid": "140da396-8e30-4f4d-b88c-c42c0cd46c01", ++ "status": "done" ++ }, ++ { ++ "label": "T04", ++ "uuid": "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d", ++ "status": "done" ++ } ++ ] ++ }, ++ { ++ "repo": "railiance-cluster", ++ "path": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md", ++ "sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3", ++ "record_id": "RAIL-BS-WP-0004", ++ "uuid": "7e8b0c20-51eb-40c9-9e3b-85dd380d7625", ++ "parent_record_id": "RAIL-BS-WP-0004", ++ "tasks": [ ++ { ++ "label": "T01", ++ "uuid": "4526a842-ea31-4874-9231-92ab556cfe7b", ++ "status": "done" ++ }, ++ { ++ "label": "T02", ++ "uuid": "a6313e06-1976-46a7-8e31-df4eb2eca880", ++ "status": "done" ++ }, ++ { ++ "label": "T03", ++ "uuid": "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3", ++ "status": "done" ++ }, ++ { ++ "label": "T04", ++ "uuid": "08233868-d522-4117-bc4e-6c0f52545665", ++ "status": "done" ++ }, ++ { ++ "label": "T05", ++ "uuid": "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8", ++ "status": "done" ++ }, ++ { ++ "label": "T06", ++ "uuid": "f8e4a094-c367-40eb-b895-da17bc144b07", ++ "status": "done" ++ } ++ ] + } + ]''') + +@@ -83,3 +188,24 @@ + raise ValueError("Historical source binding requires owner review: " + entry["path"]) + paths.add(entry["path"]) + return paths ++ ++ ++def historical_source_occurrences(repo_root): ++ """Exact frozen closed task occurrences; never a general local-ID alias.""" ++ from repo_manager.parse.workplan import parse_workplan_file ++ from uuid import UUID ++ paths=historical_source_paths(repo_root) ++ occurrences=set(paths) ++ for entry in BINDINGS: ++ if entry['repo']!=repo_root.name or 'tasks' not in entry:continue ++ parsed=parse_workplan_file(repo_root/entry['path'],repo_root=repo_root) ++ actual=[{'label':task.id,'uuid':task.state_hub_task_id,'status':task.status} for task in parsed.tasks] ++ if (parsed.id!=entry['record_id'] or parsed.state_hub_workstream_id!=entry['uuid'] ++ or parsed.status not in {'finished','archived','completed'} or actual!=entry['tasks'] ++ or any(task['status'] not in {'done','cancel'} for task in actual)): ++ raise ValueError('Historical task binding requires owner review: '+entry['path']) ++ UUID(entry['uuid']) ++ for number,task in enumerate(actual,1): ++ UUID(task['uuid']) ++ occurrences.add(entry['path']+'#task-block-'+str(number)) ++ return occurrences +--- a/src/repo_manager/identifiers.py ++++ b/src/repo_manager/identifiers.py +@@ -854,8 +854,9 @@ + # allocation candidates. Keep conformance reporting strict while allowing + # ordinary registration to preserve those bindings. Local aliases still + # require the explicit parent allowlist in the kind registry. +- from repo_manager.historical_sources import historical_source_paths +- historical_sources: set[str] = historical_source_paths(repo_root) ++ from repo_manager.historical_sources import historical_source_occurrences ++ frozen_sources = historical_source_occurrences(repo_root) ++ historical_sources: set[str] = set(frozen_sources) + for path in iter_workplan_files(repo_root): + parsed = parse_workplan_file(path, repo_root=repo_root) + if parsed.status not in {"finished", "archived", "completed"} or not parsed.id: +@@ -889,11 +890,18 @@ + item for item in identity["invalid_identifiers"] + if item["source"] in historical_sources + ] +- if invalid or identity["identity_collisions"]: ++ # Strict scans keep legacy nonconformance. Only collisions wholly contained ++ # in exact hash-bound closed task occurrences can be preserved by receiving; ++ # any unknown/live source or qualified/workplan collision remains fatal. ++ historical_collisions=[item for item in identity['identity_collisions'] ++ if item['kind']=='task' and re.fullmatch(r'T[0-9]{2}',item['id']) ++ and all(source in frozen_sources for source in item['sources'])] ++ collisions=[item for item in identity['identity_collisions'] if item not in historical_collisions] ++ if invalid or collisions: + raise ValueError( + "work-record identities are not safe to derive: " + f"invalid={invalid!r}, " +- f"collisions={identity['identity_collisions']!r}" ++ f"collisions={collisions!r}" + ) + + rendered: dict[Path, str] = {} +@@ -966,6 +974,7 @@ + "namespace": namespace, + "assignments": assignments, + "historical_invalid_identifiers": historical, ++ "historical_identity_collisions": historical_collisions, + "files_changed": sorted(str(path.relative_to(repo_root)) for path in rendered), + } + +--- /dev/null ++++ b/tests/test_cluster_frozen_history.py +@@ -0,0 +1,52 @@ ++import shutil ++from pathlib import Path ++import pytest ++from repo_manager.identifiers import ensure_missing_work_record_identifiers as ensure ++from repo_manager.record_identity import scan_record_identities ++ ++FIXTURE=Path(__file__).parent/'fixtures/railiance-cluster' ++@pytest.fixture ++def repo(tmp_path): ++ root=tmp_path/'railiance-cluster';shutil.copytree(FIXTURE,root) ++ live=root/'workplans/RCLUSTER-WP-0018.md';live.write_text('---\nid: RCLUSTER-WP-0018\nstatus: ready\n---\n\n```task\nid: RCLUSTER-WP-0018-T01\nstatus: todo\n```\n') ++ return root,live ++ ++def test_exact_three_histories_preserve_15_bindings_and_strict_findings(repo): ++ root,live=repo;archive={p:p.read_bytes() for p in (root/'workplans/archived').glob('*.md')};before=scan_record_identities(root) ++ assert len(before['invalid_identifiers'])==15 and len(before['identity_collisions'])==5 ++ preview=ensure(root,execute=False);assert len(preview['assignments'])==2 ++ assert preview['historical_invalid_identifiers']==before['invalid_identifiers'] ++ assert preview['historical_identity_collisions']==before['identity_collisions'] ++ result=ensure(root,execute=True);assert len(result['assignments'])==2 ++ assert all(p.read_bytes()==raw for p,raw in archive.items()) ++ assert not scan_record_identities(root)['ok'] ++ assert ensure(root,execute=False)['assignments']==[] ++ ++@pytest.mark.parametrize('change',['bytes','missing','symlink','live','nonterminal','uuid']) ++def test_unknown_history_change_refuses_before_any_allocation(repo,change): ++ root,live=repo;p=next((root/'workplans/archived').glob('*.md'));before=live.read_bytes() ++ raw=p.read_bytes() ++ if change=='missing':p.unlink() ++ elif change=='symlink':p.unlink();target=root/'saved';target.write_bytes(raw);p.symlink_to(target) ++ elif change=='live':p.write_bytes(raw.replace(b'status: finished',b'status: active',1)) ++ elif change=='nonterminal':p.write_bytes(raw.replace(b'status: done',b'status: todo',1)) ++ elif change=='uuid':p.write_bytes(raw.replace(b'state_hub_task_id:',b'old_task_id:',1)) ++ else:p.write_bytes(raw+b'\nunreviewed change\n') ++ with pytest.raises(ValueError):ensure(root) ++ assert live.read_bytes()==before ++ ++def test_any_unknown_or_live_source_in_collision_still_refuses(repo): ++ root,live=repo;before=live.read_bytes();other=root/'workplans/unknown.md' ++ other.write_text('---\nid: RCLUSTER-WP-0099\nstatus: active\n---\n\n```task\nid: T01\nstatus: todo\nstate_hub_task_id: 11111111-1111-4111-8111-111111111111\n```\n') ++ with pytest.raises(ValueError,match='identities are not safe'):ensure(root) ++ assert live.read_bytes()==before ++ ++def test_other_repo_does_not_inherit_exact_frozen_exception(repo): ++ root,live=repo;other=root.with_name('other');root.rename(other) ++ with pytest.raises(ValueError,match='identities are not safe'):ensure(other) ++ ++def test_qualified_collision_stays_global(repo): ++ root,live=repo;before=live.read_bytes() ++ (root/'workplans/other.md').write_text('---\nid: RCLUSTER-WP-0019\nstatus: active\n---\n\n```task\nid: RCLUSTER-WP-0018-T01\nstatus: done\nstate_hub_task_id: 11111111-1111-4111-8111-111111111111\n```\n') ++ with pytest.raises(ValueError,match='collisions='):ensure(root) ++ assert live.read_bytes()==before +--- a/workplans/RMGR-WP-0019-historical-source-admission.md ++++ b/workplans/RMGR-WP-0019-historical-source-admission.md +@@ -4,10 +4,10 @@ + title: "Admit exact historical source views after receiving preservation support" + domain: infotech + repo: repo-manager +-status: finished ++status: active + owner: codex + created: "2026-10-04" +-updated: "2026-10-06" ++updated: "2026-10-17" + flavor: implementation + depends_on: [] + state_hub_workstream_id: "efe0ecb2-2f3f-5f49-baee-b90ad1d353ab" +@@ -146,3 +146,62 @@ + primary/railiance01, applied, 0 created / 45 updated (ACTIVITY-WP only) / 0 retired + / 0 refused, 0 task changes, historical UUIDs preserved. Evidence: + state-hub docs/evidence/2026-10-06-activity-stale-retirement-rollout.json. ++ ++ ++## Cluster exact historical source successor, 2026-10-17 ++ ++This scoped extension preserves the completed Activity/Custodian work above. ++The cluster publisher now refuses three closed RAIL-BS source files containing ++15 existing UUID-bound bare task labels. Preserve all three source bytes, IDs, ++and UUIDs. No new alias or general syntax exemption is introduced. Evidence and ++exact receiver-first proposal: cluster-historical-publication-r1 in the Custodian ++NetKingdom change packet. The existing SHR-WP-0001 project driver owns the ++Repo Manager / State Hub cross-owner wait; NK-WP-0049-T35 benefits from the ++blocked cluster publication. No worker or human gate is woken here. ++ ++## Prepare exact cluster registrar successor ++ ++```task ++id: RMGR-WP-0019-T04 ++status: done ++priority: high ++``` ++ ++Prepared full-file SHA256 bindings for three archives and 15 exact task ++occurrences; strict scans still return 15 invalid labels and five collisions. ++Unknown/live sources, changed/missing/symlinked histories, UUID drift and ++qualified/global collisions refuse before allocation. Candidate tests pass 47; ++canonical application remains a separate reviewed step. The read-only current ++cluster preview proposes seven ordinary UUID bindings in two files and changes ++no bytes. One pre-existing ADHOC field is included explicitly: task ++RCLUSTER-WP-ADHOC-2026-07-27-T01 in workplans/ADHOC-2026-07-27.md receives ++3e3f39d3-5070-5ba3-97db-19f5f361f264 through managed allocation only. ++ ++## Admit cluster registrar successor only after receiving verification ++ ++```task ++id: RMGR-WP-0019-T05 ++status: wait ++priority: high ++depends_on: [STATE-WP-0095-T06] ++``` ++ ++Retain the current guard until exact reviewed receiving source has been delivered ++and verified through State Hub's normal owner deployment. Match both manifests, ++then integrate reviewed registrar source and verify ordinary allocator preview. ++No direct API patch, history rename or bypass substitutes for this gate. ++ ++## Run ordinary cluster owner publication and verify preservation ++ ++```task ++id: RMGR-WP-0019-T06 ++status: wait ++priority: high ++depends_on: [RMGR-WP-0019-T05] ++``` ++ ++Only then run normal owner consistency/publication with the reviewed expected ++file set: RCLUSTER-WP-0018 and the one-field ADHOC managed backfill, plus ordinary ++generated indexes. Preserve all three archived files and all 18 existing ++parent/task rows. Return actual receiving/publication receipts to STATE-WP-0095-T07. ++Do not infer TLS recovery, credential custody or native acceptance from sync. +--- /dev/null ++++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md +@@ -0,0 +1,177 @@ ++--- ++id: RAIL-BS-WP-0002 ++type: workplan ++title: "k3s and Kubernetes Platform Baseline" ++domain: financials ++repo: railiance-cluster ++status: finished ++owner: railiance ++topic_slug: railiance ++repo_goal_id: "70ab2379-fb9d-4fec-a09d-b2a717e4ace8" ++state_hub_workstream_id: "4c63dfc6-9eac-4e79-9f77-8f644ad7147d" ++created: "2026-03-09" ++updated: "2026-03-10" ++completed: "2026-03-10" ++--- ++ ++> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`). ++ ++# k3s and Kubernetes Platform Baseline ++ ++## Goal ++ ++Install k3s, Helm, and the baseline Kubernetes services on the converged ++HostEurope node. This workplan picks up exactly where `railiance-hosts` ++leaves off: a hardened, verified OS node that is ready for Kubernetes. ++ ++## Pre-condition ++ ++`railiance-infra` converge + Goss verify must pass before any task here ++is executed: ++ ++```bash ++# In railiance-infra/ ++make converge ++make verify # must exit 0 ++``` ++ ++## Boundary ++ ++This repo owns everything from k3s upward. It must not re-configure items ++defined in `railiance-infra/spec/server-baseline.yaml`. See ADR-003: ++`railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`. ++ ++**Out of scope here:** platform services (PostgreSQL, storage, identity) ++→ `railiance-platform`. Application deployments (Gitea, coulomb services) ++→ `railiance-apps`. ++ ++--- ++ ++## Tasks ++ ++### T01 — Ansible playbook: install k3s (server mode) ++ ++```task ++id: T01 ++status: done ++priority: high ++state_hub_task_id: "3f042630-eab0-4c6a-9167-e2b28ff20e40" ++completed: "2026-03-10" ++``` ++ ++Harden `ansible/bootstrap.yml` to a production-ready k3s install: ++ ++- Use the official k3s install script pinned to a specific version ++ (`INSTALL_K3S_VERSION=v1.35.1+k3s1`) ++- `INSTALL_K3S_EXEC="server --cluster-init --write-kubeconfig-mode=644"` ++ (cluster-init enables embedded etcd for future HA expansion) ++- Wait for node `Ready` before proceeding: ++ ```bash ++ k3s kubectl wait node --all --for=condition=Ready --timeout=120s ++ ``` ++- Fetch kubeconfig to the control node as `~/.kube/config-hosteurope` ++ ++**Done when:** `k3s kubectl get nodes` returns `Ready` from both the server ++and the control node (via kubeconfig). ++ ++--- ++ ++### T02 — Helm installation ++ ++```task ++id: T02 ++status: done ++priority: high ++state_hub_task_id: "e8510646-46ed-4697-a345-f3d3009eea78" ++completed: "2026-03-10" ++``` ++ ++Add a task (or a role `roles/helm/`) that: ++ ++1. Downloads the Helm binary (pinned version) to `/usr/local/bin/helm` ++2. Verifies the checksum ++3. Confirms `helm version` succeeds ++ ++**Done when:** `helm version` succeeds on the HostEurope node. ++ ++--- ++ ++### T03 — Smoke test: k3s + Helm ++ ++```task ++id: T03 ++status: done ++priority: high ++state_hub_task_id: "dab2c07f-8aa0-4635-8df6-857e87e93fc5" ++completed: "2026-03-10" ++``` ++ ++Extend `tests/smoke_kube.sh` to assert: ++ ++- `k3s kubectl get nodes` → node in Ready state ++- `helm version` exits 0 ++- CoreDNS pod running in `kube-system` ++- Traefik ingress controller pod running (default in k3s) ++ ++Run via: ++```bash ++ansible-playbook -i ansible/hosts.ini ansible/smoke.yml ++``` ++or directly over SSH if the kubeconfig is available locally. ++ ++**Done when:** all assertions pass and the script exits 0. ++ ++--- ++ ++### T04 — Commit kubeconfig management notes ++ ++```task ++id: T04 ++status: done ++priority: medium ++state_hub_task_id: "5c3d40e4-239b-488e-9519-6f7a38d2325f" ++completed: "2026-03-10" ++``` ++ ++Document in `docs/kubeconfig.md`: ++ ++- Where the kubeconfig is fetched to (`~/.kube/config-hosteurope`) ++- How to merge it into `~/.kube/config` ++- How to switch context: `kubectl config use-context default` ++- Security note: kubeconfig is gitignored (contains cluster CA + client cert) ++ ++**Done when:** doc written and committed. ++ ++--- ++ ++### T05 — Add `make k3s-install` and `make smoke` targets ++ ++```task ++id: T05 ++status: done ++priority: medium ++state_hub_task_id: "7f9e0e58-a130-467a-a2d0-b3f2564e496f" ++completed: "2026-03-10" ++``` ++ ++Add to Makefile (create one if none exists): ++ ++```makefile ++k3s-install: ## Install k3s and Helm on all inventory hosts ++ ansible-playbook -i ansible/hosts.ini ansible/bootstrap.yml ++ ++smoke: ## Run Kubernetes smoke tests ++ bash tests/smoke_kube.sh ++``` ++ ++**Done when:** both targets work and are listed in `make help`. ++ ++--- ++ ++## References ++ ++- Repo goal: `70ab2379-fb9d-4fec-a09d-b2a717e4ace8` (Install k3s and Kubernetes Baseline) ++- Domain goal: `6f96c712-60e6-4ea9-ab06-168878eafbce` (Three-Phoenix Secure Kubernetes Infrastructure) ++- Pre-condition: railiance-infra WP-0001 (Secure Single-Server Bootstrap) — completed 2026-03-09 ++- Boundary ADR: `railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md` ++- k3s releases: https://github.com/k3s-io/k3s/releases +--- /dev/null ++++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md +@@ -0,0 +1,196 @@ ++--- ++id: RAIL-BS-WP-0003 ++type: workplan ++title: "pgpool CrashLoopBackOff on PostgreSQL HA failover — missing secret key" ++domain: financials ++repo: railiance-cluster ++status: finished ++owner: tegwick ++created: "2026-03-10" ++updated: "2026-03-10" ++state_hub_workstream_id: "7ee9ee22-1fae-4567-9194-8d70a9e0f45b" ++--- ++ ++> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`). ++ ++# Bug Report: pgpool CrashLoopBackOff on PostgreSQL HA failover ++ ++## Summary ++ ++On 2026-03-10 a PostgreSQL HA failover caused all three postgresql pods to ++restart. pgpool — the connection pooler between Gitea and PostgreSQL — then ++entered CrashLoopBackOff and produced no logs. As a result Gitea's login ++and all write operations hung indefinitely. The root page was still served ++(from Valkey cache) which masked the failure. ++ ++The fix was to patch a missing key in a Kubernetes secret. The root cause is ++that the `gitea-12.2.0` Helm chart (postgresql-ha subchart v16.2.2) does not ++populate the `pgpool-password` key in the `gitea-postgresql-ha-postgresql` ++secret, even though the pgpool pod requires it at startup. ++ ++--- ++ ++## Timeline ++ ++| Time (UTC) | Event | ++|---|---| ++| ~09:45 | postgresql-0, postgresql-2 pods restarted (repmgr failover) | ++| ~09:45 | pgpool pod restarted and entered CrashLoopBackOff | ++| ~11:00 | User noticed Gitea login hanging; home page still loading | ++| ~13:00 | Root cause identified: missing `pgpool-password` secret key | ++| ~13:10 | Secret patched; pgpool pod deleted and restarted cleanly | ++| ~13:15 | Gitea fully operational | ++ ++--- ++ ++## Root Cause ++ ++The Bitnami `pgpool` container startup script reads the file ++`/opt/bitnami/pgpool/secrets/pgpool-password`, which is mounted from the ++`gitea-postgresql-ha-postgresql` Kubernetes Secret via a `subPath` volume ++mount. That secret key was never created by the Helm chart, so the file did ++not exist. The container exited immediately with no logs. ++ ++The pod had been running for 20 days without a restart, so this gap was ++never discovered during initial deployment. ++ ++--- ++ ++## Evidence ++ ++```bash ++# Secret was missing the pgpool-password key ++sudo k3s kubectl get secret -n default gitea-postgresql-ha-postgresql -o yaml ++# data: keys were password, postgres-password, repmgr-password only ++# pgpool-password was absent ++ ++# pgpool pod describe showed 824 back-off restarts over 173 minutes ++# No logs in either current or --previous output ++sudo k3s kubectl logs -n default --previous ++# (empty) ++ ++# Gitea process had zero TCP connections to PostgreSQL port 5432 ++# but many connections to Valkey port 6379 ++cat /proc//net/tcp | grep 1538 # 1538 = 5432 hex — no results ++``` ++ ++--- ++ ++## Immediate Fix Applied ++ ++```bash ++# Add the missing key (value = sr-check-password = changeme4 = base64: Y2hhbmdlbWU0) ++sudo k3s kubectl patch secret -n default gitea-postgresql-ha-postgresql \ ++ --type='json' \ ++ -p='[{"op":"add","path":"/data/pgpool-password","value":"Y2hhbmdlbWU0"}]' ++ ++# Restart pgpool ++sudo k3s kubectl delete pod -n default ++``` ++ ++--- ++ ++## Risk: Fix Will Be Lost on helm upgrade ++ ++The patched secret is managed by Helm (annotation: ++`meta.helm.sh/release-name: gitea`). A `helm upgrade` will regenerate the ++secret from the chart template, which does not include `pgpool-password`, ++and the bug will recur. ++ ++--- ++ ++## Tasks ++ ++### T01 — Add pgpool-password to Helm values ++ ++```task ++id: T01 ++status: done ++priority: high ++state_hub_task_id: "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc" ++``` ++ ++Create or update `helm/gitea-values.yaml` (or equivalent) to permanently ++include the pgpool-password so it survives `helm upgrade`: ++ ++```yaml ++postgresql-ha: ++ postgresql: ++ pgpoolPassword: ++``` ++ ++**Done when:** `helm upgrade gitea` completes and pgpool starts cleanly ++without manual secret patching. ++ ++--- ++ ++### T02 — Add pgpool health check to smoke test ++ ++```task ++id: T02 ++status: done ++priority: high ++state_hub_task_id: "ab166073-30a7-4702-a037-4091e8706e20" ++``` ++ ++Extend `tests/smoke_kube.sh` to assert: ++ ++```bash ++# All postgresql-ha pods Running ++kubectl get pods -n default | grep gitea-postgresql-ha | grep -v Running && exit 1 ++ ++# pgpool specifically not in CrashLoopBackOff ++kubectl get pod -n default -l app.kubernetes.io/component=pgpool \ ++ -o jsonpath='{.items[0].status.containerStatuses[0].state}' | grep -v crash ++``` ++ ++**Done when:** the smoke test catches a pgpool failure within 5 minutes. ++ ++--- ++ ++### T03 — Add HA failover test ++ ++```task ++id: T03 ++status: done ++priority: high ++state_hub_task_id: "140da396-8e30-4f4d-b88c-c42c0cd46c01" ++``` ++ ++Create `tests/test_ha_failover.sh` that: ++ ++1. Records Gitea login response time (baseline) ++2. Kills the primary PostgreSQL pod: `kubectl delete pod gitea-postgresql-ha-postgresql-0 -n default` ++3. Waits for repmgr to promote a replica (max 60s) ++4. Asserts Gitea login POST still succeeds within 10s ++5. Asserts pgpool pod is Running (not CrashLoopBackOff) ++6. Asserts all postgresql pods return to Running ++ ++This test must pass before any PostgreSQL HA deployment is considered done. ++ ++**Done when:** script exits 0 against a live cluster. ++ ++--- ++ ++### T04 — Document the incident in docs/ ++ ++```task ++id: T04 ++status: done ++priority: medium ++state_hub_task_id: "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d" ++``` ++ ++Add `docs/incidents/2026-03-10-pgpool-missing-secret.md` with the full ++timeline, root cause, and fix, so future operators understand what happened ++and how to recover. ++ ++**Done when:** doc committed and linked from `docs/README.md`. ++ ++--- ++ ++## References ++ ++- Bitnami postgresql-ha chart v16.2.2 ++- Gitea Helm chart v12.2.0 ++- Related decision: D3 (HA testing policy) in `DECISIONS.md` +--- /dev/null ++++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md +@@ -0,0 +1,275 @@ ++--- ++id: RAIL-BS-WP-0004 ++type: workplan ++title: "Integrated Backup — S2 Kubernetes Runtime Layer" ++domain: financials ++repo: railiance-cluster ++status: finished ++owner: tegwick ++topic_slug: railiance ++state_hub_workstream_id: "7e8b0c20-51eb-40c9-9e3b-85dd380d7625" ++created: "2026-02-25" ++updated: "2026-03-26" ++--- ++ ++> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`). ++ ++# Integrated Backup — S2 Kubernetes Runtime Layer ++ ++## Goal ++ ++Implement the Q3 (Operability & Resilience) integrated backup for ++railiance-cluster (S2). Backs up what S2 owns — the Kubernetes runtime state — ++encrypted with age, written to a local directory on the server. No external ++dependencies required. ++ ++## Architecture (Decision D4) ++ ++Each railiance repo implements its own backup for what it owns. No central ++backup service. See `DECISIONS.md` D4 for full rationale. ++ ++**Standard interface every railiance repo must provide:** ++ ++```bash ++make backup # encrypt + write to /opt/backup/railiance// ++make restore # restore from most recent local backup ++``` ++ ++Encryption: age, same key pair as SOPS secrets (`.sops.yaml` public key). ++Output: `/opt/backup/railiance/cluster/` on the server. ++ ++## What S2 (railiance-cluster) owns and must back up ++ ++| Asset | Why it matters | ++|---|---| ++| k3s etcd snapshots | Full cluster state — all workloads, configs, secrets | ++| Helm release values | Runtime values not in git (any manually applied overrides) | ++| kubeconfig | Admin access to the cluster | ++ ++**Not S2's responsibility:** ++- Custodian State Hub DB → the-custodian owns this ++- Operator workstation config (`.claude/`, `.gitconfig`) → operator's own concern ++- Application data (Gitea repos, uploads) → S5 (railiance-apps) owns this ++- PostgreSQL data volumes → S3 (railiance-platform) owns this ++ ++## Encryption ++ ++Reuse the age public key from `.sops.yaml`: ++ ++```bash ++AGE_PUBLIC_KEY=$(grep 'age:' .sops.yaml | awk '{print $2}') ++tar -czf - | age -r "${AGE_PUBLIC_KEY}" -o backup.tar.gz.age ++``` ++ ++Decryption requires the private key at `~/.config/sops/age/keys.txt` ++(same key used for `sops -d`). No additional key management needed. ++ ++## Extension Point EP-RAIL-005 ++ ++Once all five OAS layers implement this interface, the custodian can ++orchestrate a full-stack backup with: ++ ++```bash ++for repo in railiance-infra railiance-cluster railiance-platform \ ++ railiance-enablement railiance-apps; do ++ make -C ~/$repo backup ++done ++``` ++ ++No special protocol needed — just the standard interface. ++ ++--- ++ ++## Tasks ++ ++### T01 — Define backup directory and encryption wrapper ++ ++```task ++id: T01 ++status: done ++priority: high ++state_hub_task_id: "4526a842-ea31-4874-9231-92ab556cfe7b" ++``` ++ ++Create `tools/cmd/railiance-backup-s2` (replacing the old `railiance-backup`): ++ ++- Backup dir: `/opt/backup/railiance/cluster/` (create with `mkdir -p`) ++- Encrypt each artifact with age using public key from `.sops.yaml` ++- Write timestamp-named files: `etcd-.snap.age`, `helm-values-.tar.gz.age`, `kubeconfig-.yaml.age` ++- Keep last 7 of each type ++- Write `.last-backup` stamp ++- Exit 0 on success, non-zero on any failure ++- No network required ++ ++Also remove the old `tools/cmd/railiance-backup` (backed up Docker-based ++custodian DB — wrong scope, not applicable to this server). ++ ++**Done when:** `make backup` runs on COULOMBCORE without error and files ++appear in `/opt/backup/railiance/cluster/`. ++ ++--- ++ ++### T02 — Back up k3s state (SQLite hot backup) ++ ++```task ++id: T02 ++status: done ++priority: high ++state_hub_task_id: "a6313e06-1976-46a7-8e31-df4eb2eca880" ++``` ++ ++k3s has built-in etcd snapshot support: ++ ++```bash ++sudo k3s etcd-snapshot save --name railiance-$(date -u +%Y%m%dT%H%M%SZ) ++# Default location: /var/lib/rancher/k3s/server/db/snapshots/ ++``` ++ ++Add to the backup script: take a fresh snapshot, encrypt with age, ++copy to `/opt/backup/railiance/cluster/`. ++ ++> **Note — verify etcd is in use before implementing:** ++> `k3s etcd-snapshot` only works if k3s was started with `--cluster-init`. ++> Without it, k3s uses SQLite and this command will fail. ++> Verify first: `sudo k3s etcd-snapshot ls 2>&1` ++ ++> **Note — sudo required:** etcd snapshot requires root. See T06 for how ++> this is resolved (backup runs under root's crontab). ++ ++**Done when:** backup includes a current etcd snapshot. ++ ++--- ++ ++### T03 — Back up Helm release values ++ ++```task ++id: T03 ++status: done ++priority: medium ++state_hub_task_id: "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3" ++``` ++ ++Capture current runtime Helm values for all releases: ++ ++```bash ++KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm list -A -o json | \ ++ jq -r '.[].name + " " + .namespace' | \ ++ while read name ns; do ++ KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm get values "$name" -n "$ns" -o yaml ++ done ++``` ++ ++Tar and age-encrypt into `helm-values-.tar.gz.age`. ++ ++> **Note — kubeconfig permissions:** `/etc/rancher/k3s/k3s.yaml` is root-readable ++> only by default. The backup script must either run as root (see T06) or k3s ++> must be configured with `--write-kubeconfig-mode=644`. Running as root ++> (via root crontab) is the chosen approach — no config change needed. ++ ++**Done when:** backup includes a snapshot of all Helm release values. ++ ++--- ++ ++### T04 — Back up kubeconfig ++ ++```task ++id: T04 ++status: done ++priority: medium ++state_hub_task_id: "08233868-d522-4117-bc4e-6c0f52545665" ++``` ++ ++Age-encrypt `~/.kube/config-hosteurope` (or `/etc/rancher/k3s/k3s.yaml`) ++into `kubeconfig-.yaml.age` in the backup directory. ++ ++**Done when:** backup includes the encrypted kubeconfig. ++ ++--- ++ ++### T05 — make restore target ++ ++```task ++id: T05 ++status: done ++priority: medium ++state_hub_task_id: "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8" ++``` ++ ++Add `tools/cmd/railiance-restore-s2` that decrypts and lists available ++backups, with guided restore for the etcd snapshot case. ++ ++Restore of etcd from snapshot: ++```bash ++sudo k3s server --cluster-reset \ ++ --cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/ ++``` ++ ++**Done when:** `make restore` prints available backups and a restore guide. ++ ++--- ++ ++### T06 — Install cron job and run restore drill ++ ++```task ++id: T06 ++status: done ++priority: medium ++state_hub_task_id: "f8e4a094-c367-40eb-b895-da17bc144b07" ++``` ++ ++#### Solving the sudo problem ++ ++The backup script needs root for two reasons: ++- `k3s etcd-snapshot save` requires root ++- `/etc/rancher/k3s/k3s.yaml` (kubeconfig) is root-readable only ++ ++**Solution: run the cron under root's crontab.** ++ ++This is the correct pattern for system-level backup jobs. It avoids a ++proliferating sudoers whitelist (one entry per command, brittle to maintain) ++and matches how tools like `rsnapshot`, `bacula`, and `borgbackup` work in ++production. The backup writes to `/opt/backup/` which is root-owned anyway. ++ ++Install the cron as root: ++ ++```bash ++sudo crontab -e ++# Add: ++0 2 * * * make -C /home/tegwick/railiance-cluster backup >> /opt/backup/railiance/cluster/backup.log 2>&1 ++``` ++ ++Note: use the absolute path to the repo — `~` does not expand reliably in ++root's crontab unless HOME is set. ++ ++Verify it is installed: ++```bash ++sudo crontab -l | grep railiance ++``` ++ ++#### Restore drill ++ ++Once T01–T04 are done, run a decrypt-and-verify drill: ++ ++```bash ++# Decrypt the etcd snapshot and verify it is a valid snapshot file ++sudo age -d -i ~/.config/sops/age/keys.txt \ ++ /opt/backup/railiance/cluster/etcd-$(ls /opt/backup/railiance/cluster/etcd-*.snap.age | sort -r | head -1 | xargs basename | sed 's/etcd-//;s/.snap.age//').snap.age \ ++ | file - ++ ++# Record the drill ++echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) restore drill OK" \ ++ >> /opt/backup/railiance/cluster/restore-drill.log ++``` ++ ++**Done when:** cron installed under root, drill completes without error, ++log entry written. ++ ++--- ++ ++## References ++ ++- Decision D4: Integrated backup per capability (`DECISIONS.md`) ++- Decision D2: Nextcloud as optional offsite extension (still valid, not a requirement) ++- OAS Q3: Operability & Resilience ++- Extension point EP-RAIL-005: Custodian full-stack backup orchestration ++- k3s etcd snapshots: https://docs.k3s.io/datastore/backup-restore diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/review-freeze.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/review-freeze.json new file mode 100644 index 00000000..1a4ecc54 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/review-freeze.json @@ -0,0 +1,56 @@ +{ + "schema": "cluster-frozen-successor-review.v1", + "stable_proposal_id": "STATE-WP-0095-T06-cluster-frozen-receiving-r1", + "revision": 2, + "status": "prepared_for_independent_source_review_not_deployment", + "files": [ + { + "path": "REVIEW.md", + "sha256": "c8f13ed5d67d4743ea8e81bf720367022c820a060d7b225e84338eab2907493f" + }, + { + "path": "source-patch-manifest.json", + "sha256": "f75391603c703b71b76eaf3432adef5022e73d61e4d3b22da4820f583dd435b4" + }, + { + "path": "state-hub.patch", + "sha256": "5cb52c26c902ed1cd95156f25c2e4d9107e6b2a808212960aa766074314b819d" + }, + { + "path": "repo-manager.patch", + "sha256": "232552c3d601d185984be5a8e5bc9937bc9ddda7327923b5f0440f6deb2c87fc" + }, + { + "path": "publish_receiver_source.py", + "sha256": "98c24e9df3384d39e188019a249265a39896221d898082bcab5bbef64ba40ec8" + }, + { + "path": "lookup_ci_metadata.py", + "sha256": "05e388ad81334316653770430b153b8a040427159e7e33b730edc4a0ecf3a520" + }, + { + "path": "preflight_receiver.py", + "sha256": "1140ad022eba074cffd8bd62ab71643389a7f6992d2d59765ea0fd85bb812b92" + }, + { + "path": "test_publication_helpers.py", + "sha256": "160f9efe77e6533ca154b1876ab1221fe60029d8a59b68e7d18010731c71a156" + }, + { + "path": "bounded-preflight-commands.md", + "sha256": "f4cfbeea5758b5e48234f41eb2960adf55cd0dbb78cd4caecf64d28bc1f44cc1" + }, + { + "path": "deployed-to-current-source-closure.json", + "sha256": "a8dd674b9740b2d72425acfadb10b00abd7692ccf6b11cf81dbdda4d246c6d21" + }, + { + "path": "managed-statehub-uuid-assignment.json", + "sha256": "9310f0889ba801414a5e9274ef3ff64c0b083749cce927231bd80c712428bdf6" + }, + { + "path": "current-base-public-ci-metadata.json", + "sha256": "4a397725f7d4a861463ce8c2f15d0aecab9091c5376b9b882a9bb7d5b88598fc" + } + ] +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/serverdryrun-first-phase.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/serverdryrun-first-phase.json new file mode 100644 index 00000000..26934c93 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/serverdryrun-first-phase.json @@ -0,0 +1,24 @@ +{ + "schema": "receiver-serverdryrun-phase-diagnosis.v1", + "phase": "platform_db_secret_template", + "exit_code": 1, + "raw_diagnostics_retained": false, + "fixed_indicators": { + "Secret": true, + "state-hub-db-credentials": true, + "type": false, + "immutable": false, + "Invalid value": false, + "data": false, + "metadata.resourceVersion": false, + "required": true, + "strict": false, + "admission": true, + "annotation": true, + "Too long": false, + "forbidden": true, + "patch": true, + "invalid": true + }, + "safe_field_names": [] +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/serverdryrun-make-diagnosis.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/serverdryrun-make-diagnosis.json new file mode 100644 index 00000000..69077df6 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/serverdryrun-make-diagnosis.json @@ -0,0 +1,8 @@ +{ + "schema": "receiver-make-serverdryrun-diagnosis.v1", + "command_scope": "existing_owner_make_server_dryrun", + "exit_code": 2, + "raw_diagnostics_retained": false, + "classification": "invalid_resource", + "safe_details": [] +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/serverdryrun-nonsecret-remainder.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/serverdryrun-nonsecret-remainder.json new file mode 100644 index 00000000..a65113bd --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/serverdryrun-nonsecret-remainder.json @@ -0,0 +1,47 @@ +{ + "schema": "receiver-nonsecret-serverdryrun-remainder.v1", + "image_tag": "main-51bc70d", + "production_mutations": false, + "raw_diagnostics_retained": false, + "secret_template_phases": "refused_by_existing_policy_and_excluded_from_remainder", + "phases": [ + { + "phase": "render_nonsecret_app_chart", + "status": "passed", + "exit_code": 0, + "classification": null + }, + { + "phase": "platform_db_cluster", + "status": "passed", + "exit_code": 0, + "classification": null + }, + { + "phase": "platform_db_networkpolicies", + "status": "passed", + "exit_code": 0, + "classification": null + }, + { + "phase": "app_namespace", + "status": "passed", + "exit_code": 0, + "classification": null + }, + { + "phase": "nonsecret_app_chart_admission", + "status": "passed", + "exit_code": 0, + "classification": null + } + ], + "rendered_public_kinds": [ + "ConfigMap", + "Deployment", + "Job", + "Service", + "ServiceAccount" + ], + "status": "nonsecret_remainder_passed" +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/serverdryrun-secret-policy-classification.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/serverdryrun-secret-policy-classification.json new file mode 100644 index 00000000..cfa6057d --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/serverdryrun-secret-policy-classification.json @@ -0,0 +1,23 @@ +{ + "phase": "platform_db_secret_template", + "exit_code": 1, + "classification": "secret_last_applied_annotation_admission_refusal", + "fixed_indicators": { + "last-applied-configuration": true, + "kubectl.kubernetes.io": true, + "validatingadmissionpolicy": true, + "admission webhook": false, + "secret-last-applied": true, + "secret-annotation": false, + "forbidden:": false, + "denied the request": false, + "does not have": false, + "not a valid": false, + "cannot be": false, + "must not": false, + "must be": false, + "required": true, + "plaintext": false, + "plain-text": false + } +} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/source-patch-manifest.json b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/source-patch-manifest.json new file mode 100644 index 00000000..966e4b11 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/source-patch-manifest.json @@ -0,0 +1,99 @@ +{ + "schema": "cluster-historical-source-patches.v1", + "status": "isolated_review_candidate_not_applied", + "binding_manifests_identical": true, + "repos": { + "repo-manager": { + "base_revision": "9ebde0a9bd60d5c82910494fbe6bd1b891455cee", + "files": [ + { + "path": "src/repo_manager/historical_sources.py", + "action": "modify", + "base_sha256": "a465aa595867c5e80918168158f8a27bf429d469b2e5249eca98b4e5ef84ce03", + "candidate_sha256": "838a0422b81e1215e3ec148c3104dd603d9dfc476610bb5617d98ccf5d3be218" + }, + { + "path": "src/repo_manager/identifiers.py", + "action": "modify", + "base_sha256": "68dbbd3f137f326a49d20ba3f5855d6b00b4d62eb01dc904b51a08d77552d2bf", + "candidate_sha256": "4e35fcd009c20822574fd33171176bd14a1ff2dd5af8b0bb2820954974593a2a" + }, + { + "path": "tests/test_cluster_frozen_history.py", + "action": "add", + "base_sha256": null, + "candidate_sha256": "0490ef3eb474f653e4a6a52b711999876bef061c07de3e7a6d327a8ba54acaf3" + }, + { + "path": "workplans/RMGR-WP-0019-historical-source-admission.md", + "action": "modify", + "base_sha256": "638af4000aa442a57df8eddef2d8b5aa5dffb9dbc9ccf547d9e04cf25012ffe4", + "candidate_sha256": "6378d96b70259ae81435b6ce8f97864698bdc5b6679cabbe9dc61a1fc1d09699" + }, + { + "path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md", + "action": "add", + "base_sha256": null, + "candidate_sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9" + }, + { + "path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md", + "action": "add", + "base_sha256": null, + "candidate_sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a" + }, + { + "path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md", + "action": "add", + "base_sha256": null, + "candidate_sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3" + } + ], + "patch_sha256": "232552c3d601d185984be5a8e5bc9937bc9ddda7327923b5f0440f6deb2c87fc" + }, + "state-hub": { + "base_revision": "4ab66e363ed00a67ad1ea0434854bc4284c7a26e", + "files": [ + { + "path": "api/services/historical_projection.py", + "action": "modify", + "base_sha256": "4d7cd127462bedaaf1c0915c6a11f58b25d28ae6f000eb89c95d903bf4a0e94e", + "candidate_sha256": "e3c4bf965764b32a6df5d61c71272ee22569df9a9ddae2a3fc3df97ec554e0b2" + }, + { + "path": "tests/test_cluster_historical_projection.py", + "action": "add", + "base_sha256": null, + "candidate_sha256": "183f0574aa01f69269a06d162b29b5b95c3415f6fce0560bb92188c09be26507" + }, + { + "path": "workplans/STATE-WP-0095-historical-source-receiving.md", + "action": "modify", + "base_sha256": "9be4b0661f0cc817d55dcf02f959339cb6c13f56956b111a6ecd87d1bec79405", + "candidate_sha256": "aeb307597dba0d92594d55242bd69ed6a391e7fdd58bb16c839ec8cbce44b349" + }, + { + "path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md", + "action": "add", + "base_sha256": null, + "candidate_sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9" + }, + { + "path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md", + "action": "add", + "base_sha256": null, + "candidate_sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a" + }, + { + "path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md", + "action": "add", + "base_sha256": null, + "candidate_sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3" + } + ], + "patch_sha256": "5cb52c26c902ed1cd95156f25c2e4d9107e6b2a808212960aa766074314b819d" + } + }, + "revision": "r2", + "supersedes": "r1 manifest and REVIEW rollout assumptions only; originals preserved" +}