From 2f18be788087dfaa6c368d405828f299196022c5 Mon Sep 17 00:00:00 2001 From: codex Date: Sun, 1 Nov 2026 22:57:27 +0100 Subject: [PATCH] docs: preserve reviewed NetKingdom session artifacts --- .../api/schemas/work_record_projection.py | 42 + .../state-hub/api/schemas/workplan.py | 182 ++ .../api/schemas/workplan_dependency.py | 71 + .../state-hub/api/schemas/workstream.py | 41 + .../api/schemas/workstream_dependency.py | 19 + .../state-hub/api/services/__init__.py | 1 + .../api/services/consistency_sweep.py | 215 ++ .../state-hub/api/services/execution_queue.py | 112 + .../state-hub/api/services/fabric_graph.py | 689 +++++ .../api/services/forge_credential.py | 122 + .../api/services/forge_projection.py | 2257 +++++++++++++++++ .../api/services/forge_repository.py | 247 ++ .../api/services/historical_projection.py | 195 ++ .../services/informed_decision_contract.py | 388 +++ .../state-hub/api/services/legacy_compat.py | 152 ++ .../state-hub/api/services/legacy_meter.py | 438 ++++ .../state-hub/api/services/lifecycle.py | 135 + .../api/services/markitect_templates.py | 149 ++ .../state-hub/api/services/memo_dispatch.py | 174 ++ .../state-hub/api/services/memo_forwarding.py | 447 ++++ .../api/services/memo_notification.py | 67 + .../api/services/message_receipts.py | 212 ++ .../api/services/ops_run_projection.py | 130 + .../api/services/recently_on_scope.py | 632 +++++ .../state-hub/api/services/reconciliation.py | 152 ++ .../api/services/repo_manager_dual_run.py | 544 ++++ .../api/services/repository_aliases.py | 229 ++ .../api/services/repository_identity.py | 23 + .../api/services/repository_rename.py | 1845 ++++++++++++++ .../api/services/review_contracts.py | 336 +++ .../state-hub/api/services/sbom_nexus.py | 70 + .../state-hub/api/services/schema_state.py | 63 + .../api/services/suggestion_relevance.py | 118 + .../state-hub/api/services/suggestion_wsjf.py | 17 + .../state-hub/api/services/summary_cache.py | 298 +++ .../api/services/task_record_id_backfill.py | 214 ++ .../api/services/token_sources/__init__.py | 16 + .../api/services/token_sources/attribution.py | 171 ++ .../api/services/token_sources/base.py | 71 + .../api/services/token_sources/claude.py | 120 + .../api/services/token_sources/codex.py | 124 + .../state-hub/api/services/wait_kind.py | 138 + .../work_record_identifier_migration.py | 800 ++++++ .../state-hub/api/services/workplan_files.py | 175 ++ .../api/services/write_idempotency.py | 227 ++ .../state-hub/api/task_status.py | 84 + .../state-hub/api/work_record_flavor.py | 48 + .../state-hub/api/workplan_status.py | 190 ++ .../260622-RAIL-BS-WP-0002-k3s-baseline.md | 177 ++ ...-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md | 196 ++ 50 files changed, 13563 insertions(+) create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/work_record_projection.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workplan.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workplan_dependency.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workstream.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workstream_dependency.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/__init__.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/consistency_sweep.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/execution_queue.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/fabric_graph.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/forge_credential.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/forge_projection.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/forge_repository.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/historical_projection.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/informed_decision_contract.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/legacy_compat.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/legacy_meter.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/lifecycle.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/markitect_templates.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/memo_dispatch.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/memo_forwarding.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/memo_notification.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/message_receipts.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/ops_run_projection.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/recently_on_scope.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/reconciliation.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repo_manager_dual_run.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repository_aliases.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repository_identity.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repository_rename.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/review_contracts.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/sbom_nexus.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/schema_state.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/suggestion_relevance.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/suggestion_wsjf.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/summary_cache.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/task_record_id_backfill.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/__init__.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/attribution.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/base.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/claude.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/codex.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/wait_kind.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/work_record_identifier_migration.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/workplan_files.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/write_idempotency.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/task_status.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/work_record_flavor.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/workplan_status.py create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md create mode 100644 docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/work_record_projection.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/work_record_projection.py new file mode 100644 index 00000000..a3101e9f --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/work_record_projection.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import re +import uuid +from typing import Literal + +from pydantic import BaseModel, Field, field_validator + +from api.schemas.task import TaskRead +from api.schemas.workplan import WorkplanRead +from api.schemas.workplan_dependency import WorkplanDependencyRead + + +class RepositoryProjectionReconcile(BaseModel): + expected_commit: str + acknowledge_retirements: bool = False + # None keeps today's blanket acknowledgement. An empty list is present and + # means the caller expects zero retirements. Combined with + # acknowledge_retirements, the call is ambiguous and retires nothing. + expected_retirement_uuids: list[uuid.UUID] | None = None + + @field_validator("expected_commit") + @classmethod + def validate_commit(cls, value: str) -> str: + value = value.strip().lower() + if not re.fullmatch(r"[0-9a-f]{40}", value): + raise ValueError("expected_commit must be a full 40-character Git SHA") + return value + + +class RepositoryProjectionSnapshot(BaseModel): + """One bounded read of a repository's complete work-record projection.""" + + schema_version: Literal["state-hub.repository-projection-snapshot.v1"] = Field( + validation_alias="schema", + serialization_alias="schema", + ) + repo_slug: str + repo_id: uuid.UUID + workplans: list[WorkplanRead] + tasks: list[TaskRead] + dependencies: list[WorkplanDependencyRead] diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workplan.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workplan.py new file mode 100644 index 00000000..eb716185 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workplan.py @@ -0,0 +1,182 @@ +import uuid +from datetime import date, datetime +from typing import Literal + +from pydantic import BaseModel, ConfigDict, field_validator + +from api.schemas.workplan_dependency import WorkplanDepStub +from api.work_record_flavor import ( + FLAVOR_PROMOTION_REASONS, + WORK_RECORD_FLAVORS, + normalize_flavor, + normalize_promotion_reason, +) +from api.workplan_status import normalize_workplan_status + +WorkplanStatus = Literal[ + "proposed", + "ready", + "active", + "blocked", + "backlog", + "finished", + "archived", +] +ExecutionState = Literal["manual", "queued", "scheduled", "launching", "paused", "completed", "cancelled"] +LaunchMode = Literal["manual", "queued", "scheduled", "immediate"] +ConcurrencyMode = Literal["sequential", "parallel"] + + +class WorkplanStatusMixin(BaseModel): + @field_validator("status", mode="before", check_fields=False) + @classmethod + def _normalise_status(cls, value): + return normalize_workplan_status(value) + + +class WorkplanFlavorMixin(BaseModel): + @field_validator("flavor", mode="before", check_fields=False) + @classmethod + def _normalise_flavor(cls, value): + flavor = normalize_flavor(value) + if flavor is not None and flavor not in WORK_RECORD_FLAVORS: + raise ValueError( + f"Unknown work-record flavor {flavor!r}; " + f"expected one of {', '.join(WORK_RECORD_FLAVORS)}" + ) + return flavor + + @field_validator("flavor_promotion_reason", mode="before", check_fields=False) + @classmethod + def _normalise_promotion_reason(cls, value): + reason = normalize_promotion_reason(value) + if reason is not None and reason not in FLAVOR_PROMOTION_REASONS: + raise ValueError( + f"Unknown flavor_promotion_reason {reason!r}; " + f"expected one of {', '.join(FLAVOR_PROMOTION_REASONS)}" + ) + return reason + + @field_validator("flavor_promoted_from", mode="before", check_fields=False) + @classmethod + def _normalise_promoted_from(cls, value): + flavor = normalize_flavor(value) + if flavor is not None and flavor not in WORK_RECORD_FLAVORS: + raise ValueError( + f"Unknown flavor_promoted_from {flavor!r}; " + f"expected one of {', '.join(WORK_RECORD_FLAVORS)}" + ) + return flavor + + +class WorkplanCreate(WorkplanStatusMixin, WorkplanFlavorMixin): + id: uuid.UUID | None = None + repo_id: uuid.UUID + topic_id: uuid.UUID | None = None + slug: str + title: str + description: str | None = None + status: WorkplanStatus = "active" + owner: str | None = None + due_date: date | None = None + planning_priority: str | None = None + planning_order: int | None = None + flavor: str | None = None + flavor_promotion_reason: str | None = None + flavor_promoted_from: str | None = None + execution_state: ExecutionState = "manual" + launch_mode: LaunchMode = "manual" + concurrency_mode: ConcurrencyMode = "sequential" + queue_rank: int | None = None + execution_group: str | None = None + scheduled_for: datetime | None = None + repo_goal_id: uuid.UUID | None = None + + +class WorkplanUpdate(WorkplanStatusMixin, WorkplanFlavorMixin): + title: str | None = None + description: str | None = None + status: WorkplanStatus | None = None + owner: str | None = None + due_date: date | None = None + planning_priority: str | None = None + planning_order: int | None = None + flavor: str | None = None + flavor_promotion_reason: str | None = None + flavor_promoted_from: str | None = None + execution_state: ExecutionState | None = None + launch_mode: LaunchMode | None = None + concurrency_mode: ConcurrencyMode | None = None + queue_rank: int | None = None + execution_group: str | None = None + scheduled_for: datetime | None = None + topic_id: uuid.UUID | None = None + repo_id: uuid.UUID | None = None + repo_goal_id: uuid.UUID | None = None + + +class WorkplanFileBinding(BaseModel): + workplan_id: uuid.UUID + filename: str + relative_path: str + repo_slug: str + archived: bool = False + status: WorkplanStatus | None = None + + +class WorkplanBindingsSync(BaseModel): + bindings: list[WorkplanFileBinding] + + +class WorkplanRead(WorkplanStatusMixin): + model_config = ConfigDict(from_attributes=True) + id: uuid.UUID + repo_id: uuid.UUID + topic_id: uuid.UUID | None = None + repo_goal_id: uuid.UUID | None = None + slug: str + title: str + description: str | None = None + status: WorkplanStatus + owner: str | None = None + due_date: date | None = None + planning_priority: str | None = None + planning_order: int | None = None + flavor: str | None = None + flavor_promotion_reason: str | None = None + flavor_promoted_from: str | None = None + execution_state: ExecutionState = "manual" + launch_mode: LaunchMode = "manual" + concurrency_mode: ConcurrencyMode = "sequential" + queue_rank: int | None = None + execution_group: str | None = None + scheduled_for: datetime | None = None + backing_filename: str | None = None + backing_relative_path: str | None = None + backing_archived: bool | None = None + backing_synced_at: datetime | None = None + # Set when forge projection retired the row (slug stamped "@retired-"). + # A retired row is a tombstone, not live work, whatever its status. + projection_retired_at: datetime | None = None + # Derived (CUST-WP-0074 rule 5): "human" | "external" | "none" while + # status is blocked, else None. Set by + # api.services.wait_kind.annotate_workplans on read routes. + blocked_kind: str | None = None + created_at: datetime + updated_at: datetime + + +class WorkplanWithTaskCounts(WorkplanRead): + tasks_total: int = 0 + tasks_wait: int = 0 + tasks_todo: int = 0 + tasks_progress: int = 0 + tasks_done: int = 0 + tasks_cancel: int = 0 + + +class WorkplanWithDeps(WorkplanWithTaskCounts): + """WorkplanWithTaskCounts enriched with dependency graph edges.""" + depends_on: list[WorkplanDepStub] = [] + blocks: list[WorkplanDepStub] = [] + blocked_reasons: list[dict] = [] diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workplan_dependency.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workplan_dependency.py new file mode 100644 index 00000000..6cf1cd5f --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workplan_dependency.py @@ -0,0 +1,71 @@ +import uuid +from datetime import datetime + +from pydantic import AliasChoices, BaseModel, ConfigDict, Field, computed_field + + +class WorkplanDependencyCreate(BaseModel): + to_workplan_id: uuid.UUID | None = Field( + default=None, + validation_alias=AliasChoices("to_workplan_id", "to_workstream_id"), + ) + to_task_id: uuid.UUID | None = None + from_task_id: uuid.UUID | None = None + relationship_type: str = "blocks" + description: str | None = None + + +class WorkplanDependencyRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + id: uuid.UUID + from_workplan_id: uuid.UUID + from_task_id: uuid.UUID | None = None + to_workplan_id: uuid.UUID | None = None + to_task_id: uuid.UUID | None = None + relationship_type: str + description: str | None = None + source_owner: str | None = None + source_commit: str | None = None + created_at: datetime + updated_at: datetime + + +class WorkplanDepStub(BaseModel): + """Minimal projection of the other end of a dependency edge.""" + dep_id: uuid.UUID + target_type: str = "workplan" + relationship_type: str = "blocks" + workplan_id: uuid.UUID | None = Field( + default=None, + validation_alias=AliasChoices("workplan_id", "workstream_id"), + ) + workplan_slug: str | None = Field( + default=None, + validation_alias=AliasChoices("workplan_slug", "workstream_slug"), + ) + workplan_title: str | None = Field( + default=None, + validation_alias=AliasChoices("workplan_title", "workstream_title"), + ) + task_id: uuid.UUID | None = None + task_title: str | None = None + # The waiting task on a task-scoped edge (workplan_dependencies.from_task_id); + # None on frontmatter edges. + from_task_id: uuid.UUID | None = None + from_task_title: str | None = None + description: str | None = None + + @computed_field # type: ignore[prop-decorator] + @property + def workstream_id(self) -> uuid.UUID | None: + return self.workplan_id + + @computed_field # type: ignore[prop-decorator] + @property + def workstream_slug(self) -> str | None: + return self.workplan_slug + + @computed_field # type: ignore[prop-decorator] + @property + def workstream_title(self) -> str | None: + return self.workplan_title \ No newline at end of file diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workstream.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workstream.py new file mode 100644 index 00000000..3b3d719f --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workstream.py @@ -0,0 +1,41 @@ +"""Legacy aliases — prefer ``api.schemas.workplan``.""" +from api.schemas.workplan import ( + ConcurrencyMode, + ExecutionState, + LaunchMode, + WorkplanCreate, + WorkplanRead, + WorkplanStatus, + WorkplanStatusMixin, + WorkplanUpdate, + WorkplanWithDeps, + WorkplanWithTaskCounts, +) + +WorkstreamStatus = WorkplanStatus +WorkstreamStatusMixin = WorkplanStatusMixin +WorkstreamCreate = WorkplanCreate +WorkstreamUpdate = WorkplanUpdate +WorkstreamRead = WorkplanRead +WorkstreamWithTaskCounts = WorkplanWithTaskCounts +WorkstreamWithDeps = WorkplanWithDeps + +__all__ = [ + "WorkstreamStatus", + "WorkstreamStatusMixin", + "WorkstreamCreate", + "WorkstreamUpdate", + "WorkstreamRead", + "WorkstreamWithTaskCounts", + "WorkstreamWithDeps", + "WorkplanStatus", + "WorkplanStatusMixin", + "WorkplanCreate", + "WorkplanUpdate", + "WorkplanRead", + "WorkplanWithTaskCounts", + "WorkplanWithDeps", + "ExecutionState", + "LaunchMode", + "ConcurrencyMode", +] \ No newline at end of file diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workstream_dependency.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workstream_dependency.py new file mode 100644 index 00000000..d238d6bf --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/schemas/workstream_dependency.py @@ -0,0 +1,19 @@ +"""Legacy aliases — prefer ``api.schemas.workplan_dependency``.""" +from api.schemas.workplan_dependency import ( + WorkplanDepStub, + WorkplanDependencyCreate, + WorkplanDependencyRead, +) + +WorkstreamDependencyCreate = WorkplanDependencyCreate +WorkstreamDependencyRead = WorkplanDependencyRead +WorkstreamDepStub = WorkplanDepStub + +__all__ = [ + "WorkstreamDependencyCreate", + "WorkstreamDependencyRead", + "WorkstreamDepStub", + "WorkplanDependencyCreate", + "WorkplanDependencyRead", + "WorkplanDepStub", +] \ No newline at end of file diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/__init__.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/__init__.py new file mode 100644 index 00000000..9dbc675b --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/__init__.py @@ -0,0 +1 @@ +"""Service-layer helpers for State Hub API features.""" diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/consistency_sweep.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/consistency_sweep.py new file mode 100644 index 00000000..3dbcb4e0 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/consistency_sweep.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import asyncio +import json +import re +import subprocess +import sys +import uuid +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from api.config import settings +from api.models.progress_event import ProgressEvent +from api.schemas.consistency_sweep import ( + ConsistencySweepIssueSummary, + ConsistencySweepRemoteAllRun, + ConsistencySweepRepoResult, +) + +_LOCK_SKIP_MARKER = "another fix-consistency-remote --all run is already active" +_CLEAN_RE = re.compile(r"^\s*CLEAN \(skipped\):\s*(.+)$", re.MULTILINE) +_MISSING_RE = re.compile(r"^\s*NOT ON THIS HOST \(skipped\):\s*(.+)$", re.MULTILINE) +_BUDGET_RE = re.compile( + r"^\s*BUDGET EXHAUSTED after \d+s \(skipped\):\s*(.+)$", + re.MULTILINE, +) + + +def _script_path() -> Path: + return Path(__file__).parent.parent.parent / "scripts" / "consistency_check.py" + + +def _split_slug_list(value: str) -> list[str]: + return [part.strip() for part in value.split(",") if part.strip()] + + +def _parse_stderr(stderr: str) -> dict[str, list[str]]: + return { + "skipped_clean": _split_slug_list(_CLEAN_RE.search(stderr).group(1)) + if _CLEAN_RE.search(stderr) + else [], + "skipped_missing": _split_slug_list(_MISSING_RE.search(stderr).group(1)) + if _MISSING_RE.search(stderr) + else [], + "skipped_budget": _split_slug_list(_BUDGET_RE.search(stderr).group(1)) + if _BUDGET_RE.search(stderr) + else [], + } + + +def _extract_json_payload(text: str) -> Any: + stripped = text.strip() + if not stripped: + return [] + decoder = json.JSONDecoder() + for index, char in enumerate(stripped): + if char not in "{[": + continue + try: + payload, _end = decoder.raw_decode(stripped, index) + return payload + except json.JSONDecodeError: + continue + raise json.JSONDecodeError("No JSON payload found", stripped, 0) + + +def _parse_stdout(stdout: str) -> list[ConsistencySweepRepoResult]: + text = stdout.strip() + if not text: + return [] + payload = _extract_json_payload(text) + items = payload if isinstance(payload, list) else [payload] + results: list[ConsistencySweepRepoResult] = [] + for item in items: + summary = item.get("summary") or {} + results.append( + ConsistencySweepRepoResult( + repo_slug=str(item.get("repo_slug") or ""), + repo_path=str(item.get("repo_path") or ""), + result=str(item.get("result") or "pass"), + summary=ConsistencySweepIssueSummary( + fail=int(summary.get("fail", 0)), + automation_error=int(summary.get("automation_error", 0)), + warn=int(summary.get("warn", 0)), + info=int(summary.get("info", 0)), + ), + fixes_applied=list(item.get("fixes_applied") or []), + ) + ) + return results + + +async def run_remote_all_sweep( + session: AsyncSession, + *, + max_seconds: int, + source: str = "api", +) -> ConsistencySweepRemoteAllRun: + started_at = datetime.now(tz=UTC) + cmd = [ + sys.executable, + str(_script_path()), + "--remote", + "--all", + "--json", + "--api-base", + settings.api_base, + "--max-seconds", + str(max_seconds), + ] + result = await asyncio.to_thread( + subprocess.run, + cmd, + capture_output=True, + text=True, + ) + completed_at = datetime.now(tz=UTC) + lock_skipped = _LOCK_SKIP_MARKER in result.stderr + stderr_meta = _parse_stderr(result.stderr) + repos_processed = [] if lock_skipped else _parse_stdout(result.stdout) + + automation_error = result.returncode != 0 and not lock_skipped + progress_event_id = await _log_sweep_progress( + session, + started_at=started_at, + completed_at=completed_at, + max_seconds=max_seconds, + source=source, + exit_code=result.returncode, + automation_error=automation_error, + lock_skipped=lock_skipped, + repos_processed=repos_processed, + **stderr_meta, + ) + return ConsistencySweepRemoteAllRun( + started_at=started_at, + completed_at=completed_at, + max_seconds=max_seconds, + source=source, + exit_code=result.returncode, + automation_error=automation_error, + lock_skipped=lock_skipped, + repos_processed=repos_processed, + skipped_clean=stderr_meta["skipped_clean"], + skipped_missing=stderr_meta["skipped_missing"], + skipped_budget=stderr_meta["skipped_budget"], + progress_event_id=progress_event_id, + ) + + +async def _log_sweep_progress( + session: AsyncSession, + *, + started_at: datetime, + completed_at: datetime, + max_seconds: int, + source: str, + exit_code: int, + automation_error: bool, + lock_skipped: bool, + repos_processed: list[ConsistencySweepRepoResult], + skipped_clean: list[str], + skipped_missing: list[str], + skipped_budget: list[str], +) -> uuid.UUID: + processed_count = len(repos_processed) + error_count = sum(1 for repo in repos_processed if repo.result == "error") + assessment_fail_count = sum(1 for repo in repos_processed if repo.result == "fail") + warn_count = sum(1 for repo in repos_processed if repo.result == "warn") + if lock_skipped: + summary = "State Hub consistency sweep skipped: prior remote-all run still active" + elif automation_error: + summary = ( + "State Hub consistency sweep automation error: " + f"exit_code={exit_code}, {processed_count} repos partially processed" + ) + else: + summary = ( + "State Hub consistency sweep completed: " + f"{processed_count} processed, {len(skipped_clean)} clean, " + f"{len(skipped_missing)} missing, {len(skipped_budget)} budget-skipped, " + f"{assessment_fail_count} assessment-fail, {error_count} automation-error, " + f"{warn_count} warned" + ) + event = ProgressEvent( + event_type="consistency_sweep_remote_all", + summary=summary, + detail={ + "started_at": _iso(started_at), + "completed_at": _iso(completed_at), + "max_seconds": max_seconds, + "source": source, + "exit_code": exit_code, + "automation_error": automation_error, + "assessment_failures": assessment_fail_count, + "automation_errors": error_count, + "lock_skipped": lock_skipped, + "repos_processed": [item.model_dump(mode="json") for item in repos_processed], + "skipped_clean": skipped_clean, + "skipped_missing": skipped_missing, + "skipped_budget": skipped_budget, + }, + author="state-hub", + ) + session.add(event) + await session.commit() + await session.refresh(event) + return event.id + + +def _iso(value: datetime) -> str: + return value.astimezone(UTC).isoformat().replace("+00:00", "Z") \ No newline at end of file diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/execution_queue.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/execution_queue.py new file mode 100644 index 00000000..afd4dba9 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/execution_queue.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from typing import Any + +from api.workplan_status import normalize_workplan_status + + +EXECUTION_STATES = { + "manual": "Not queued for autonomous pickup; humans or agents may still work manually.", + "queued": "Candidate for ordered pickup when dependencies and concurrency allow it.", + "scheduled": "Waiting for an external launch window; State Hub stores the requested time.", + "launching": "Legacy state only; it does not prove that any consumer accepted pickup.", + "paused": "Temporarily held outside the pickup stack.", + "completed": "Execution intent is closed; lifecycle status remains authoritative.", + "cancelled": "Execution intent was cancelled without changing lifecycle status.", +} + +LAUNCH_MODES = { + "manual": "Do not request automation; keep intent visible only.", + "queued": "Place in the prioritized stack for later pickup.", + "scheduled": "Request pickup at or after a selected time.", + "immediate": "Legacy intent only; workplan launch-request pickup is retired.", +} + +CONCURRENCY_MODES = { + "sequential": "Respect queue order and avoid parallel pickup for the same group.", + "parallel": "Eligible for concurrent pickup with other ready work.", +} + +STATE_HUB_RESPONSIBILITIES = [ + "store lifecycle status separately from execution intent", + "rank candidate workplans and expose dependency-aware eligibility", + "preserve historical launch-request and execution-intent rows during retirement", + "reject new workplan launch requests because no pickup consumer exists", +] + +ACTIVITY_CORE_RESPONSIBILITIES = [ + "own schedules, wakeups, and recurring automation", + "claim and complete ops runs created by ActivityDefinition fires", + "do not consume State Hub workplan launch requests", +] + +EXECUTION_REPLACEMENTS = { + "POST /execution/launch-requests": ( + "Queue work in the authoritative repository file; for recurring or operational " + "fires use an ActivityDefinition and activity-core ops_run" + ), + "GET /execution/launch-requests": "GET /ops-runs on activity-core for automation history", + "GET /execution/workplan-stack": "Repo Manager work index or hub-core projection", + "PATCH /execution/workplans/{id}/intent": "Edit the authoritative workplan file", +} + +EXECUTION_STATE_RANK = { + "launching": 0, + "queued": 1, + "scheduled": 2, + "manual": 3, + "paused": 4, + "completed": 5, + "cancelled": 6, +} + +PRIORITY_RANK = { + "critical": 0, + "high": 1, + "medium": 2, + "low": 3, +} + +CLOSED_WORKPLAN_STATUSES = {"finished", "archived"} + + +def execution_state_for_launch(launch_mode: str, immediate_pickup: bool = False) -> str: + mode = (launch_mode or "queued").strip().lower() + if immediate_pickup or mode == "immediate": + return "launching" + if mode == "scheduled": + return "scheduled" + if mode == "manual": + return "manual" + return "queued" + + +def workplan_blockers( + workplan_id: Any, + dependency_targets: dict[Any, list[Any]], + workplan_status: dict[Any, str], + workstream_id: Any = None, +) -> list[Any]: + scope_id = workplan_id if workplan_id is not None else workstream_id + blockers = [] + for target_id in dependency_targets.get(scope_id, []): + target_status = normalize_workplan_status(workplan_status.get(target_id)) + if target_status not in CLOSED_WORKPLAN_STATUSES: + blockers.append(target_id) + return blockers + + +workstream_blockers = workplan_blockers + + +def queue_sort_key(workstream: Any, *, eligible: bool) -> list[int | str]: + priority = str(getattr(workstream, "planning_priority", "") or "").strip().lower() + execution_state = str(getattr(workstream, "execution_state", "") or "manual").strip().lower() + return [ + 0 if eligible else 1, + EXECUTION_STATE_RANK.get(execution_state, 99), + PRIORITY_RANK.get(priority, 50), + getattr(workstream, "queue_rank", None) if getattr(workstream, "queue_rank", None) is not None else 999_999, + getattr(workstream, "planning_order", None) if getattr(workstream, "planning_order", None) is not None else 999_999, + str(getattr(workstream, "slug", "") or ""), + ] diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/fabric_graph.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/fabric_graph.py new file mode 100644 index 00000000..48f1fef6 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/fabric_graph.py @@ -0,0 +1,689 @@ +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from typing import Any + +from pydantic import ValidationError +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from api.models.fabric_graph import FabricGraphEdge, FabricGraphImport, FabricGraphNode +from api.models.progress_event import ProgressEvent +from api.schemas.fabric_graph import FabricGraphExportPayload + +DISPLAY_ONLY_EDGE_TYPES = { + "collapsed_into", + "declares", + "grouped_with", + "highlight_path", + "near", + "owns_deployment", + "same_color_group", +} + + +class FabricGraphValidationError(Exception): + def __init__(self, detail: dict[str, Any]) -> None: + super().__init__(str(detail)) + self.detail = detail + + +def split_graph_ingest_body(body: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: + """Support both direct FabricGraphExport payloads and the documented wrapper.""" + if isinstance(body.get("graph"), dict): + metadata = {key: value for key, value in body.items() if key != "graph"} + return body["graph"], metadata + return body, {} + + +async def ingest_fabric_graph_export( + session: AsyncSession, + payload: dict[str, Any], + *, + source_repo_slug: str, + source_url: str | None, + requested_by: str, +) -> tuple[FabricGraphImport, bool, bool]: + now = datetime.now(timezone.utc) + content_hash = graph_content_hash(payload) + try: + export = validate_fabric_graph_export(payload) + except ValueError as exc: + import_run = await _record_invalid_import( + session, + payload, + source_repo_slug=source_repo_slug, + source_url=source_url, + content_hash=content_hash, + error=str(exc), + now=now, + requested_by=requested_by, + ) + raise FabricGraphValidationError( + { + "message": str(exc), + "import_id": str(import_run.id), + "validation_status": import_run.validation_status, + } + ) from exc + + existing = await _find_import(session, source_repo_slug, content_hash) + if existing and existing.validation_status == "valid": + await _mark_latest(session, existing) + existing.last_seen_at = now + await _record_progress( + session, + "Fabric graph export already ingested; refreshed latest marker.", + { + "source_repo_slug": source_repo_slug, + "source_url": source_url, + "content_hash": content_hash, + "node_count": existing.node_count, + "edge_count": existing.edge_count, + "requested_by": requested_by, + "idempotent": True, + }, + ) + await session.commit() + await session.refresh(existing) + return existing, False, True + + source = export.source + import_run = FabricGraphImport( + source_repo_slug=source_repo_slug, + source_url=source_url, + source_commit=source.commit if source else None, + source_path=source.path if source else None, + api_version=export.api_version, + schema_version=export.schema_version, + export_kind=export.kind, + exported_at=export.generated_at, + netkingdom_id=export.netkingdom.id if export.netkingdom else None, + king_actor_id=export.netkingdom.king_actor_id if export.netkingdom else None, + actor_count=len(export.actors), + fabric_count=len(export.fabrics), + unresolved_count=len(export.unresolved), + compatibility=export.compatibility, + content_hash=content_hash, + node_count=len(export.nodes), + edge_count=len(export.edges), + validation_status="valid", + error_details=None, + graph_json=export.model_dump(mode="json", by_alias=True), + is_latest=True, + last_seen_at=now, + ) + await _mark_previous_not_latest(session, source_repo_slug) + session.add(import_run) + await session.flush() + + for node in export.nodes: + raw = node.model_dump(mode="json") + containment = raw.get("containment") if isinstance(raw.get("containment"), dict) else {} + ownership = raw.get("ownership") if isinstance(raw.get("ownership"), dict) else {} + accounting = raw.get("accounting") if isinstance(raw.get("accounting"), dict) else {} + evidence = raw.get("evidence") if isinstance(raw.get("evidence"), dict) else {} + session.add( + FabricGraphNode( + import_id=import_run.id, + source_repo_slug=source_repo_slug, + graph_id=node.id, + kind=node.kind, + name=node.name, + repo_slug=node.repo or "", + domain_slug=node.domain or "", + lifecycle=node.lifecycle or "", + canonical_type=raw.get("canonical_type"), + canon_category=node.canon_category, + canon_anchor=node.canon_anchor, + mapping_fit=node.mapping_fit, + evidence_state=node.evidence_state or evidence.get("state"), + evidence_review_state=evidence.get("review_state"), + evidence_confidence=_float_or_none(evidence.get("confidence")), + netkingdom_id=containment.get("netkingdom_id"), + fabric_id=containment.get("fabric_id"), + subfabric_id=containment.get("subfabric_id"), + environment=containment.get("environment"), + deployment_scenario_id=containment.get("deployment_scenario_id"), + owner_actor_id=ownership.get("owner_actor_id"), + owner_role=ownership.get("owner_role"), + ownership_resolution=ownership.get("resolution"), + cost_center_id=accounting.get("cost_center_id"), + profit_center_id=accounting.get("profit_center_id"), + display_only=bool(raw.get("display_only", False)), + attributes=node.attributes, + raw_json=raw, + ) + ) + + for edge in export.edges: + raw = edge.model_dump(mode="json", by_alias=True) + provider = raw.get("provider") if isinstance(raw.get("provider"), dict) else {} + consumer = raw.get("consumer") if isinstance(raw.get("consumer"), dict) else {} + boundary = raw.get("boundary") if isinstance(raw.get("boundary"), dict) else {} + utility = raw.get("utility") if isinstance(raw.get("utility"), dict) else {} + accounting = raw.get("accounting") if isinstance(raw.get("accounting"), dict) else {} + evidence = raw.get("evidence") if isinstance(raw.get("evidence"), dict) else {} + session.add( + FabricGraphEdge( + import_id=import_run.id, + source_repo_slug=source_repo_slug, + edge_key=edge_key(raw), + from_graph_id=edge.from_graph_id, + to_graph_id=edge.to_graph_id, + edge_type=edge.edge_type, + canonical_type=edge.canonical_type, + canon_anchor=edge.canon_anchor, + mapping_fit=edge.mapping_fit, + evidence_state=edge.evidence_state or evidence.get("state"), + evidence_review_state=evidence.get("review_state"), + evidence_confidence=_float_or_none(evidence.get("confidence")), + relationship_category=edge.relationship_category, + provider_owner_actor_id=provider.get("owner_actor_id"), + provider_fabric_id=provider.get("fabric_id"), + provider_subfabric_id=provider.get("subfabric_id"), + consumer_owner_actor_id=consumer.get("owner_actor_id"), + consumer_fabric_id=consumer.get("fabric_id"), + consumer_subfabric_id=consumer.get("subfabric_id"), + crosses_fabric_boundary=boundary.get("crosses_fabric_boundary"), + crosses_subfabric_boundary=boundary.get("crosses_subfabric_boundary"), + utility_type=utility.get("utility_type"), + utility_contract_id=utility.get("contract_id"), + utility_payment_schema_id=utility.get("payment_schema_id"), + utility_metering_basis=utility.get("metering_basis"), + utility_business_model=utility.get("business_model"), + cost_center_id=accounting.get("cost_center_id"), + profit_center_id=accounting.get("profit_center_id"), + provider_profit_center_id=accounting.get("provider_profit_center_id"), + consumer_cost_center_id=accounting.get("consumer_cost_center_id"), + display_only=bool(edge.display_only), + attributes=edge.attributes, + raw_json=raw, + ) + ) + + await _record_progress( + session, + "Fabric graph export ingested as State Hub read model.", + { + "source_repo_slug": source_repo_slug, + "source_url": source_url, + "content_hash": content_hash, + "node_count": len(export.nodes), + "edge_count": len(export.edges), + "requested_by": requested_by, + }, + ) + await session.commit() + await session.refresh(import_run) + return import_run, True, False + + +async def activate_fabric_graph_import( + session: AsyncSession, + import_id: Any, + *, + requested_by: str, +) -> tuple[FabricGraphImport | None, Any | None, bool]: + """Select a retained valid import as the active read model. + + Imports are immutable. Activation only moves the per-source ``is_latest`` + marker, which makes cutover rollback fast and avoids re-fetching an older + authority payload during an incident. + """ + result = await session.execute( + select(FabricGraphImport).where(FabricGraphImport.id == import_id) + ) + import_run = result.scalar_one_or_none() + if import_run is None: + return None, None, False + if import_run.validation_status != "valid": + raise ValueError("Only a valid Fabric graph import can be activated.") + + previous_result = await session.execute( + select(FabricGraphImport).where( + FabricGraphImport.source_repo_slug == import_run.source_repo_slug, + FabricGraphImport.is_latest.is_(True), + ) + ) + previous = previous_result.scalars().first() + if previous is not None and previous.id == import_run.id: + return import_run, previous.id, False + + await _mark_latest(session, import_run) + import_run.last_seen_at = datetime.now(timezone.utc) + await _record_progress( + session, + "Fabric graph read model activation changed.", + { + "source_repo_slug": import_run.source_repo_slug, + "import_id": str(import_run.id), + "previous_import_id": str(previous.id) if previous else None, + "content_hash": import_run.content_hash, + "requested_by": requested_by, + }, + ) + await session.commit() + await session.refresh(import_run) + return import_run, previous.id if previous else None, True + + +def validate_fabric_graph_export(payload: dict[str, Any]) -> FabricGraphExportPayload: + try: + export = FabricGraphExportPayload.model_validate(payload) + except ValidationError as exc: + first = exc.errors()[0] + location = ".".join(str(part) for part in first.get("loc", [])) or "" + message = first.get("msg", "invalid payload") + raise ValueError(f"invalid FabricGraphExport at {location}: {message}") from exc + + contract_errors = _contract_errors(export, payload) + if contract_errors: + raise ValueError(f"invalid FabricGraphExport contract: {contract_errors[0]}") + canon_errors = _canon_metadata_errors(export) + if canon_errors: + raise ValueError(f"invalid FabricGraphExport canon metadata: {canon_errors[0]}") + return export + + +def graph_content_hash(payload: dict[str, Any]) -> str: + canonical = _canonical_payload(payload) + raw = json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def edge_key(edge: dict[str, Any]) -> str: + raw = json.dumps(edge, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def _float_or_none(value: Any) -> float | None: + return float(value) if isinstance(value, (int, float)) else None + + +async def record_fabric_graph_error( + session: AsyncSession, + summary: str, + *, + source_repo_slug: str, + source_url: str | None, + error: str, + requested_by: str, +) -> None: + await _record_progress( + session, + summary, + { + "source_repo_slug": source_repo_slug, + "source_url": source_url, + "error": error, + "requested_by": requested_by, + }, + ) + await session.commit() + + +async def _find_import( + session: AsyncSession, source_repo_slug: str, content_hash: str +) -> FabricGraphImport | None: + result = await session.execute( + select(FabricGraphImport).where( + FabricGraphImport.source_repo_slug == source_repo_slug, + FabricGraphImport.content_hash == content_hash, + ) + ) + return result.scalar_one_or_none() + + +async def _mark_latest(session: AsyncSession, import_run: FabricGraphImport) -> None: + await _mark_previous_not_latest(session, import_run.source_repo_slug, exclude_id=import_run.id) + import_run.is_latest = True + + +async def _mark_previous_not_latest( + session: AsyncSession, + source_repo_slug: str, + *, + exclude_id: Any | None = None, +) -> None: + stmt = update(FabricGraphImport).where(FabricGraphImport.source_repo_slug == source_repo_slug) + if exclude_id is not None: + stmt = stmt.where(FabricGraphImport.id != exclude_id) + await session.execute(stmt.values(is_latest=False)) + + +async def _record_invalid_import( + session: AsyncSession, + payload: dict[str, Any], + *, + source_repo_slug: str, + source_url: str | None, + content_hash: str, + error: str, + now: datetime, + requested_by: str, +) -> FabricGraphImport: + existing = await _find_import(session, source_repo_slug, content_hash) + if existing and existing.validation_status == "invalid": + existing.last_seen_at = now + import_run = existing + else: + import_run = FabricGraphImport( + source_repo_slug=source_repo_slug, + source_url=source_url, + # Invalid payloads are retained for diagnostics, but their + # denormalized provenance must not make the rejection path fail. + # The complete input remains available in ``graph_json``. + source_commit=_bounded_source_value(payload, "commit", max_length=255), + source_path=_source_value(payload, "path"), + api_version=str(payload.get("apiVersion")) if payload.get("apiVersion") else None, + export_kind=str(payload.get("kind")) if payload.get("kind") else None, + exported_at=_parse_datetime(payload.get("generated_at")), + content_hash=content_hash, + node_count=0, + edge_count=0, + validation_status="invalid", + error_details={"error": error}, + graph_json=payload, + is_latest=False, + last_seen_at=now, + ) + session.add(import_run) + await session.flush() + + await _record_progress( + session, + "Fabric graph export rejected during validation.", + { + "source_repo_slug": source_repo_slug, + "source_url": source_url, + "content_hash": content_hash, + "error": error, + "requested_by": requested_by, + }, + ) + await session.commit() + await session.refresh(import_run) + return import_run + + +async def _record_progress(session: AsyncSession, summary: str, detail: dict[str, Any]) -> None: + session.add( + ProgressEvent( + event_type="fabric_graph_import", + summary=summary, + detail=detail, + author="state-hub", + ) + ) + + +def _canonical_payload(payload: dict[str, Any]) -> dict[str, Any]: + canonical = json.loads(json.dumps(payload, sort_keys=True, default=str)) + canonical.pop("generated_at", None) + return canonical + + +def _contract_errors(export: FabricGraphExportPayload, payload: dict[str, Any]) -> list[str]: + if export.api_version == "railiance.fabric/v1alpha2": + return _financial_contract_errors(export, payload) + return _legacy_contract_errors(export) + + +def _legacy_contract_errors(export: FabricGraphExportPayload) -> list[str]: + errors: list[str] = [] + if export.schema_version: + errors.append("v1alpha1 exports must not set schema_version") + for index, node in enumerate(export.nodes): + _require_fields( + errors, + f"nodes[{index}]", + { + "repo": node.repo, + "domain": node.domain, + "lifecycle": node.lifecycle, + }, + ("repo", "domain", "lifecycle"), + ) + return errors + + +def _financial_contract_errors( + export: FabricGraphExportPayload, payload: dict[str, Any] +) -> list[str]: + errors: list[str] = [] + for field in ("schema_version", "netkingdom", "actors", "fabrics"): + if field not in payload: + errors.append(f"missing required financial export field {field!r}") + if export.schema_version != "financial-fabric-v1": + errors.append("schema_version must be 'financial-fabric-v1' for v1alpha2 exports") + if export.netkingdom is None: + errors.append("netkingdom must be an object for v1alpha2 exports") + netkingdom_id = "" + king_actor_id = "" + else: + netkingdom_id = export.netkingdom.id + king_actor_id = export.netkingdom.king_actor_id + + actor_roles: dict[str, str] = {} + for index, actor in enumerate(export.actors): + if actor.id in actor_roles: + errors.append(f"actors[{index}].id {actor.id!r} is duplicated") + actor_roles[actor.id] = actor.role + if king_actor_id and actor_roles.get(king_actor_id) != "king": + errors.append("netkingdom.king_actor_id must reference an actor with role 'king'") + + fabric_kinds: dict[str, str] = {} + for index, fabric in enumerate(export.fabrics): + if fabric.id in fabric_kinds: + errors.append(f"fabrics[{index}].id {fabric.id!r} is duplicated") + fabric_kinds[fabric.id] = fabric.kind + if fabric.netkingdom_id != netkingdom_id: + errors.append(f"fabrics[{index}].netkingdom_id must match netkingdom.id") + if fabric.kind == "Fabric": + if not fabric.lord_actor_id: + errors.append(f"fabrics[{index}].lord_actor_id is required for Fabric") + elif actor_roles.get(fabric.lord_actor_id) not in {"lord", "king"}: + errors.append(f"fabrics[{index}].lord_actor_id must reference a lord or king actor") + if fabric.kind == "Subfabric": + if not fabric.parent_fabric_id: + errors.append(f"fabrics[{index}].parent_fabric_id is required for Subfabric") + elif fabric.parent_fabric_id not in fabric_kinds: + errors.append( + f"fabrics[{index}].parent_fabric_id references unknown fabric {fabric.parent_fabric_id!r}" + ) + if not fabric.tenant_actor_id: + errors.append(f"fabrics[{index}].tenant_actor_id is required for Subfabric") + elif actor_roles.get(fabric.tenant_actor_id) != "tenant": + errors.append(f"fabrics[{index}].tenant_actor_id must reference a tenant actor") + + node_ids: set[str] = set() + for index, node in enumerate(export.nodes): + path = f"nodes[{index}]" + if node.id in node_ids: + errors.append(f"{path}.id {node.id!r} is duplicated") + node_ids.add(node.id) + if node.containment is None: + errors.append(f"{path}.containment must be an object for v1alpha2 exports") + else: + if node.containment.netkingdom_id != netkingdom_id: + errors.append(f"{path}.containment.netkingdom_id must match netkingdom.id") + _validate_fabric_ref(errors, f"{path}.containment.fabric_id", node.containment.fabric_id, fabric_kinds, "Fabric") + if node.containment.subfabric_id: + _validate_fabric_ref( + errors, + f"{path}.containment.subfabric_id", + node.containment.subfabric_id, + fabric_kinds, + "Subfabric", + ) + if node.ownership is None: + errors.append(f"{path}.ownership must be an object for v1alpha2 exports") + else: + _validate_actor_ref(errors, f"{path}.ownership.owner_actor_id", node.ownership.owner_actor_id, actor_roles) + if actor_roles.get(node.ownership.owner_actor_id) not in {node.ownership.owner_role, ""}: + errors.append(f"{path}.ownership.owner_role does not match referenced actor role") + if node.evidence is None: + errors.append(f"{path}.evidence must be an object for v1alpha2 exports") + elif ( + node.evidence.review_state == "accepted" + and node.ownership + and node.ownership.resolution not in {"explicit", "inherited"} + ): + errors.append(f"{path}.ownership.resolution must be explicit or inherited for accepted nodes") + + for index, edge in enumerate(export.edges): + path = f"edges[{index}]" + if edge.from_graph_id not in node_ids: + errors.append(f"{path}.from references unknown node {edge.from_graph_id!r}") + if edge.to_graph_id not in node_ids: + errors.append(f"{path}.to references unknown node {edge.to_graph_id!r}") + if edge.relationship_category is None: + errors.append(f"{path}.relationship_category is required for v1alpha2 exports") + if edge.evidence is None: + errors.append(f"{path}.evidence must be an object for v1alpha2 exports") + if edge.edge_type == "provides_utility_to" and edge.relationship_category != "utility": + errors.append(f"{path}.relationship_category must be 'utility' for provides_utility_to edges") + if edge.relationship_category == "utility": + if edge.provider is None: + errors.append(f"{path}.provider is required for utility edges") + else: + _validate_utility_side(errors, f"{path}.provider", edge.provider, actor_roles, fabric_kinds) + if edge.consumer is None: + errors.append(f"{path}.consumer is required for utility edges") + else: + _validate_utility_side(errors, f"{path}.consumer", edge.consumer, actor_roles, fabric_kinds) + if edge.boundary is None: + errors.append(f"{path}.boundary is required for utility edges") + if edge.utility is None: + errors.append(f"{path}.utility is required for utility edges") + return errors + + +def _validate_actor_ref( + errors: list[str], + path: str, + actor_id: str, + actor_roles: dict[str, str], +) -> None: + if actor_id not in actor_roles: + errors.append(f"{path} references unknown actor {actor_id!r}") + + +def _validate_fabric_ref( + errors: list[str], + path: str, + fabric_id: str, + fabric_kinds: dict[str, str], + expected_kind: str, +) -> None: + actual_kind = fabric_kinds.get(fabric_id) + if actual_kind is None: + errors.append(f"{path} references unknown fabric {fabric_id!r}") + elif actual_kind != expected_kind: + errors.append(f"{path} must reference a {expected_kind}") + + +def _validate_utility_side( + errors: list[str], + path: str, + side: Any, + actor_roles: dict[str, str], + fabric_kinds: dict[str, str], +) -> None: + _validate_actor_ref(errors, f"{path}.owner_actor_id", side.owner_actor_id, actor_roles) + if side.fabric_id not in fabric_kinds: + errors.append(f"{path}.fabric_id references unknown fabric {side.fabric_id!r}") + if side.subfabric_id: + _validate_fabric_ref(errors, f"{path}.subfabric_id", side.subfabric_id, fabric_kinds, "Subfabric") + + +def _canon_metadata_errors(export: FabricGraphExportPayload) -> list[str]: + errors: list[str] = [] + for index, node in enumerate(export.nodes): + if any( + value is not None + for value in ( + node.canon_category, + node.canon_anchor, + node.mapping_fit, + node.evidence_state, + ) + ): + _require_fields( + errors, + f"nodes[{index}]", + { + "canon_category": node.canon_category, + "mapping_fit": node.mapping_fit, + "evidence_state": node.evidence_state, + }, + ("canon_category", "mapping_fit", "evidence_state"), + ) + for index, edge in enumerate(export.edges): + has_canon_fields = any( + value is not None + for value in ( + edge.canonical_type, + edge.canon_anchor, + edge.mapping_fit, + edge.display_only, + edge.evidence_state, + ) + ) + if has_canon_fields: + _require_fields( + errors, + f"edges[{index}]", + { + "mapping_fit": edge.mapping_fit, + "display_only": edge.display_only, + "evidence_state": edge.evidence_state, + }, + ("mapping_fit", "display_only", "evidence_state"), + ) + if edge.edge_type in DISPLAY_ONLY_EDGE_TYPES and edge.display_only is not True: + errors.append( + f"edges[{index}] uses display-only edge type {edge.edge_type!r} without display_only=true" + ) + if edge.display_only is True and edge.edge_type and not has_canon_fields: + errors.append(f"edges[{index}] is display-only but lacks canon metadata") + return errors + + +def _require_fields( + errors: list[str], + path: str, + item: dict[str, Any], + fields: tuple[str, ...], +) -> None: + for field in fields: + if item.get(field) in (None, ""): + errors.append(f"{path} missing required canon metadata field {field!r}") + + +def _source_value(payload: dict[str, Any], field: str) -> str | None: + source = payload.get("source") + if not isinstance(source, dict): + return None + value = source.get(field) + return str(value) if value else None + + +def _bounded_source_value( + payload: dict[str, Any], field: str, *, max_length: int +) -> str | None: + value = _source_value(payload, field) + return value[:max_length] if value is not None else None + + +def _parse_datetime(value: Any) -> datetime | None: + if not isinstance(value, str) or not value: + return None + try: + normalized = value.replace("Z", "+00:00") + return datetime.fromisoformat(normalized) + except ValueError: + return None diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/forge_credential.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/forge_credential.py new file mode 100644 index 00000000..5714b650 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/forge_credential.py @@ -0,0 +1,122 @@ +"""Resolve the forge read credential (STATE-WP-0084-T03, MASON-WP-0003-T05). + +Three sources, tried in order: a mounted file, an environment variable, and +OpenBao via Kubernetes auth. Production uses the third — this cluster has no +agent injector and no secrets-store CSI driver, so the pod authenticates with +its projected ServiceAccount token and reads the KV path itself, which is what +`MASON-WP-0003-T02` built the Kubernetes auth role for. The first two exist so +the code is runnable and testable outside the cluster. + +**Absent is a supported state, never an error.** A hub with no credential still +derives every public repository; only private ones become unreadable, and +`STATE-WP-0084-T01` already reports that as its own named condition rather than +as records that stopped deriving. So every failure here — no configuration, no +network, OpenBao down, permission denied — resolves to `None`. Raising would +convert "cannot read nine repositories" into "the whole pass failed". +""" + +from __future__ import annotations + +import logging +import os +import time +from pathlib import Path + +import httpx + +logger = logging.getLogger(__name__) + +TOKEN_ENV = "FORGE_READ_TOKEN" +TOKEN_FILE_ENV = "FORGE_READ_TOKEN_FILE" +OPENBAO_ADDR_ENV = "OPENBAO_ADDR" +OPENBAO_ROLE_ENV = "OPENBAO_K8S_ROLE" +OPENBAO_JWT_PATH_ENV = "OPENBAO_K8S_TOKEN_PATH" +OPENBAO_AUTH_MOUNT_ENV = "OPENBAO_K8S_AUTH_MOUNT" +SECRET_PATH_ENV = "FORGE_READ_SECRET_PATH" +SECRET_KEY_ENV = "FORGE_READ_SECRET_KEY" + +# Long enough that a fleet reset of 121 repositories does not re-authenticate +# 121 times; short enough that a rotated token is picked up without a redeploy, +# which is what MASON-WP-0003-T02 requires of this lane. +CACHE_TTL_SECONDS = 300.0 + +_cache: tuple[float, str | None] | None = None + + +def reset_cache() -> None: + global _cache + _cache = None + + +def _from_file() -> str | None: + path = os.environ.get(TOKEN_FILE_ENV) + if not path: + return None + try: + return Path(path).read_text(encoding="utf-8").strip() or None + except OSError: + # Deliberately not falling through to the environment: a broken mount + # that silently used a stale value would look like success. + logger.warning("forge credential: token file %s is unreadable", path) + return None + + +def _from_env() -> str | None: + return (os.environ.get(TOKEN_ENV) or "").strip() or None + + +def _from_openbao() -> str | None: + addr = (os.environ.get(OPENBAO_ADDR_ENV) or "").strip().rstrip("/") + secret_path = (os.environ.get(SECRET_PATH_ENV) or "").strip().strip("/") + role = (os.environ.get(OPENBAO_ROLE_ENV) or "").strip() + jwt_path = os.environ.get(OPENBAO_JWT_PATH_ENV) or "/var/run/secrets/openbao/token" + mount = (os.environ.get(OPENBAO_AUTH_MOUNT_ENV) or "kubernetes").strip("/") + key = (os.environ.get(SECRET_KEY_ENV) or "token").strip() + if not (addr and secret_path and role): + return None + try: + jwt = Path(jwt_path).read_text(encoding="utf-8").strip() + except OSError: + logger.warning("forge credential: no ServiceAccount token at %s", jwt_path) + return None + try: + with httpx.Client(timeout=10.0) as client: + login = client.post( + f"{addr}/v1/auth/{mount}/login", json={"role": role, "jwt": jwt} + ) + login.raise_for_status() + client_token = login.json()["auth"]["client_token"] + read = client.get( + f"{addr}/v1/{secret_path}", headers={"X-Vault-Token": client_token} + ) + read.raise_for_status() + data = read.json()["data"] + # KV v2 nests the payload under a second "data"; v1 does not. + if isinstance(data.get("data"), dict): + data = data["data"] + except (httpx.HTTPError, KeyError, ValueError) as exc: + # Never include the response body: a failed KV read can echo content. + logger.warning("forge credential: OpenBao lookup failed (%s)", type(exc).__name__) + return None + value = data.get(key) + if not isinstance(value, str) or not value.strip(): + logger.warning("forge credential: key %r absent at the KV path", key) + return None + return value.strip() + + +def forge_read_token(*, use_cache: bool = True) -> str | None: + """The forge read credential, or `None` if this instance has none.""" + global _cache + now = time.monotonic() + if use_cache and _cache is not None and now - _cache[0] < CACHE_TTL_SECONDS: + return _cache[1] + if os.environ.get(TOKEN_FILE_ENV): + # Configured to use a file means *that* file and nothing else. Falling + # back would let a broken mount quietly resolve to a stale environment + # value that nobody knows is in use. + token = _from_file() + else: + token = _from_env() or _from_openbao() + _cache = (now, token) + return token diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/forge_projection.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/forge_projection.py new file mode 100644 index 00000000..ce339d9e --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/forge_projection.py @@ -0,0 +1,2257 @@ +"""Derive a repository's work-record projection from the forge (STATE-WP-0083-T01). + +`ADR-012` decision 1 makes the forge the projection source. Central does its own +reading: it clones the repository's default branch and derives from that, rather +than accepting a projection computed elsewhere — which `ADR-010` decision 5 +forbids. + +This module is read-only. Deriving must be safe to run at any time, because it is +what makes the reset in `T03` verifiable: you can always ask what the projection +*should* be without changing anything. +""" + +from __future__ import annotations + +import os +import base64 +import re +import subprocess +import tempfile +import uuid +from dataclasses import dataclass, field +from datetime import date +from pathlib import Path +from typing import Any + +import yaml + +# Same derivation as ADR-007 / repo-manager, so a forge-derived projection and a +# preliminary overlay compute identical identities for the same record. +from api.services.task_record_id_backfill import qualify_task_id +from api.work_record_flavor import ( + FLAVOR_PROMOTION_REASONS, WORK_RECORD_FLAVORS, + normalize_flavor, normalize_promotion_reason, +) + +_WORK_RECORD_NAMESPACE = uuid.UUID("a4058507-5c4a-5a00-ab06-fffa4fb46009") +_TASK_BLOCK_RE = re.compile(r"```task\s*\n(.*?)\n```", re.DOTALL) +_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)$", re.MULTILINE) +DEFAULT_FORGE_BASE = "https://forgejo.coulomb.social/coulomb" + + +_IDENTIFIER_SLUG = re.compile( + r"^[a-z0-9-]*-wp-\d{3,}(?:-t\d{2})?$" # PREFIX-WP-0001, -T01 + r"|^[a-z0-9-]*adhoc[a-z0-9-]*?-\d{4}-\d{2}-\d{2}(?:-t\d{2})?$" # daily ad-hoc, qualified either side +) + + +def _slug_is_identifier(slug: str) -> bool: + """Whether a row's slug is a work-record identifier or a title slug. + + This is what separates a re-key from a rename when the path cannot. A row + whose slug is an identifier is *claiming* to be that record, so if the forge + derives a different identifier for the same file, they are two records and + the old one retires. A row whose slug is a title (`three-phoenix-ha-cluster` + for `RCLUSTER-WP-0007`) never claimed one: those are hub-first rows from + before ADR-001, where the backing path is the only link there has ever been, + and path matching is the only thing that can hold them together. + """ + return bool(_IDENTIFIER_SLUG.match((slug or "").strip().lower())) + + +def _identity_is_derived(row: Any) -> bool: + """Whether this row's UUID was derived from its identifier (ADR-007). + + Derived rows are UUIDv5 over the work-record namespace. Rows predating + derived identity carry a random v4 UUID, and for those the identifier is a + label rather than the identity — so the path fallback still applies, which + is what keeps rename detection working for legacy records. + """ + rid = getattr(row, "id", None) + return getattr(rid, "version", None) == 5 + + +def derived_record_uuid(record_id: str) -> str: + return str(uuid.uuid5(_WORK_RECORD_NAMESPACE, f"helixforge\n{record_id}")) + + +class ForgeDeriveError(RuntimeError): + """The repository could not be read from the forge.""" + + +class ForgeUnreadableError(ForgeDeriveError): + """Central is not permitted to read this repository — a policy, not a fault. + + `ADR-012`'s premise (the forge is the projection source) holds only for + repositories central can read, and until `STATE-WP-0084` nothing said so: + a private repository failed the same way a broken one did, so "cannot read" + and "does not exist" were indistinguishable from the outside. + + They must never be confused, because they authorise opposite things. A + repository that does not derive may have had its files removed deliberately; + a repository we cannot read tells us nothing at all about its files. Only + the first can justify retiring a record. + + Forgejo answers an unauthenticated request for a private repository with a + 404, so "not found" is classified as unreadable too. That is deliberate: the + two cases are genuinely indistinguishable at this layer, and the safe + reading of an ambiguous answer is the one that cannot destroy a record. + """ + + +# git says this in several ways depending on version, transport, and whether a +# credential helper is installed; all of them mean the same thing here. +_UNREADABLE_MARKERS = ( + "could not read username", + "could not read password", + "authentication failed", + "terminal prompts disabled", + "invalid username or password", + "403 forbidden", + "the requested url returned error: 403", + "the requested url returned error: 401", + "repository not found", + "remote: not found", + "does not appear to be a git repository", +) + + +def _is_unreadable(message: str) -> bool: + low = message.lower() + if "not found" in low and "fatal: repository" in low: + return True + return any(marker in low for marker in _UNREADABLE_MARKERS) + + +@dataclass +class DerivedTask: + record_id: str + uuid: str + title: str | None + status: str | None + priority: str | None + description: str | None = None + needs_human: bool = False + intervention_note: str | None = None + blocking_reason: str | None = None + flavor: str | None = None + decision_id: str | None = None + blocked_on: str | None = None + wait_until: date | None = None + # Task-block ``depends_on`` targets (workplan or task record ids), in + # source order, deduplicated. Unqualified ``T##`` targets are qualified + # with the owning workplan. + depends_on: list[str] = field(default_factory=list) + + +@dataclass +class DerivedWorkplan: + record_id: str + uuid: str + title: str | None + status: str | None + relative_path: str + archived: bool + owner: str | None = None + description: str | None = None + tasks: list[DerivedTask] = field(default_factory=list) + flavor: str | None = None + flavor_promotion_reason: str | None = None + flavor_promoted_from: str | None = None + historical_preservation: bool = False + # Frontmatter ``depends_on`` / ``depends_on_workplans`` / ``depends_on_tasks`` + # split as C-20 splits them: task record ids vs workplan record ids. + depends_on_workplans: list[str] = field(default_factory=list) + depends_on_tasks: list[str] = field(default_factory=list) + + +@dataclass +class DerivedProjection: + repo_slug: str + commit: str + workplans: list[DerivedWorkplan] = field(default_factory=list) + # False when the checkout has no `workplans/` directory at all. An empty + # projection then means "we did not find the records", which is not the same + # claim as "this repository has no records" — and only the second one could + # ever justify retiring anything. + records_source_present: bool = True + + @property + def task_count(self) -> int: + return sum(len(w.tasks) for w in self.workplans) + + @property + def retirement_eligible(self) -> bool: + """Whether an absence in this projection is evidence of an absence. + + A projection that could not be read never reaches this: it raises. What + this rules out is the quieter case — a clone that succeeded and returned + nothing, which is what would have retired every record in a repository + had a clone ever come back empty instead of failing (`STATE-WP-0084-T01`; + the near-miss was `vergabe-teilnahme`). + """ + return self.records_source_present + + def to_dict(self) -> dict[str, Any]: + return { + "schema": "state-hub.forge-projection.v1", + "repo_slug": self.repo_slug, + # Provenance is not optional: a projection that cannot name the + # commit it came from cannot be audited (ADR-012 decision 2). + "commit": self.commit, + "records_source_present": self.records_source_present, + "retirement_eligible": self.retirement_eligible, + "workplans": [ + { + "record_id": w.record_id, + "uuid": w.uuid, + "title": w.title, + "status": w.status, + "owner": w.owner, + "description": w.description, + "flavor": w.flavor, + "flavor_promotion_reason": w.flavor_promotion_reason, + "flavor_promoted_from": w.flavor_promoted_from, + "relative_path": w.relative_path, + "archived": w.archived, + "historical_preservation": w.historical_preservation, + "depends_on_workplans": list(w.depends_on_workplans), + "depends_on_tasks": list(w.depends_on_tasks), + "tasks": [ + { + "record_id": t.record_id, + "uuid": t.uuid, + "title": t.title, + "status": t.status, + "priority": t.priority, + "description": t.description, + "needs_human": t.needs_human, + "intervention_note": t.intervention_note, + "blocking_reason": t.blocking_reason, + "flavor": t.flavor, + "decision_id": t.decision_id, + "blocked_on": t.blocked_on, + "wait_until": t.wait_until.isoformat() if t.wait_until else None, + "depends_on": list(t.depends_on), + } + for t in w.tasks + ], + } + for w in self.workplans + ], + } + + +from api.services.forge_credential import forge_read_token # noqa: F401 + +# Kept as module attributes so callers and tests that reached for them here +# still resolve after the sources moved to `forge_credential`. +FORGE_TOKEN_ENV = "FORGE_READ_TOKEN" +FORGE_TOKEN_FILE_ENV = "FORGE_READ_TOKEN_FILE" + + +def _credential_env(token: str | None) -> dict[str, str]: + """Git config carrying the credential, passed by environment not argv. + + `-c http.extraHeader=...` would place the token in the process command line, + where it is readable by anything that can run `ps` and lands in any log that + records invocations. GIT_CONFIG_* achieves the same configuration without + that exposure. + """ + if not token: + return {} + header = base64.b64encode(f"x-access-token:{token}".encode()).decode() + return { + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "http.extraHeader", + "GIT_CONFIG_VALUE_0": f"Authorization: Basic {header}", + } + + +def _run_git( + *args: str, cwd: str | None = None, timeout: float = 120.0, token: str | None = None +) -> str: + # Without this a clone of a private repository blocks on a username prompt + # instead of failing, and an unattended derivation pass hangs rather than + # reporting. Failing fast is what makes the unreadable case observable. + env = {**os.environ, "GIT_TERMINAL_PROMPT": "0", "GIT_ASKPASS": "", "GCM_INTERACTIVE": "never"} + env.update(_credential_env(token)) + proc = subprocess.run( + ["git", *args], cwd=cwd, capture_output=True, text=True, timeout=timeout, env=env + ) + if proc.returncode != 0: + detail = (proc.stderr or proc.stdout).strip()[:400] + if token: + # Never let a credential reach an exception that is logged, stored + # in a reset outcome, or returned over the API. + detail = detail.replace(token, "***") + raise ForgeDeriveError(detail) + return proc.stdout.strip() + + +def _split_frontmatter(text: str) -> tuple[dict, str]: + if not text.startswith("---"): + return {}, text + end = text.find("\n---", 3) + if end == -1: + return {}, text + raw = text[3:end] + body = text[end + 4 :] + try: + meta = yaml.safe_load(raw) or {} + except yaml.YAMLError: + return {}, text + return (meta if isinstance(meta, dict) else {}), body + + +def _clean_task_description(raw: str) -> str | None: + """Match the file-authoritative consistency parser's section boundaries.""" + lines = raw.splitlines() + while lines and (not lines[0].strip() or lines[0].strip() in {"---", "***", "___"}): + lines.pop(0) + while lines and (not lines[-1].strip() or lines[-1].strip() in {"---", "***", "___"}): + lines.pop() + return "\n".join(lines).strip() or None + + + +def _source_flavor(value: Any, field_name: str = "flavor") -> str | None: + flavor = normalize_flavor(value) + if flavor is not None and flavor not in WORK_RECORD_FLAVORS: + raise ForgeDeriveError(f"Unknown source {field_name}; no projection was applied") + return flavor + + +def _source_promotion_reason(value: Any) -> str | None: + reason = normalize_promotion_reason(value) + if reason is not None and reason not in FLAVOR_PROMOTION_REASONS: + raise ForgeDeriveError("Unknown source flavor_promotion_reason; no projection was applied") + return reason + + +def _source_wait_until(value: Any) -> date | None: + if value is None or value == "": + return None + if type(value) is date: + return value + text = str(value).strip() + if re.fullmatch(r"\d{4}-\d{2}-\d{2}", text): + try: + return date.fromisoformat(text) + except ValueError: + pass + raise ForgeDeriveError("Source wait_until must be a valid YYYY-MM-DD date; no projection was applied") + + +def _source_wait_text(value: Any) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise ForgeDeriveError("Source wait qualifier must be text; no projection was applied") + return value.strip() or None + + +def _source_needs_human(value: Any) -> bool: + if isinstance(value, bool): + return value + if value is None: + return False + text = str(value).strip().lower() + if text in {"true", "yes", "1", "on"}: + return True + if text in {"false", "no", "0", "off", ""}: + return False + raise ForgeDeriveError("Source needs_human must be boolean; no projection was applied") + + +_TASK_TARGET_RE = re.compile(r"-T\d{2,3}$") +_SHORT_TASK_TARGET_RE = re.compile(r"^T\d{2,3}$", re.IGNORECASE) + + +def _source_dependency_list(value: Any, field_name: str) -> list[str]: + """Return declared dependency targets, refusing shapes C-20 cannot read. + + A mapping or nested list is not a dependency declaration; dropping it would + silently lose an edge, so the whole derive is refused instead. + """ + if value is None or value == "": + return [] + if isinstance(value, str): + items = value.split(",") + elif isinstance(value, list): + if any(isinstance(item, (dict, list)) for item in value): + raise ForgeDeriveError(f"Malformed source {field_name}; no projection was applied") + items = [str(item) for item in value if item is not None] + elif isinstance(value, dict): + raise ForgeDeriveError(f"Malformed source {field_name}; no projection was applied") + else: + items = [str(value)] + out: list[str] = [] + for item in items: + target = item.strip().strip('"').strip("'").strip() + if target and target not in out: + out.append(target) + return out + + +def _split_dependency_targets(meta: dict, owner_record_id: str) -> tuple[list[str], list[str]]: + """Split declared targets into (workplan ids, task ids) like C-20 does.""" + declared = _source_dependency_list(meta.get("depends_on"), "depends_on") + workplans = [t for t in _source_dependency_list(meta.get("depends_on_workplans"), "depends_on_workplans")] + tasks = _source_dependency_list(meta.get("depends_on_tasks"), "depends_on_tasks") + for target in declared: + if _SHORT_TASK_TARGET_RE.match(target): + target = qualify_task_id(target, owner_record_id) or target + (tasks if _TASK_TARGET_RE.search(target) else workplans).append(target) + tasks = [qualify_task_id(t, owner_record_id) or t if _SHORT_TASK_TARGET_RE.match(t) else t for t in tasks] + dedupe = lambda items: list(dict.fromkeys(items)) # noqa: E731 + return dedupe(workplans), dedupe(tasks) + + +def _parse_tasks(body: str, workplan_id: str) -> list[DerivedTask]: + headings = [ + (m.start(), len(m.group(1)), m.group(2).strip()) for m in _HEADING_RE.finditer(body) + ] + out: list[DerivedTask] = [] + task_matches = list(_TASK_BLOCK_RE.finditer(body)) + for index, m in enumerate(task_matches): + try: + block = yaml.safe_load(m.group(1).strip()) or {} + except (yaml.YAMLError, ValueError) as exc: + # PyYAML raises ValueError for impossible bare ISO dates. + # Refuse the entire derive instead of dropping a stored gate. + raise ForgeDeriveError("Malformed source task metadata; no projection was applied") from exc + if not isinstance(block, dict): + continue + rid = str(block.get("id") or "").strip() + if not rid: + continue + prev = [(pos, level, text) for pos, level, text in headings if pos < m.start()] + heading = prev[-1] if prev else None + title = block.get("title") or (heading[2] if heading else None) + # Nested evidence belongs to this task. Stop at the next peer/ancestor + # heading or task block, as the ordinary consistency path does. + description_end = task_matches[index + 1].start() if index + 1 < len(task_matches) else len(body) + following_heading = next(( + pos for pos, level, _text in headings + if pos > m.end() and (heading is None or level <= heading[1]) + ), None) + if following_heading is not None: + description_end = min(description_end, following_heading) + description = _clean_task_description(body[m.end():description_end]) + if description is None and block.get("description") is not None: + description = str(block["description"]) + out.append( + DerivedTask( + # A bare `T01` is not an identifier: it is unique only within + # its workplan, so `uuid5("T01")` is the same UUID for every + # workplan in the fleet. llm-connect's 91 task blocks derive + # just 49 distinct UUIDs unqualified, and creating its + # workplans fails on a duplicate task primary key. + # + # Qualifying with the owning workplan is the same rule + # `task_record_id_backfill.qualify_task_id` applies to stored + # ids; both must agree or the backfill and the projection + # disagree about what a task is called. + record_id=qualify_task_id(rid, workplan_id) or rid, + # Derived, not read from the file: the forge projection must not + # inherit an identifier the file happens to carry. + uuid=derived_record_uuid(qualify_task_id(rid, workplan_id) or rid), + title=title, + status=(str(block["status"]).strip() if block.get("status") else None), + priority=(str(block["priority"]).strip() if block.get("priority") else None), + description=description, + needs_human=_source_needs_human(block.get("needs_human", False)), + intervention_note=( + str(block["intervention_note"]) + if block.get("intervention_note") is not None + else (str(block["blocking_reason"]) if _source_needs_human(block.get("needs_human")) and block.get("blocking_reason") is not None else None) + ), + blocking_reason=( + str(block["blocking_reason"]) + if block.get("blocking_reason") is not None + else None + ), + flavor=_source_flavor(block.get("flavor")), + decision_id=_source_wait_text(block.get("decision_id")), + blocked_on=_source_wait_text(block.get("blocked_on")), + wait_until=_source_wait_until(block.get("wait_until")), + depends_on=_task_dependency_targets(block, workplan_id), + ) + ) + return out + + +def _task_dependency_targets(block: dict, workplan_id: str) -> list[str]: + workplans, tasks = _split_dependency_targets(block, workplan_id) + return list(dict.fromkeys(workplans + tasks)) + + +def _workplan_description(body: str) -> str | None: + """Return bounded prose under ``## Goal`` when one is present.""" + match = re.search(r"^##\s+Goal\s*$", body, re.MULTILINE | re.IGNORECASE) + if match is None: + return None + remainder = body[match.end() :] + next_heading = re.search(r"^##\s+", remainder, re.MULTILINE) + if next_heading is not None: + remainder = remainder[: next_heading.start()] + value = remainder.strip() + return value[:4000] or None + + +def derive_from_checkout(repo_root: Path, repo_slug: str, commit: str) -> DerivedProjection: + """Derive a projection from an already-materialised checkout.""" + proj = DerivedProjection(repo_slug=repo_slug, commit=commit) + wp_dir = repo_root / "workplans" + if not wp_dir.is_dir(): + proj.records_source_present = False + return proj + # Exact immutable histories are preserved by existing UUID, never allocated. + from api.services.historical_projection import bound_sources + try: + historical = bound_sources(repo_root, repo_slug) + except (ValueError, OSError, UnicodeError) as exc: + raise ForgeDeriveError(str(exc)) from exc + consumed = {entry["path"] for entry, _text in historical} + groups = {} + for entry, text in historical: + meta, body = _split_frontmatter(text) + parent_id = entry["parent_record_id"] + # Split views carry the parent's existing UUID and repeat its task list. + # Only the reviewed archived parent supplies the projected row. + if entry["record_id"] != parent_id: + continue + if repo_slug == "activity-core": + tasks = [DerivedTask(record_id=item["id"], uuid=item["state_hub_task_id"], + title=item.get("title"), status=item.get("status"), + priority=item.get("priority")) for item in meta["tasks"]] + else: + tasks = _parse_tasks(body, parent_id) + bindings = [yaml.safe_load(m.group(1)) for m in _TASK_BLOCK_RE.finditer(body)] + for task, binding in zip(tasks, bindings, strict=True): + task.uuid = binding["state_hub_task_id"] + groups[parent_id] = DerivedWorkplan( + record_id=parent_id, uuid=entry["uuid"], title=meta.get("title"), + status=meta.get("status"), relative_path=entry["path"], + archived="/archived/" in entry["path"], tasks=tasks, + historical_preservation=True, + ) + proj.workplans.extend(groups.values()) + for path in sorted(wp_dir.rglob("*.md")): + if path.relative_to(repo_root).as_posix() in consumed: + continue + if path.name.startswith("."): + continue + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + meta, body = _split_frontmatter(text) + if str(meta.get("type") or "").strip() != "workplan": + continue + rid = str(meta.get("id") or "").strip() + if not rid: + continue + frontmatter_deps = _split_dependency_targets(meta, rid) + proj.workplans.append( + DerivedWorkplan( + record_id=rid, + uuid=derived_record_uuid(rid), + title=(str(meta["title"]).strip() if meta.get("title") else None), + status=(str(meta["status"]).strip() if meta.get("status") else None), + relative_path=str(path.relative_to(repo_root).as_posix()), + archived=path.parent.name == "archived", + owner=(str(meta["owner"]).strip() if meta.get("owner") else None), + description=( + str(meta["description"]).strip() + if meta.get("description") + else _workplan_description(body) + ), + tasks=_parse_tasks(body, rid), + flavor=_source_flavor(meta.get("flavor")), + flavor_promotion_reason=_source_promotion_reason(meta.get("flavor_promotion_reason")), + flavor_promoted_from=_source_flavor(meta.get("flavor_promoted_from"), "flavor_promoted_from"), + depends_on_workplans=frontmatter_deps[0], + depends_on_tasks=frontmatter_deps[1], + ) + ) + proj.workplans.sort(key=lambda w: w.record_id) + return proj + + +def derive_from_forge( + repo_slug: str, *, forge_base: str = DEFAULT_FORGE_BASE, ref: str | None = None +) -> DerivedProjection: + """Clone the repository's default branch from the forge and derive from it. + + A fresh shallow clone every time, deliberately: the source is what the forge + holds now, and a reused working copy is how a projection ends up reflecting + someone's local state instead (ADR-012 context). + """ + url = f"{forge_base.rstrip('/')}/{repo_slug}.git" + token = forge_read_token() + with tempfile.TemporaryDirectory(prefix=f"forge-{repo_slug}-") as tmp: + args = ["clone", "--depth", "1", "--quiet"] + if ref: + args += ["--branch", ref] + try: + _run_git(*args, url, tmp, token=token) + except subprocess.TimeoutExpired as exc: + raise ForgeDeriveError(f"clone timed out for {repo_slug}") from exc + except ForgeDeriveError as exc: + # Classify before propagating. A caller that cannot tell "not + # permitted" from "broken" will eventually treat one as the other. + if _is_unreadable(str(exc)): + raise ForgeUnreadableError( + f"{repo_slug} could not be read from the forge: {exc}" + ) from exc + raise + commit = _run_git("rev-parse", "HEAD", cwd=tmp) + return derive_from_checkout(Path(tmp), repo_slug, commit) + + +# --------------------------------------------------------------------------- +# Comparison against what the hub currently holds (STATE-WP-0083-T02) +# --------------------------------------------------------------------------- + + + +_ARCHIVE_PREFIX_RE = re.compile(r"^\d{6}-") + + +def _path_key(path: str) -> str: + """Normalise a workplan path so an archived copy matches its live one.""" + name = path.rsplit("/", 1)[-1] + return _ARCHIVE_PREFIX_RE.sub("", name).strip().lower() + + +@dataclass +class ProjectionDiff: + """What a reset would change, computed without changing anything. + + This is what makes `ADR-012` decision 7's "verifiable" real: the reset can + always be inspected before it runs, and its result compared against the forge + afterwards. + """ + + repo_slug: str + commit: str + missing: list[dict[str, Any]] = field(default_factory=list) # forge has, hub lacks + stale: list[dict[str, Any]] = field(default_factory=list) # hub has, forge lacks + differing: list[dict[str, Any]] = field(default_factory=list) # both, fields differ + # More than one row/file claims one canonical identity. Choosing a winner + # would turn a diagnostic into an implicit identity decision. + ambiguous: list[dict[str, Any]] = field(default_factory=list) + # Set when the source could not support a claim of absence, so `stale` was + # deliberately left empty rather than computed (`STATE-WP-0084-T01`). + stale_withheld: str | None = None + # Source-owned dependency-edge preview (STATE-WP-0094-T05); None when the + # caller supplied no hub edge state. Keys: create, remove, retain, + # unresolved, external. + edges: dict[str, list[dict[str, Any]]] | None = None + + @property + def clean(self) -> bool: + edge_changes = bool(self.edges and (self.edges["create"] or self.edges["remove"])) + return not (self.missing or self.stale or self.differing or self.ambiguous or edge_changes) + + @property + def would_remove(self) -> int: + return len(self.stale) + + def to_dict(self) -> dict[str, Any]: + return { + "schema": "state-hub.projection-diff.v1", + "repo_slug": self.repo_slug, + "commit": self.commit, + "clean": self.clean, + "counts": { + "missing": len(self.missing), + "stale": len(self.stale), + "differing": len(self.differing), + "ambiguous": len(self.ambiguous), + }, + "stale_withheld": self.stale_withheld, + "missing": self.missing, + "stale": self.stale, + "differing": self.differing, + "ambiguous": self.ambiguous, + **({} if self.edges is None else { + "edge_counts": {k: len(v) for k, v in self.edges.items()}, + "edges": self.edges, + }), + } + + +def _derived_identity_ambiguities(derived: DerivedProjection) -> list[dict[str, Any]]: + """Return duplicate canonical identities declared by one forge projection.""" + workplans: dict[str, list[DerivedWorkplan]] = {} + tasks: dict[str, list[tuple[DerivedWorkplan, DerivedTask]]] = {} + for workplan in derived.workplans: + workplans.setdefault(workplan.record_id.strip().lower(), []).append(workplan) + for task in workplan.tasks: + tasks.setdefault(task.record_id.strip().lower(), []).append((workplan, task)) + + out: list[dict[str, Any]] = [] + for rows in workplans.values(): + if len(rows) > 1: + out.append( + { + "kind": "workplan", + "record_id": rows[0].record_id, + "source": "forge", + "paths": [row.relative_path for row in rows], + } + ) + for rows in tasks.values(): + if len(rows) > 1: + out.append( + { + "kind": "task", + "record_id": rows[0][1].record_id, + "source": "forge", + "workplans": [row[0].record_id for row in rows], + } + ) + return out + + +def diff_against_hub( + derived: DerivedProjection, + hub_workplans: list[dict[str, Any]], + hub_tasks_by_workplan: dict[str, list[dict[str, Any]]], + *, + hub_edges: list[dict[str, Any]] | None = None, + edge_targets: dict[str, list[str]] | None = None, +) -> ProjectionDiff: + """Compare a derived projection with the hub's current records. + + Pure: takes the hub's state as data rather than reading it, so the comparison + is testable without a database and cannot accidentally mutate anything. + + Dependency edges are previewed only when ``hub_edges`` is given: the + ``workplan_dependencies`` rows whose from-workplan belongs to this + repository (with ``source_owner``). ``edge_targets`` maps a target record id + to its fleet-wide, non-retired candidate ids; a target absent from it but + declared in this projection resolves to the id the receipt would create. + """ + d = ProjectionDiff(repo_slug=derived.repo_slug, commit=derived.commit) + d.ambiguous.extend(_derived_identity_ambiguities(derived)) + if not derived.retirement_eligible: + # Compute what is missing and what differs as usual — those only ever + # add or correct. Absence is the one conclusion this source cannot + # support, so it is not drawn at all rather than drawn and then filtered. + d.stale_withheld = ( + "the checkout has no workplans/ directory, so an absent record is " + "unexplained rather than evidence of removal" + ) + + # Match on canonical identity, never on UUID. Most hub records still carry + # pre-ADR-007 random identifiers, so a UUID-keyed comparison reports every + # record as simultaneously missing and stale — and a reset built on that + # would destroy and recreate the entire projection. The canonical record id + # is what is stable across the identifier migration; the backing file is the + # fallback when a hub slug was derived from a filename rather than an id. + def _wp_key(record_id: str | None, slug: str | None, path: str | None) -> str: + if record_id: + return record_id.strip().lower() + if path: + return _path_key(path) + return (slug or "").strip().lower() + + want_wp = {w.record_id.strip().lower(): w for w in derived.workplans} + have_wp: dict[str, dict[str, Any]] = {} + want_paths = {_path_key(w.relative_path): k for k, w in want_wp.items()} + preserved_by_uuid = {w.uuid: key for key, w in want_wp.items() if w.historical_preservation} + for w in hub_workplans: + if str(w["id"]) in preserved_by_uuid: + have_wp[preserved_by_uuid[str(w["id"])]] = w + continue + slug = str(w.get("slug") or "") + key = slug.strip().lower() + if key not in want_wp: + # Slugs were not always the canonical id; fall back to the file. + bp = w.get("backing_relative_path") + if bp and _path_key(bp) in want_paths: + key = want_paths[_path_key(bp)] + else: + cand = [k for k in want_wp if slug.lower().startswith(k + "-")] + if len(cand) == 1: + key = cand[0] + have_wp[key] = w + + for key, w in want_wp.items(): + if key not in have_wp: + d.missing.append({"kind": "workplan", "record_id": w.record_id, "uuid": w.uuid}) + for key, w in have_wp.items(): + if key not in want_wp: + if d.stale_withheld: + continue + uid = str(w["id"]) + d.stale.append( + { + "kind": "workplan", + "uuid": uid, + "slug": w.get("slug"), + "status": w.get("status"), + # A stale record with no backing file is the case that must + # never be destroyed silently (ADR-012 decision 7). + "has_backing_file": bool( + w.get("backing_filename") or w.get("backing_relative_path") + ), + } + ) + for key, w in want_wp.items(): + cur = have_wp.get(key) + uid = str(cur["id"]) if cur else w.uuid + if not cur: + continue + if w.historical_preservation: + if cur.get("status") not in {"done", "finished", "archived", "completed"}: + d.differing.append({"kind": "workplan", "record_id": w.record_id, "uuid": uid, + "changed": {"historical_terminal_status": cur.get("status")}}) + continue + changed = {} + if (cur.get("status") or None) != (w.status or None): + changed["status"] = {"hub": cur.get("status"), "forge": w.status} + for field_name in ("flavor", "flavor_promotion_reason", "flavor_promoted_from"): + expected = getattr(w, field_name) + if cur.get(field_name) != expected: + changed[field_name] = {"hub": cur.get(field_name), "forge": expected} + cur_path = cur.get("backing_relative_path") or None + if cur_path != w.relative_path: + changed["backing_relative_path"] = {"hub": cur_path, "forge": w.relative_path} + if changed: + d.differing.append( + {"kind": "workplan", "record_id": w.record_id, "uuid": uid, "changed": changed} + ) + + for w in derived.workplans: + hub_rows = hub_tasks_by_workplan.get(w.uuid, []) + if not hub_rows: + cur = have_wp.get(w.record_id.strip().lower()) + if cur: + hub_rows = hub_tasks_by_workplan.get(str(cur["id"]), []) + if w.historical_preservation: + by_uuid = {str(task["id"]): task for task in hub_rows} + for task in w.tasks: + current = by_uuid.get(task.uuid) + if current is None: + d.missing.append({"kind": "task", "record_id": task.record_id, + "uuid": task.uuid, "workplan": w.record_id}) + elif current.get("status") not in {"done", "cancel"}: + d.differing.append({"kind": "task", "record_id": task.record_id, + "uuid": task.uuid, "changed": { + "historical_terminal_status": current.get("status")}}) + continue + # Match on the canonical record id where the hub has one. Rows created + # before STATE-WP-0083-T06 fall back to title, which is why those are + # reported rather than acted on: a renamed heading is indistinguishable + # from a replaced task under title matching. + def _task_key(record_id: str | None, title: str | None) -> str: + if record_id: + return record_id.strip().lower() + return "title:" + (title or "").strip().lower() + + have_groups: dict[str, list[dict[str, Any]]] = {} + for task in hub_rows: + have_groups.setdefault( + _task_key(task.get("record_id"), task.get("title")), [] + ).append(task) + for key, rows in have_groups.items(): + if len(rows) > 1: + d.ambiguous.append( + { + "kind": "task", + "record_id": rows[0].get("record_id"), + "source": "hub", + "workplan": w.record_id, + "uuids": [str(row["id"]) for row in rows], + "match_key": key, + } + ) + # Keep one representative so the rest of the diff remains useful, but + # `ambiguous` makes the result non-clean and forbids treating it as an + # actionable reset plan. + have = {key: rows[0] for key, rows in have_groups.items()} + want = {_task_key(t.record_id, t.title): t for t in w.tasks} + for key, t in want.items(): + if key not in have: + d.missing.append( + {"kind": "task", "record_id": t.record_id, "uuid": t.uuid, + "workplan": w.record_id} + ) + for key, t in have.items(): + if key not in want: + if d.stale_withheld: + continue + uid = str(t["id"]) + d.stale.append( + {"kind": "task", "uuid": uid, "title": t.get("title"), + "status": t.get("status"), "workplan": w.record_id, + "has_backing_file": False} + ) + for key, t in want.items(): + cur = have.get(key) + if not cur: + continue + changed = {} + if (cur.get("status") or None) != (t.status or None): + changed["status"] = {"hub": cur.get("status"), "forge": t.status} + for field_name in ("needs_human", "intervention_note", "blocking_reason", + "flavor", "decision_id", "blocked_on", "wait_until"): + expected = getattr(t, field_name) + actual = cur.get(field_name, False if field_name == "needs_human" else None) + if isinstance(actual, date): + actual = actual.isoformat() + if isinstance(expected, date): + expected = expected.isoformat() + if actual != expected: + changed[field_name] = {"hub": actual, "forge": expected} + if changed: + d.differing.append( + {"kind": "task", "record_id": t.record_id, "uuid": t.uuid, + "workplan": w.record_id, "changed": changed} + ) + if hub_edges is not None: + d.edges = _preview_source_edges( + derived, want_wp, have_wp, hub_tasks_by_workplan, hub_edges, edge_targets or {} + ) + return d + + +def _preview_source_edges( + derived: DerivedProjection, + want_wp: dict[str, DerivedWorkplan], + have_wp: dict[str, dict[str, Any]], + hub_tasks_by_workplan: dict[str, list[dict[str, Any]]], + hub_edges: list[dict[str, Any]], + edge_targets: dict[str, list[str]], +) -> dict[str, list[dict[str, Any]]]: + """What receiving would do to source-owned edges, using the same planner.""" + from_ids: dict[str, str] = {} + task_ids: dict[str, dict[str, str]] = {} + pending: dict[str, str] = {} # record id -> id the receipt would create + for key, w in want_wp.items(): + cur = have_wp.get(key) + wp_id = str(cur["id"]) if cur else w.uuid + from_ids[key] = wp_id + if not cur: + pending[w.record_id] = wp_id + hub_rows = hub_tasks_by_workplan.get(wp_id) or hub_tasks_by_workplan.get(w.uuid) or [] + by_record = {(r.get("record_id") or "").strip().lower(): str(r["id"]) + for r in hub_rows if r.get("record_id")} + for t in w.tasks: + k = t.record_id.strip().lower() + if k not in by_record: + by_record[k] = t.uuid + pending[t.record_id] = t.uuid + task_ids[wp_id] = by_record + targets: dict[str, tuple[str, Any]] = {} + for target in source_edge_targets(want_wp): + cands = sorted({str(c) for c in edge_targets.get(target, [])}) + if not cands and target in pending: + cands = [pending[target]] + if not cands: + targets[target] = ("missing", None) + elif len(cands) > 1: + targets[target] = ("ambiguous", cands) + else: + targets[target] = ("ok", cands[0]) + desired, unresolved, held_sides = plan_source_edges(want_wp, from_ids, task_ids, targets) + + def _s(v: Any) -> str | None: + return str(v) if v else None + + existing = [ + ((str(e["from_workplan_id"]), _s(e.get("from_task_id")), _s(e.get("to_workplan_id")), + _s(e.get("to_task_id")), e.get("relationship_type") or "blocks"), + str(e.get("id")), e.get("source_owner")) + for e in hub_edges + ] + plan = classify_source_edges(desired, held_sides, existing, edge_source_owner(derived.repo_slug)) + return { + "create": [_edge_dict(k, i) for k, i in plan["create"]], + "remove": [_edge_dict(k, {"id": rid}) for k, rid in plan["remove"]], + "retain": [_edge_dict(k, {"id": rid}) for k, rid in plan["retain"]], + "unresolved": unresolved, + "external": [_edge_dict(k, i) for k, i in plan["external"]], + } + + +# --------------------------------------------------------------------------- +# Applying the reset (STATE-WP-0083-T03) +# --------------------------------------------------------------------------- + + +@dataclass +class ResetOutcome: + repo_slug: str + commit: str + status: str # applied | refused | noop | unreadable + created: list[str] = field(default_factory=list) + updated: list[str] = field(default_factory=list) + retired: list[str] = field(default_factory=list) + refused: list[dict[str, Any]] = field(default_factory=list) + # Identifiers freed from rows retired before retirement released them. + released: list[str] = field(default_factory=list) + created_tasks: list[str] = field(default_factory=list) + updated_tasks: list[str] = field(default_factory=list) + cancelled_tasks: list[str] = field(default_factory=list) + # Source-owned dependency edges (STATE-WP-0094-T05). + created_edges: list[dict[str, Any]] = field(default_factory=list) + removed_edges: list[dict[str, Any]] = field(default_factory=list) + # Declared edges whose target is missing, ambiguous or invalid: recorded, + # never silently dropped; the from-side's owned edges are retained. + unresolved_edges: list[dict[str, Any]] = field(default_factory=list) + # Declared edges already present as externally authored rows: preserved + # and not adopted, so receiving can never remove them. + external_edges: list[dict[str, Any]] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "schema": "state-hub.projection-reset.v1", + "repo_slug": self.repo_slug, + "commit": self.commit, + "status": self.status, + "counts": { + "created": len(self.created), + "updated": len(self.updated), + "retired": len(self.retired), + "refused": len(self.refused), + "released": len(self.released), + "created_tasks": len(self.created_tasks), + "updated_tasks": len(self.updated_tasks), + "cancelled_tasks": len(self.cancelled_tasks), + "created_edges": len(self.created_edges), + "removed_edges": len(self.removed_edges), + "unresolved_edges": len(self.unresolved_edges), + "external_edges": len(self.external_edges), + }, + "created": self.created, + "updated": self.updated, + "retired": self.retired, + "released": self.released, + "created_tasks": self.created_tasks, + "updated_tasks": self.updated_tasks, + "cancelled_tasks": self.cancelled_tasks, + "created_edges": self.created_edges, + "removed_edges": self.removed_edges, + "unresolved_edges": self.unresolved_edges, + "external_edges": self.external_edges, + "refused": self.refused, + "notes": self.notes, + } + + +RETIRED_SLUG_MARK = "@retired-" + + +def _tombstone_slug(slug: str, when: datetime) -> str: + """Release the identifier a retired row was holding. + + `slug` is unique across the whole table, so retirement that only sets a + timestamp leaves the identifier locked to a record nothing derives any + more — and the repository that legitimately owns it can never claim it. + That is what kept repo-seed refused after core-hub's inherited REPO-WP rows + were retired. + + The row, its UUID, and its progress events are untouched; only the + human-facing identifier is stamped, so history stays attached to the record + it happened under. Re-retiring an already-stamped row must not stack marks, + or the column overflows after a few passes. + """ + base = (slug or "").split(RETIRED_SLUG_MARK)[0] + stamped = f"{base}{RETIRED_SLUG_MARK}{when:%Y%m%d}" + return stamped[:100] + + +RETIRE_REASON = "no longer derived from the forge" + + +def _uuid_key(value: Any) -> str: + return str(uuid.UUID(str(value))) + + +async def _exact_retirement_blockers(session: Any, stale: list[Any]) -> list[dict[str, Any]]: + """Block an exact retirement when any stale row still has tasks or edges. + + Zero means zero. Terminal tasks count, and so does a dependency edge in + either direction. The caller retires nothing unless this returns empty. + """ + if not stale: + return [] + + from sqlalchemy import or_, select + + from api.models.task import Task + from api.models.workplan_dependency import WorkplanDependency + + stale_ids = [row.id for row in stale] + tasks = list( + ( + await session.execute(select(Task).where(Task.workplan_id.in_(stale_ids))) + ).scalars() + ) + edges = list( + ( + await session.execute( + select(WorkplanDependency).where( + or_( + WorkplanDependency.from_workplan_id.in_(stale_ids), + WorkplanDependency.to_workplan_id.in_(stale_ids), + ) + ) + ) + ).scalars() + ) + refused: list[dict[str, Any]] = [] + for row in stale: + key = str(row.id) + row_tasks = [task for task in tasks if str(task.workplan_id) == key] + row_edges = [ + edge + for edge in edges + if str(edge.from_workplan_id) == key + or (edge.to_workplan_id is not None and str(edge.to_workplan_id) == key) + ] + if row_tasks: + refused.append( + { + "reason": "stale projection has tasks", + "uuid": key, + "slug": row.slug, + "task_ids": [str(task.id) for task in row_tasks], + } + ) + if row_edges: + refused.append( + { + "reason": "stale projection has dependency edges", + "uuid": key, + "slug": row.slug, + "dependency_ids": [str(edge.id) for edge in row_edges], + } + ) + if (tasks or edges) and not refused: + refused.append( + { + "reason": "stale projection has tasks or dependency edges", + "task_ids": [str(task.id) for task in tasks], + "dependency_ids": [str(edge.id) for edge in edges], + } + ) + return refused + + +def _coerce_task_status(raw: str | None) -> Any: + from api.models.task import TaskStatus + + if not raw: + return None + try: + return TaskStatus(str(raw).strip().lower()) + except ValueError: + return None + + +def _task_status_value(status: Any) -> str: + return str(getattr(status, "value", status) or "") + + +def _sync_existing_workplan_tasks( + session: Any, + row: Any, + derived_wp: DerivedWorkplan, + hub_tasks: list[Any], + outcome: ResetOutcome, +) -> None: + """Match identified tasks of an existing workplan by record_id. + + Rows with no ``record_id`` predate the backfill and cannot be qualified + from the file alone (CUST-WP-0068-T09). They are not cancelled or + overwritten. + """ + from api.models.task import Task, TaskPriority, TaskStatus + + want = { + t.record_id.strip().lower(): t + for t in derived_wp.tasks + if t.record_id and t.record_id.strip() + } + matched: dict[str, Any] = {} + claimed: set[str] = set() + stale: list[Any] = [] + by_uuid = {str(ht.id): ht for ht in hub_tasks} + for ht in hub_tasks: + rid = (ht.record_id or "").strip().lower() + if rid and rid in want: + matched[rid] = ht + claimed.add(str(ht.id)) + elif rid: + stale.append(ht) + + for key, dt in want.items(): + ht = matched.get(key) + if ht is None and dt.uuid in by_uuid: + # Already on the hub under the derived UUID, but record_id was never + # written (the registrar-minted case). Overwriting would collide. + ht = by_uuid[dt.uuid] + matched[key] = ht + claimed.add(str(ht.id)) + if ht is None: + kwargs: dict[str, Any] = { + "id": uuid.UUID(dt.uuid), + "workplan_id": row.id, + "record_id": dt.record_id, + "title": (dt.title or dt.record_id), + "description": dt.description, + "needs_human": dt.needs_human, + "intervention_note": dt.intervention_note, + "blocking_reason": dt.blocking_reason, + "flavor": dt.flavor, + "decision_id": dt.decision_id, + "blocked_on": dt.blocked_on, + "wait_until": dt.wait_until, + } + st = _coerce_task_status(dt.status) + if st is not None: + kwargs["status"] = st + if dt.priority: + try: + kwargs["priority"] = TaskPriority(dt.priority.strip().lower()) + except ValueError: + pass + session.add(Task(**kwargs)) + outcome.created_tasks.append(dt.record_id) + continue + changed = False + if not (ht.record_id or "").strip(): + ht.record_id = dt.record_id + changed = True + if dt.title and dt.title.strip() and ht.title != dt.title.strip(): + ht.title = dt.title.strip() + changed = True + if getattr(ht, "description", None) != dt.description: + ht.description = dt.description + changed = True + st = _coerce_task_status(dt.status) + if st is not None and ht.status != st: + ht.status = st + changed = True + if getattr(ht, "needs_human", False) != dt.needs_human: + ht.needs_human = dt.needs_human + changed = True + if getattr(ht, "intervention_note", None) != dt.intervention_note: + ht.intervention_note = dt.intervention_note + changed = True + if getattr(ht, "blocking_reason", None) != dt.blocking_reason: + ht.blocking_reason = dt.blocking_reason + changed = True + for field_name in ("flavor", "decision_id", "blocked_on", "wait_until"): + expected = getattr(dt, field_name) + if getattr(ht, field_name, None) != expected: + setattr(ht, field_name, expected) + changed = True + if dt.priority: + try: + task_priority = TaskPriority(dt.priority.strip().lower()) + except ValueError: + task_priority = None + if task_priority is not None and getattr(ht, "priority", None) != task_priority: + ht.priority = task_priority + changed = True + if changed: + outcome.updated_tasks.append(dt.record_id) + + for ht in stale: + if _task_status_value(ht.status) in {"wait", "todo", "progress"}: + ht.status = TaskStatus.cancel + outcome.cancelled_tasks.append(ht.record_id or str(ht.id)) + + +async def reset_repository_projection( + session: Any, + repo_slug: str, + *, + acknowledge_retirements: bool = False, + expected_retirement_uuids: list[uuid.UUID] | None = None, + forge_base: str = DEFAULT_FORGE_BASE, + derived: DerivedProjection | None = None, +) -> ResetOutcome: + """Reconcile one repository's workplan projection against the forge. + + Creates what the forge has and the hub lacks, updates what differs, and + retires what no longer derives. It does not delete: hub-native records + reference workplans with `ON DELETE RESTRICT`, and destroying a progress + event to tidy a derived projection would lose hub-native truth to fix a + derived-state problem (`ADR-012` decision 7 as amended). + + Retirement is refused by default. A record that stops deriving may mean the + file was removed deliberately — or that someone pointed this at the wrong + branch. The caller must say which, either by blanket acknowledgement or by + the exact hub UUIDs expected to retire. The two authorizations cannot be + combined. Omitting the UUID list keeps blanket acknowledgement unchanged; + an empty list expects zero retirements. + + Scope: workplans, and their tasks. Tasks of existing workplans are matched + by ``record_id`` (STATE-WP-0083-T06); rows with no ``record_id`` are left + alone. A derived task the hub lacks is created; an identified hub task the + forge no longer derives is cancelled if it is still open. + """ + from datetime import datetime, timezone + + from sqlalchemy import select + + from api.models.managed_repo import ManagedRepo + from api.models.task import Task + from api.models.workplan import Workplan + + if derived is None: + try: + derived = derive_from_forge(repo_slug, forge_base=forge_base) + except ForgeUnreadableError as exc: + # Not an error: a statement about what central is permitted to see. + # Reported as its own status so a caller cannot mistake it for a + # repository whose records stopped deriving (`STATE-WP-0084-T01`). + out = ResetOutcome(repo_slug=repo_slug, commit="", status="unreadable") + out.refused.append( + {"reason": "repository could not be read from the forge", "slug": repo_slug, + "detail": str(exc)[:300]} + ) + out.notes.append( + "Nothing was changed and nothing was retired. This says nothing " + "about whether the repository's records still exist." + ) + return out + outcome = ResetOutcome(repo_slug=repo_slug, commit=derived.commit, status="noop") + + repo = ( + await session.execute(select(ManagedRepo).where(ManagedRepo.slug == repo_slug)) + ).scalar_one_or_none() + if repo is None: + outcome.status = "refused" + outcome.refused.append({"reason": "repository is not registered", "slug": repo_slug}) + return outcome + + source_ambiguities = _derived_identity_ambiguities(derived) + if source_ambiguities: + outcome.status = "refused" + for ambiguity in source_ambiguities: + outcome.refused.append( + { + "reason": "forge projection contains duplicate canonical identity", + **ambiguity, + } + ) + outcome.notes.append( + "Duplicate identities in the authoritative projection require a file-level " + "decision; the reset will not choose a winner." + ) + return outcome + + rows = list( + ( + await session.execute(select(Workplan).where(Workplan.repo_id == repo.id)) + ).scalars() + ) + + want = {w.record_id.strip().lower(): w for w in derived.workplans} + want_paths = {_path_key(w.relative_path): k for k, w in want.items()} + + # The derived UUID is the strongest key there is: if a row's id equals + # uuid5 of a wanted record's identifier, that row *is* that record, whatever + # its slug says. Checking it first is what stops a correct record with a + # legacy title slug — `testdrive-jsui-publication` for MARKITECT-WP-0002 — + # from being read as a re-key and proposed for retirement. + want_by_uuid = {w.uuid: k for k, w in want.items()} + + # Release identifiers held by rows retired before retirement freed them. + # This runs ahead of every refusal path, because it completes a decision + # already taken rather than making a new one — gating it behind an unrelated + # pending decision left four identifiers locked in repositories refusing for + # reasons having nothing to do with them. + # + # A row the forge still derives is exempt: stamping it would change the slug + # out from under the matching below, so the record would fail to match its + # own file and never be un-retired. + for r in rows: + if r.projection_retired_at is None or RETIRED_SLUG_MARK in (r.slug or ""): + continue + if (r.slug or "").strip().lower() in want or str(r.id) in want_by_uuid: + continue + before = r.slug + r.slug = _tombstone_slug(r.slug or str(r.id), r.projection_retired_at) + outcome.released.append(before) + + # How a row came to claim a key, strongest first. Two rows can claim the + # same record — `cust-wp-0010` by its own identifier and + # `workstream-lifecycle-documentation` by path, both pointing at + # CUST-WP-0010's file. Assigning into `matched` unconditionally let the + # later one win and dropped the other silently: never matched, so never + # stale, so never reported by any pass. + UUID_MATCH, SLUG_MATCH, PATH_MATCH, PREFIX_MATCH = 0, 1, 2, 3 + + matched: dict[str, Any] = {} + claim: dict[str, int] = {} + displaced: list[Any] = [] + + def _claim(key: str, row: Any, strength: int) -> None: + held = claim.get(key) + if held is None: + matched[key], claim[key] = row, strength + return + if strength < held: + # The new claim is stronger; the incumbent loses the record and + # becomes a retirement candidate rather than disappearing. + displaced.append(matched[key]) + matched[key], claim[key] = row, strength + else: + displaced.append(row) + + for row in rows: + key = (row.slug or "").strip().lower() + by_uuid = want_by_uuid.get(str(row.id)) + if by_uuid is not None: + _claim(by_uuid, row, UUID_MATCH) + continue + if key in want: + # The identifier itself derives again — including for a retired row, + # which is then deliberately un-retired below. + _claim(key, row, SLUG_MATCH) + continue + if row.projection_retired_at is None and RETIRED_SLUG_MARK in (row.slug or ""): + # A stamped slug with a cleared flag is a resurrected row: the mark + # is evidence a retirement happened and the flag says it did not. + # It must land in `matched` under its own key so it shows up as + # stale — skipping it outright, as a genuinely retired row is + # skipped, is what left these reporting `noop` forever. + matched[key] = row + continue + if row.projection_retired_at is not None: + # Already retired: nothing to do, and nothing to re-decide. + # + # Past this point only the heuristics remain, and a retired row must + # not be matched by those. Retirement is a + # decision, and re-deriving the same file must not silently undo it. + # + # Releasing the identifier makes this necessary: the tombstoned slug + # is no longer an identifier and a legacy row is not UUID-derived, so + # both re-key guards fall through to path matching — which matches + # the file the row was retired *for* and resurrects it, alongside the + # correct record already created from that same file. + # + # A record that genuinely returns matches by UUID or by its own + # identifier above, and is un-retired there. + continue + if key not in want: + if _identity_is_derived(row) or _slug_is_identifier(row.slug or ""): + # ADR-007: a record identified by an identifier *is* that + # identifier. Two cases reach here — a derived (v5) row, and a + # legacy row whose slug is still a work-record identifier. Both + # are re-keys, not renames. + # + # ADR-007: a derived record's identity *is* a function of its + # identifier, so a changed identifier means a different record — + # the old one retires and the new one is created. Matching it to + # the file by path instead would update the row in place while + # its UUID still encodes the old identifier, leaving the file + # and the hub disagreeing about what the record is called. + # + # This is not hypothetical: the ad-hoc requalification + # (CUST-WP-0066) deliberately keeps the filename, so for those + # records a re-key *never* changes the path. Path matching + # cannot tell a re-key from a rename there, and silently chose + # rename for all 30 of them. + # + # The row keeps its own slug as key, so it lands in `stale` and + # becomes a retirement candidate rather than vanishing. + matched[key] = row + continue + bp = row.backing_relative_path + if bp and _path_key(bp) in want_paths: + _claim(want_paths[_path_key(bp)], row, PATH_MATCH) + continue + cand = [k for k in want if key.startswith(k + "-")] + if len(cand) == 1: + _claim(cand[0], row, PREFIX_MATCH) + continue + matched[key] = row + + # An identifier this repository would create may already belong to another + # repository. Two repositories creating the same daily identifier on the + # same day is a documented case (CUST-WP-0066), and the derivation is + # deliberately deterministic, so the collision is real rather than + # incidental. Refuse and say so: a constraint violation is a stack trace, + # a refusal is something the caller can rule on. + creating = [w for k, w in want.items() if k not in matched] + if creating: + foreign = list( + ( + await session.execute( + select(Workplan).where( + Workplan.id.in_([uuid.UUID(w.uuid) for w in creating]), + Workplan.repo_id != repo.id, + ) + ) + ).scalars() + ) + # `slug` carries its own unique constraint across the whole table, so an + # identifier check alone is not enough: two repositories can derive + # different identifiers whose slugs still collide. Missing this is what + # left disaster-control raising IntegrityError after the identifier + # refusal was added. + slug_clash = list( + ( + await session.execute( + select(Workplan).where( + Workplan.slug.in_([w.record_id.lower() for w in creating]), + Workplan.repo_id != repo.id, + ) + ) + ).scalars() + ) + if slug_clash: + held = {(r.slug or "").lower(): r for r in slug_clash} + outcome.status = "refused" + for w in creating: + row = held.get(w.record_id.lower()) + if row is None: + continue + outcome.refused.append( + { + "reason": "slug already belongs to another repository", + "record_id": w.record_id, + "slug": w.record_id.lower(), + "held_by_id": str(row.id), + } + ) + outcome.notes.append( + "Identifier collision is an identity decision, not a projection " + "one; acknowledging retirements does not authorise it." + ) + return outcome + + if foreign: + owned = {str(r.id): r for r in foreign} + outcome.status = "refused" + for w in creating: + held = owned.get(w.uuid) + if held is None: + continue + outcome.refused.append( + { + "reason": "derived identifier already belongs to another repository", + "record_id": w.record_id, + "uuid": w.uuid, + "held_by_slug": held.slug, + } + ) + outcome.notes.append( + "Identifier collision is an identity decision, not a projection " + "one; acknowledging retirements does not authorise it." + ) + return outcome + + stale = [ + r for k, r in matched.items() + if k not in want and r.projection_retired_at is None + ] + [r for r in displaced if r.projection_retired_at is None] + # Exact UUIDs and blanket acknowledgement are different authorizations. + # A call that sets both retires nothing, before any retirement mutation. + if expected_retirement_uuids is not None and acknowledge_retirements: + outcome.status = "refused" + outcome.refused.append( + { + "reason": "ambiguous authorization", + "detail": ( + "expected_retirement_uuids and acknowledge_retirements " + "cannot both be set" + ), + } + ) + outcome.notes.append( + "Nothing was retired. Pass either an exact retirement set or " + "blanket acknowledgement, not both." + ) + return outcome + if stale and not derived.retirement_eligible: + # Acknowledgement cannot authorise this. The caller is confirming that + # records which stopped deriving should be retired; here nothing has + # been shown to have stopped deriving, because the source produced no + # records to compare against. Consenting to a conclusion is not the + # same as the evidence for it existing. + outcome.status = "refused" + for r in stale: + outcome.refused.append( + { + "reason": "source produced no records; absence is unexplained", + "slug": r.slug, + "status": r.status, + "backing_relative_path": r.backing_relative_path, + } + ) + outcome.notes.append( + "The checkout has no workplans/ directory. Retirement withheld " + "regardless of acknowledgement (STATE-WP-0084-T01)." + ) + return outcome + if expected_retirement_uuids is not None: + # `stale` is the unmatched live rows plus the displaced live rows. + # A row that still matches, including a historical parent, is not in it. + actual = {_uuid_key(row.id) for row in stale} + expected = {_uuid_key(value) for value in expected_retirement_uuids} + missing = sorted(actual - expected) + extra = sorted(expected - actual) + if missing or extra: + outcome.status = "refused" + outcome.refused.append( + { + "reason": "expected retirement set does not match stale projections", + "missing": missing, + "extra": extra, + } + ) + outcome.notes.append( + "Nothing was retired. The expected set must equal the stale " + "set; partial retirement is not applied." + ) + return outcome + blockers = await _exact_retirement_blockers(session, stale) + if blockers: + outcome.status = "refused" + outcome.refused.extend(blockers) + outcome.notes.append( + "Nothing was retired. A stale projection with tasks or " + "dependency edges cannot be retired." + ) + return outcome + elif stale and not acknowledge_retirements: + outcome.status = "refused" + for r in stale: + outcome.refused.append( + { + "reason": "would be retired; the forge no longer derives it", + "slug": r.slug, + "status": r.status, + "backing_relative_path": r.backing_relative_path, + } + ) + outcome.notes.append( + "Re-run with acknowledgement to retire these. Nothing was changed." + ) + return outcome + + now = datetime.now(tz=timezone.utc) + + existing_ids = [matched[k].id for k, _w in want.items() if k in matched] + tasks_by_wp: dict[Any, list[Any]] = {} + if existing_ids: + loaded = list( + ( + await session.execute(select(Task).where(Task.workplan_id.in_(existing_ids))) + ).scalars() + ) + for task_row in loaded: + tasks_by_wp.setdefault(task_row.workplan_id, []).append(task_row) + + # Bound histories are receiving support for already existing records only. + # Verify every parent and task before any mutation; preserve labels, status, + # retirement stamps and all UUIDs. Missing rows cannot authorize recreation. + for key, workplan in want.items(): + if not workplan.historical_preservation: + continue + row = matched.get(key) + local = {str(task.id): task for task in tasks_by_wp.get(row.id, [])} if row else {} + missing = [task.uuid for task in workplan.tasks if task.uuid not in local] + nonterminal = [task.uuid for task in workplan.tasks + if task.uuid in local and local[task.uuid].status not in {"done", "cancel"}] + if (row is None or str(row.id) != workplan.uuid or missing or nonterminal + or row.status not in {"done", "finished", "archived", "completed"}): + outcome.status = "refused" + outcome.refused.append({ + "reason": "historical binding requires existing parent and task UUIDs", + "workplan": workplan.record_id, "uuid": workplan.uuid, + "missing_task_uuids": missing, "nonterminal_task_uuids": nonterminal, + }) + return outcome + + # Canonical task identity is scoped to the matched current workplan while + # reconciling. Duplicate rows attached to a retired/displaced workplan are + # historical evidence and are deliberately not loaded above. Two rows with + # one identity inside the matched workplan, however, are genuinely + # ambiguous: the reset cannot know which row owns progress or dependencies. + current_task_ambiguities: list[dict[str, Any]] = [] + for key, workplan in want.items(): + row = matched.get(key) + if row is None: + continue + desired_record_ids = { + task.record_id.strip().lower() for task in workplan.tasks + } + groups: dict[str, list[Any]] = {} + for task_row in tasks_by_wp.get(row.id, []): + record_id = (task_row.record_id or "").strip().lower() + if record_id and record_id in desired_record_ids: + groups.setdefault(record_id, []).append(task_row) + for task_rows in groups.values(): + if len(task_rows) > 1: + current_task_ambiguities.append( + { + "reason": "current workplan contains duplicate canonical task identity", + "kind": "task", + "record_id": task_rows[0].record_id, + "workplan": workplan.record_id, + "uuids": [str(task_row.id) for task_row in task_rows], + } + ) + if current_task_ambiguities: + outcome.status = "refused" + outcome.refused.extend(current_task_ambiguities) + outcome.notes.append( + "Retired workplan copies do not block reset, but duplicate identities " + "inside the matched current workplan require manual disposition." + ) + return outcome + + # Check every task the reset would create in one query. A derived UUID held + # by another workplan is a global identity collision. A repeated record_id + # on a different workplan is not sufficient by itself: terminal rows from + # older, wrong projections retain that label as history and tasks have no + # projection-retirement marker. Letting the INSERT discover a UUID collision + # would turn a deterministic refusal into an IntegrityError. + creating_tasks: dict[str, tuple[DerivedWorkplan, DerivedTask]] = {} + for key, workplan in want.items(): + row = matched.get(key) + local_rows = tasks_by_wp.get(row.id, []) if row is not None else [] + local_record_ids = { + (task_row.record_id or "").strip().lower() + for task_row in local_rows + if (task_row.record_id or "").strip() + } + local_uuids = {str(task_row.id) for task_row in local_rows} + for task in workplan.tasks: + if ( + task.record_id.strip().lower() not in local_record_ids + and task.uuid not in local_uuids + ): + creating_tasks[task.uuid] = (workplan, task) + + if creating_tasks: + task_holders = list( + ( + await session.execute( + select(Task) + .where( + Task.id.in_( + [uuid.UUID(task_id) for task_id in creating_tasks] + ), + ) + ) + ).scalars() + ) + if task_holders: + outcome.status = "refused" + for holder in task_holders: + workplan, task = creating_tasks[str(holder.id)] + outcome.refused.append( + { + "reason": "derived task identifier already belongs to another workplan", + "kind": "task", + "record_id": task.record_id, + "uuid": task.uuid, + "workplan": workplan.record_id, + "held_by_workplan_id": str(holder.workplan_id), + } + ) + outcome.notes.append( + "Task identity collisions require an identity decision; the reset " + "will not overwrite or move the existing row." + ) + return outcome + + received_rows: dict[str, Any] = {} + for key, w in want.items(): + row = matched.get(key) + if row is not None: + received_rows[key] = row + if row is None: + row = Workplan( + id=uuid.UUID(w.uuid), + repo_id=repo.id, + topic_id=repo.topic_id, + slug=w.record_id.lower(), + title=w.title or w.record_id, + description=w.description, + status=w.status or "proposed", + owner=w.owner, + flavor=w.flavor, + flavor_promotion_reason=w.flavor_promotion_reason, + flavor_promoted_from=w.flavor_promoted_from, + backing_filename=w.relative_path.rsplit("/", 1)[-1], + backing_relative_path=w.relative_path, + backing_archived=w.archived, + derived_from_commit=derived.commit, + ) + session.add(row) + await session.flush() + received_rows[key] = row + for t in w.tasks: + # Safe only because nothing exists to mis-match against: this + # workplan is new to the hub. + session.add( + Task( + id=uuid.UUID(t.uuid), + workplan_id=row.id, + record_id=t.record_id, + title=t.title or t.record_id, + description=t.description, + status=t.status or "todo", + priority=t.priority or "medium", + needs_human=t.needs_human, + intervention_note=t.intervention_note, + blocking_reason=t.blocking_reason, + flavor=t.flavor, + decision_id=t.decision_id, + blocked_on=t.blocked_on, + wait_until=t.wait_until, + ) + ) + outcome.created.append(w.record_id) + continue + + if w.historical_preservation: + outcome.notes.append("Preserved bound historical row and task UUIDs: " + w.record_id) + continue + + changed = False + # The title is derived like every other field, and not syncing it left + # `cust-wp-0010` reading "Domain and Repository Goals" while its file + # said "Workplan Lifecycle Documentation" — a record correctly + # identified and correctly backed, describing the wrong work. + # + # An empty derived title is not an answer: three activity-core files + # parse to no title at all, and blanking a real one is worse than + # leaving it stale. + if w.title and w.title.strip() and row.title != w.title.strip(): + row.title = w.title.strip() + changed = True + if w.description and getattr(row, "description", None) != w.description: + row.description = w.description + changed = True + if getattr(row, "owner", None) != w.owner: + row.owner = w.owner + changed = True + if w.status and row.status != w.status: + row.status = w.status + changed = True + for field_name in ("flavor", "flavor_promotion_reason", "flavor_promoted_from"): + expected = getattr(w, field_name) + if getattr(row, field_name, None) != expected: + setattr(row, field_name, expected) + changed = True + if row.backing_relative_path != w.relative_path: + row.backing_relative_path = w.relative_path + row.backing_filename = w.relative_path.rsplit("/", 1)[-1] + row.backing_archived = w.archived + changed = True + if row.projection_retired_at is not None: + # It derives again; un-retire rather than leaving a contradiction. + row.projection_retired_at = None + row.projection_retired_reason = None + changed = True + if row.derived_from_commit != derived.commit: + row.derived_from_commit = derived.commit + changed = True + if changed: + outcome.updated.append(w.record_id) + _sync_existing_workplan_tasks( + session, row, w, tasks_by_wp.get(row.id, []), outcome + ) + + for r in stale: + outcome.retired.append(r.slug or str(r.id)) + r.projection_retired_at = now + r.projection_retired_reason = RETIRE_REASON + r.slug = _tombstone_slug(r.slug or str(r.id), now) + + await _reconcile_source_edges(session, repo, repo_slug, derived, want, received_rows, outcome) + + if ( + outcome.created + or outcome.updated + or outcome.retired + or outcome.released + or outcome.created_tasks + or outcome.updated_tasks + or outcome.cancelled_tasks + or outcome.created_edges + or outcome.removed_edges + ): + outcome.status = "applied" + return outcome + + +# --------------------------------------------------------------------------- +# Source-owned dependency edges (STATE-WP-0094-T05) +# docs/forge-dependency-edge-receiving.md +# --------------------------------------------------------------------------- + +EDGE_SOURCE_PREFIX = "forge:" +_CLOSED_TASK_STATUSES = {"done", "cancel"} + + +def _edge_workplan_closed(status: str | None) -> bool: + from api.workplan_status import CLOSED_WORKPLAN_STATUSES, normalize_workplan_status + + try: + return normalize_workplan_status(status or "") in CLOSED_WORKPLAN_STATUSES + except ValueError: + return False + + +def _edge_task_closed(status: str | None) -> bool: + from api.task_status import normalize_task_status + + try: + return normalize_task_status(status or "todo", default="todo") in _CLOSED_TASK_STATUSES + except ValueError: + return False + + +def edge_source_owner(repo_slug: str) -> str: + return f"{EDGE_SOURCE_PREFIX}{repo_slug}" + + +def _iter_edge_declarations(want: dict[str, DerivedWorkplan]): + """Yield ``(key, workplan, task_or_None, target)`` for every desired declaration. + + Shared by receiving and the dry-run preview so both apply the same rules: + historical workplans declare nothing; task-block edges of closed workplans + and closed tasks are history (C-20 parity). + """ + for key, w in want.items(): + if w.historical_preservation: + continue + for target in w.depends_on_workplans + w.depends_on_tasks: + yield key, w, None, target + if _edge_workplan_closed(w.status): + continue + for t in w.tasks: + if not t.depends_on or _edge_task_closed(t.status): + continue + for target in t.depends_on: + yield key, w, t, target + + +def source_edge_targets(want: dict[str, DerivedWorkplan]) -> set[str]: + """Every target record id a receipt of ``want`` would resolve.""" + return {target for _, _, _, target in _iter_edge_declarations(want)} + + +def plan_source_edges( + want: dict[str, DerivedWorkplan], + from_workplan_ids: dict[str, Any], + task_ids: dict[Any, dict[str, Any]], + targets: dict[str, tuple[str, Any]], +) -> tuple[dict[tuple, dict[str, Any]], list[dict[str, Any]], set[tuple]]: + """Pure: desired edge keys, unresolved declarations and held from-sides. + + ``from_workplan_ids`` maps a ``want`` key to the from workplan's id (keys + absent are skipped); ``task_ids`` maps that id to ``{record_id.lower(): task + id}``; ``targets`` maps a target record id to ``("ok", id)`` or + ``("missing"|"ambiguous", detail)`` (absent means missing). + Edge key: ``(from_workplan_id, from_task_id, to_workplan_id, to_task_id, rel)``. + """ + desired: dict[tuple, dict[str, Any]] = {} + unresolved: list[dict[str, Any]] = [] + # From-sides with an unresolved declaration keep every owned edge: the + # unresolved target may be one of them (e.g. newly ambiguous). + held_sides: set[tuple] = set() + for key, w, t, target in _iter_edge_declarations(want): + wp_id = from_workplan_ids.get(key) + if wp_id is None: + continue + from_task_id = None + if t is not None: + from_task_id = task_ids.get(wp_id, {}).get(t.record_id.strip().lower()) + if from_task_id is None: + held_sides.add((wp_id, "*")) + unresolved.append({"workplan": w.record_id, "from": t.record_id, + "target": target, "reason": "from-task-missing"}) + continue + side = (wp_id, from_task_id) + is_task = bool(_TASK_TARGET_RE.search(target)) + state, value = targets.get(target, ("missing", None)) + base = {"workplan": w.record_id, "from": t.record_id if t is not None else w.record_id, + "target": target} + if state == "ok" and ((not is_task and value == wp_id) or (is_task and value == from_task_id)): + state, value = "self", None + if state != "ok": + held_sides.add(side) + unresolved.append({**base, "reason": state, **({"candidates": value} if value else {})}) + continue + rel = "starts_after" if is_task else "blocks" + edge_key = (wp_id, from_task_id, None if is_task else value, value if is_task else None, rel) + desired.setdefault(edge_key, {**base, "relationship_type": rel}) + return desired, unresolved, held_sides + + +def _edge_dict(edge_key: tuple, info: dict[str, Any] | None = None) -> dict[str, Any]: + from_wp, from_task, to_wp, to_task, rel = edge_key + return { + **(info or {}), + "from_workplan_id": str(from_wp), + "from_task_id": str(from_task) if from_task else None, + "to_workplan_id": str(to_wp) if to_wp else None, + "to_task_id": str(to_task) if to_task else None, + "relationship_type": rel, + } + + +def classify_source_edges( + desired: dict[tuple, dict[str, Any]], + held_sides: set[tuple], + existing: list[tuple[tuple, Any, str | None]], + owner: str, +) -> dict[str, list]: + """Pure: split desired/existing edges into create/external/remove/retain. + + ``existing`` lists ``(edge key, row id, source_owner)`` for every row whose + from-workplan belongs to the repository. Several rows may share a key (an + owned row and an API-authored duplicate); they are never collapsed, so an + external duplicate can neither hide nor protect an owned row. + """ + owned_keys = {k for k, _, o in existing if o == owner} + any_keys = {k for k, _, _ in existing} + plan: dict[str, list] = {"create": [], "external": [], "remove": [], "retain": []} + for edge_key in sorted(desired, key=str): + if edge_key in owned_keys: + continue + bucket = "external" if edge_key in any_keys else "create" + plan[bucket].append((edge_key, desired[edge_key])) + for edge_key, row_id, src_owner in sorted(existing, key=lambda r: (str(r[0]), str(r[1]))): + if src_owner != owner or edge_key in desired: + continue + if (edge_key[0], edge_key[1]) in held_sides or (edge_key[0], "*") in held_sides: + plan["retain"].append((edge_key, row_id)) + else: + plan["remove"].append((edge_key, row_id)) + return plan + + +async def _resolve_edge_target(session: Any, record_id: str, cache: dict) -> tuple[str, Any]: + """Resolve a declared target fleet-wide to ``("ok", uuid)`` or a reason. + + Workplans match by ADR-007 derived UUID or record-id slug; tasks by derived + UUID or ``record_id``. Projection-retired workplans (and their tasks) are + not targets. More than one candidate is ambiguous: no winner is chosen. + """ + if record_id in cache: + return cache[record_id] + from sqlalchemy import func, or_, select + + from api.models.task import Task + from api.models.workplan import Workplan + + derived = uuid.UUID(derived_record_uuid(record_id)) + if _TASK_TARGET_RE.search(record_id): + stmt = ( + select(Task.id) + .join(Workplan, Workplan.id == Task.workplan_id) + .where(or_(Task.id == derived, Task.record_id == record_id)) + .where(Workplan.projection_retired_at.is_(None)) + ) + else: + stmt = select(Workplan.id).where( + or_(Workplan.id == derived, func.lower(Workplan.slug) == record_id.lower()) + ).where(Workplan.projection_retired_at.is_(None)) + ids = sorted({row[0] for row in (await session.execute(stmt)).all()}, key=str) + if not ids: + result: tuple[str, Any] = ("missing", None) + elif len(ids) > 1: + result = ("ambiguous", [str(i) for i in ids]) + else: + result = ("ok", ids[0]) + cache[record_id] = result + return result + + +async def _reconcile_source_edges( + session: Any, + repo: Any, + repo_slug: str, + derived: DerivedProjection, + want: dict[str, DerivedWorkplan], + received_rows: dict[str, Any], + outcome: ResetOutcome, +) -> None: + """Reconcile ``workplan_dependencies`` rows owned by this repository's source. + + Ownership: a row is source-owned iff ``source_owner == forge:`` + and its from-side belongs to this repository. Only those rows are created or + removed here. Rows with any other (or no) owner are externally authored and + preserved, even when they duplicate a declared edge. + """ + from sqlalchemy import select + + from api.models.task import Task + from api.models.workplan import Workplan + from api.models.workplan_dependency import WorkplanDependency + + owner = edge_source_owner(repo_slug) + await session.flush() + + from_ids = {key: row.id for key, row in received_rows.items() if key in want} + targets: dict[str, tuple[str, Any]] = {} + for target in sorted(source_edge_targets(want)): + targets[target] = await _resolve_edge_target(session, target, targets) + task_ids: dict[Any, dict[str, Any]] = {} + for key, w, t, _ in _iter_edge_declarations(want): + wp_id = from_ids.get(key) + if t is None or wp_id is None or wp_id in task_ids: + continue + task_ids[wp_id] = { + (tr.record_id or "").strip().lower(): tr.id + for tr in (await session.execute(select(Task).where(Task.workplan_id == wp_id))).scalars() + } + desired, unresolved, held_sides = plan_source_edges(want, from_ids, task_ids, targets) + outcome.unresolved_edges.extend(unresolved) + + repo_workplan_ids = list( + (await session.execute(select(Workplan.id).where(Workplan.repo_id == repo.id))).scalars() + ) + existing_rows = list( + ( + await session.execute( + select(WorkplanDependency).where( + WorkplanDependency.from_workplan_id.in_(repo_workplan_ids) + ) + ) + ).scalars() + ) if repo_workplan_ids else [] + rows_by_id = {e.id: e for e in existing_rows} + plan = classify_source_edges( + desired, + held_sides, + [((e.from_workplan_id, e.from_task_id, e.to_workplan_id, e.to_task_id, e.relationship_type), + e.id, e.source_owner) for e in existing_rows], + owner, + ) + for edge_key, info in plan["external"]: + outcome.external_edges.append(_edge_dict(edge_key, info)) + for edge_key, info in plan["create"]: + from_wp, from_task, to_wp, to_task, rel = edge_key + session.add( + WorkplanDependency( + from_workplan_id=from_wp, + from_task_id=from_task, + to_workplan_id=to_wp, + to_task_id=to_task, + relationship_type=rel, + description=f"{info['from']} depends_on {info['target']}", + source_owner=owner, + source_commit=derived.commit, + ) + ) + outcome.created_edges.append(_edge_dict(edge_key, info)) + for edge_key, row_id in plan["retain"]: + outcome.notes.append( + f"Retained source-owned edge {row_id}: its from-side has an unresolved declaration." + ) + for edge_key, row_id in plan["remove"]: + await session.delete(rows_by_id[row_id]) + outcome.removed_edges.append(_edge_dict(edge_key, {"id": str(row_id)})) + # Integrity failures surface here, inside the caller's transaction. + await session.flush() + + +# --------------------------------------------------------------------------- +# Fleet form (STATE-WP-0083-T04) +# --------------------------------------------------------------------------- + + +@dataclass +class FleetResetOutcome: + results: dict[str, dict[str, Any]] = field(default_factory=dict) + errors: dict[str, str] = field(default_factory=dict) + # Kept apart from `errors` on purpose. Nine repositories sitting in an error + # bucket read as nine broken repositories; they were nine we were not + # allowed to read, which is a different thing to go and fix + # (`STATE-WP-0083-T04`, 2026-08-26). + unreadable: dict[str, str] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + by_status: dict[str, int] = {} + for r in self.results.values(): + by_status[r["status"]] = by_status.get(r["status"], 0) + 1 + return { + "schema": "state-hub.fleet-projection-reset.v1", + "repositories": len(self.results) + len(self.errors) + len(self.unreadable), + "by_status": by_status, + "errored": len(self.errors), + "unreadable_count": len(self.unreadable), + "unreadable": self.unreadable, + "totals": { + k: sum(r["counts"][k] for r in self.results.values()) + for k in ("created", "updated", "retired", "released", "refused") + }, + "results": self.results, + "errors": self.errors, + } + + +async def reset_fleet_projection( + session_factory: Any, + repo_slugs: list[str], + *, + acknowledge_retirements: bool = False, + forge_base: str = DEFAULT_FORGE_BASE, +) -> FleetResetOutcome: + """Reset every repository, one at a time, sharing the per-repository path. + + The fleet form is a loop over the repository form and nothing else + (`ADR-012` decision 7). The rarely-run wide operation must be the frequently + run narrow one, or the wide one is trusted on the strength of never having + been exercised. + + A repository that refuses or errors is recorded and the pass continues. + Aborting on the first refusal would mean one unresolved repository blocks + reconstruction everywhere — which in practice means permanently. + + Each repository gets its own session, so one failure cannot roll back + another's work or leave a poisoned transaction behind. + """ + outcome = FleetResetOutcome() + for slug in repo_slugs: + try: + async with session_factory() as session: + result = await reset_repository_projection( + session, + slug, + acknowledge_retirements=acknowledge_retirements, + forge_base=forge_base, + ) + # A released identifier is a repair that stands on its own, so + # it must survive a refusal in the same repository — otherwise + # moving the release ahead of the refusal returns achieves + # nothing and the rollback quietly undoes it. + if result.status == "applied" or result.released: + await session.commit() + else: + await session.rollback() + if result.status == "unreadable": + detail = next( + (r.get("detail", "") for r in result.refused), "" + ) + outcome.unreadable[slug] = detail or "could not be read from the forge" + else: + outcome.results[slug] = result.to_dict() + except ForgeUnreadableError as exc: + # Reachable when the caller supplied its own derivation path. + outcome.unreadable[slug] = str(exc)[:300] + except Exception as exc: # noqa: BLE001 - one repo must not end the pass + outcome.errors[slug] = f"{type(exc).__name__}: {exc}"[:300] + return outcome diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/forge_repository.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/forge_repository.py new file mode 100644 index 00000000..25cb788f --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/forge_repository.py @@ -0,0 +1,247 @@ +"""Narrow Forgejo repository boundary used by repository rename operations. + +The lifecycle service depends on this protocol instead of calling Forgejo +directly. Tests replace it with an in-memory gateway, which makes interruption +and resume tests deterministic and guarantees that API tests cannot rename a +live repository. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Protocol +from urllib.parse import quote, urljoin + +import httpx + +from api.config import settings +from api.services.forge_credential import forge_read_token + + +WRITE_TOKEN_ENV = "FORGE_RENAME_TOKEN" +WRITE_TOKEN_FILE_ENV = "FORGE_RENAME_TOKEN_FILE" + + +class ForgeRepositoryError(RuntimeError): + pass + + +class ForgeRepositoryUnreadable(ForgeRepositoryError): + pass + + +class ForgeRepositoryRedirected(ForgeRepositoryUnreadable): + """Requested coordinate redirects; no credentials are forwarded.""" + + def __init__(self, source_url: str, location: str): + super().__init__("Forge repository coordinate redirects") + self.target_url = urljoin(source_url, location) if location else None + + +class ForgeRepositoryConflict(ForgeRepositoryError): + pass + + +@dataclass(frozen=True) +class ForgeRepositorySnapshot: + repository_id: int + owner: str + name: str + full_name: str + default_branch: str + head_commit: str + clone_url: str | None + html_url: str | None + projection_readable: bool + projection_source_present: bool + projection_entry_count: int | None + + def as_dict(self) -> dict: + return { + "repository_id": self.repository_id, + "owner": self.owner, + "name": self.name, + "full_name": self.full_name, + "default_branch": self.default_branch, + "head_commit": self.head_commit, + "clone_url": self.clone_url, + "html_url": self.html_url, + "projection_readable": self.projection_readable, + "projection_source_present": self.projection_source_present, + "projection_entry_count": self.projection_entry_count, + } + + +class ForgeRepositoryGateway(Protocol): + async def inspect( + self, *, instance: str, owner: str, name: str + ) -> ForgeRepositorySnapshot | None: ... + + async def rename( + self, *, instance: str, owner: str, old_name: str, new_name: str + ) -> ForgeRepositorySnapshot: ... + + async def descends_from( + self, *, instance: str, owner: str, name: str, base: str, head: str + ) -> bool: ... + + +def _write_token() -> str | None: + path = os.environ.get(WRITE_TOKEN_FILE_ENV) + if path: + try: + return Path(path).read_text(encoding="utf-8").strip() or None + except OSError: + return None + return (os.environ.get(WRITE_TOKEN_ENV) or "").strip() or None + + +class ForgejoRepositoryGateway: + """Forgejo v1 API adapter with separate read and rename credentials.""" + + @staticmethod + def _headers(token: str | None) -> dict[str, str]: + return {"Authorization": f"token {token}"} if token else {} + + @staticmethod + def _api(instance: str, owner: str, name: str) -> str: + base = instance.rstrip("/") + return f"{base}/api/v1/repos/{quote(owner, safe='')}/{quote(name, safe='')}" + + async def _inspect_with_token( + self, *, instance: str, owner: str, name: str, token: str | None + ) -> ForgeRepositorySnapshot | None: + url = self._api(instance, owner, name) + timeout = settings.repository_rename_forge_timeout_seconds + try: + async with httpx.AsyncClient(timeout=timeout) as client: + response = await client.get(url, headers=self._headers(token)) + if response.status_code in {301, 302, 303, 307, 308}: + raise ForgeRepositoryRedirected( + url, response.headers.get("location", "") + ) + if response.status_code == 404: + if token is None: + raise ForgeRepositoryUnreadable( + "Forge returned not-found without an authenticated absence proof" + ) + return None + response.raise_for_status() + repo = response.json() + branch_name = str(repo.get("default_branch") or "") + if not branch_name: + raise ForgeRepositoryUnreadable("Forge repository has no default branch") + branch = await client.get( + f"{url}/branches/{quote(branch_name, safe='')}", + headers=self._headers(token), + ) + branch.raise_for_status() + branch_data = branch.json() + commit = branch_data.get("commit") or {} + head = str(commit.get("id") or commit.get("sha") or "") + if not head: + raise ForgeRepositoryUnreadable("Forge default branch has no readable head") + projection = await client.get( + f"{url}/contents/workplans", + params={"ref": head}, + headers=self._headers(token), + ) + source_present = projection.status_code == 200 + if projection.status_code not in {200, 404}: + projection.raise_for_status() + entries = projection.json() if source_present else None + except ForgeRepositoryError: + raise + except (httpx.HTTPError, ValueError, KeyError) as exc: + raise ForgeRepositoryUnreadable( + f"Forge repository inspection failed ({type(exc).__name__})" + ) from exc + return ForgeRepositorySnapshot( + repository_id=int(repo["id"]), + owner=str((repo.get("owner") or {}).get("login") or owner), + name=str(repo["name"]), + full_name=str(repo.get("full_name") or f"{owner}/{name}"), + default_branch=branch_name, + head_commit=head, + clone_url=repo.get("clone_url"), + html_url=repo.get("html_url"), + projection_readable=True, + projection_source_present=source_present, + projection_entry_count=(len(entries) if isinstance(entries, list) else None), + ) + + async def inspect( + self, *, instance: str, owner: str, name: str + ) -> ForgeRepositorySnapshot | None: + return await self._inspect_with_token( + instance=instance, owner=owner, name=name, token=forge_read_token() + ) + + async def rename( + self, *, instance: str, owner: str, old_name: str, new_name: str + ) -> ForgeRepositorySnapshot: + token = _write_token() + if not token: + raise ForgeRepositoryUnreadable("Forge rename credential is unavailable") + url = self._api(instance, owner, old_name) + try: + async with httpx.AsyncClient( + timeout=settings.repository_rename_forge_timeout_seconds + ) as client: + response = await client.patch( + url, + json={"name": new_name}, + headers=self._headers(token), + ) + if response.status_code in {409, 422}: + raise ForgeRepositoryConflict("Forge rejected the target repository name") + response.raise_for_status() + except ForgeRepositoryError: + raise + except httpx.HTTPError as exc: + raise ForgeRepositoryUnreadable( + f"Forge repository rename failed ({type(exc).__name__})" + ) from exc + renamed = await self._inspect_with_token( + instance=instance, owner=owner, name=new_name, token=token + ) + if renamed is None: + raise ForgeRepositoryUnreadable("Renamed Forge repository is not readable") + return renamed + + async def descends_from( + self, *, instance: str, owner: str, name: str, base: str, head: str + ) -> bool: + """True when ``base`` is an ancestor of (or equal to) ``head``. + + ``compare/...`` lists commits reachable from ``base`` but not + from ``head``; it is empty exactly when ``base`` is contained in ``head``. + """ + if base == head: + return True + url = f"{self._api(instance, owner, name)}/compare/{quote(head, safe='')}...{quote(base, safe='')}" + try: + async with httpx.AsyncClient( + timeout=settings.repository_rename_forge_timeout_seconds + ) as client: + response = await client.get(url, headers=self._headers(forge_read_token())) + if response.status_code == 404: + return False + response.raise_for_status() + data = response.json() + except (httpx.HTTPError, ValueError) as exc: + raise ForgeRepositoryUnreadable( + f"Forge ancestry check failed ({type(exc).__name__})" + ) from exc + commits = data.get("commits") + total = data.get("total_commits") + return (isinstance(commits, list) and not commits) or total == 0 + + +_gateway = ForgejoRepositoryGateway() + + +def get_forge_repository_gateway() -> ForgeRepositoryGateway: + return _gateway diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/historical_projection.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/historical_projection.py new file mode 100644 index 00000000..1a8a79ca --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/historical_projection.py @@ -0,0 +1,195 @@ +"""Exact closed source bindings predating canonical Forge identity. + +These are frozen historical source views, not aliases or UUID allocation rules. +An altered/missing source requires explicit owner review; generic source UUIDs +remain untrusted. Shared Activity views represent the one archived parent. +""" +from pathlib import Path +from hashlib import sha256 +import json + +BINDINGS = json.loads(r'''[ + { + "repo": "activity-core", + "path": "workplans/archived/260707-custodian-WP-0003-event-bridge.md", + "sha256": "83ea8bcb151e003419b73c59dbd028f4a1ce3d1120a82e4a128a772503ff3b76", + "record_id": "custodian-WP-0003", + "uuid": "b4eb45a9-69e3-4ab0-b00c-67a53c3117c5", + "parent_record_id": "custodian-WP-0003" + }, + { + "repo": "activity-core", + "path": "workplans/custodian-WP-0001-temporal-backbone.md", + "sha256": "020e2eac973f2daa41c1687f8e867da9d521f945547e6aa82c87993b3ff6fc64", + "record_id": "custodian-WP-0001", + "uuid": "e22516cc-4fdf-4dad-a928-a9e04a9fa7c7", + "parent_record_id": "custodian-WP-0001" + }, + { + "repo": "activity-core", + "path": "workplans/custodian-WP-0002-triggers-ops.md", + "sha256": "d8cbc61631c01b6f11a03a63f2fb6275fc853ff7f7459ce65973ed1a233daa01", + "record_id": "custodian-WP-0002", + "uuid": "3a4f47d9-8bc1-434e-acb4-bed5d4dacda0", + "parent_record_id": "custodian-WP-0002" + }, + { + "repo": "activity-core", + "path": "workplans/custodian-WP-0003a-event-bridge-model-rules-registry.md", + "sha256": "ffa6682e9ba3aaf559bd44203d561c90dea69bcf1b1e778209a7b209e6e5cdd3", + "record_id": "custodian-WP-0003a", + "uuid": "b4eb45a9-69e3-4ab0-b00c-67a53c3117c5", + "parent_record_id": "custodian-WP-0003" + }, + { + "repo": "activity-core", + "path": "workplans/custodian-WP-0003b-event-bridge-parser-workflow-triggers.md", + "sha256": "4fc34e0f821e2df51e5c08c7ab34da35b65690381d01f0cfdf55d00c970b414c", + "record_id": "custodian-WP-0003b", + "uuid": "b4eb45a9-69e3-4ab0-b00c-67a53c3117c5", + "parent_record_id": "custodian-WP-0003" + }, + { + "repo": "activity-core", + "path": "workplans/custodian-WP-0003c-event-bridge-context-integration.md", + "sha256": "b6cd62b26515e509670ca39736b6bcec9633beedd46c9bd8f7598c06bac7276b", + "record_id": "custodian-WP-0003c", + "uuid": "b4eb45a9-69e3-4ab0-b00c-67a53c3117c5", + "parent_record_id": "custodian-WP-0003" + }, + { + "repo": "activity-core", + "path": "workplans/custodian-WP-0004-railiance-ops.md", + "sha256": "20df02eea740e109c937130e7f8801a636fc0990ed8c6d2ced55659f8957adb1", + "record_id": "custodian-WP-0004", + "uuid": "759b1255-aa78-42b5-8fab-c5fcf168f5f4", + "parent_record_id": "custodian-WP-0004" + }, + { + "repo": "the-custodian", + "path": "workplans/ADHOC-2026-08-25-retired.md", + "sha256": "344a1293dbb3a67a944eb34ca4a0337487d8f047226027a2107922346fbab2dd", + "record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED", + "uuid": "1c53d3db-6633-5e2d-987f-10706d73be9e", + "parent_record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED" + }, + { + "repo": "railiance-cluster", + "path": "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md", + "sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9", + "record_id": "RAIL-BS-WP-0002", + "uuid": "4c63dfc6-9eac-4e79-9f77-8f644ad7147d", + "parent_record_id": "RAIL-BS-WP-0002", + "tasks": [ + { + "label": "T01", + "uuid": "3f042630-eab0-4c6a-9167-e2b28ff20e40", + "status": "done" + }, + { + "label": "T02", + "uuid": "e8510646-46ed-4697-a345-f3d3009eea78", + "status": "done" + }, + { + "label": "T03", + "uuid": "dab2c07f-8aa0-4635-8df6-857e87e93fc5", + "status": "done" + }, + { + "label": "T04", + "uuid": "5c3d40e4-239b-488e-9519-6f7a38d2325f", + "status": "done" + }, + { + "label": "T05", + "uuid": "7f9e0e58-a130-467a-a2d0-b3f2564e496f", + "status": "done" + } + ] + }, + { + "repo": "railiance-cluster", + "path": "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md", + "sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a", + "record_id": "RAIL-BS-WP-0003", + "uuid": "7ee9ee22-1fae-4567-9194-8d70a9e0f45b", + "parent_record_id": "RAIL-BS-WP-0003", + "tasks": [ + { + "label": "T01", + "uuid": "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc", + "status": "done" + }, + { + "label": "T02", + "uuid": "ab166073-30a7-4702-a037-4091e8706e20", + "status": "done" + }, + { + "label": "T03", + "uuid": "140da396-8e30-4f4d-b88c-c42c0cd46c01", + "status": "done" + }, + { + "label": "T04", + "uuid": "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d", + "status": "done" + } + ] + }, + { + "repo": "railiance-cluster", + "path": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md", + "sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3", + "record_id": "RAIL-BS-WP-0004", + "uuid": "7e8b0c20-51eb-40c9-9e3b-85dd380d7625", + "parent_record_id": "RAIL-BS-WP-0004", + "tasks": [ + { + "label": "T01", + "uuid": "4526a842-ea31-4874-9231-92ab556cfe7b", + "status": "done" + }, + { + "label": "T02", + "uuid": "a6313e06-1976-46a7-8e31-df4eb2eca880", + "status": "done" + }, + { + "label": "T03", + "uuid": "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3", + "status": "done" + }, + { + "label": "T04", + "uuid": "08233868-d522-4117-bc4e-6c0f52545665", + "status": "done" + }, + { + "label": "T05", + "uuid": "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8", + "status": "done" + }, + { + "label": "T06", + "uuid": "f8e4a094-c367-40eb-b895-da17bc144b07", + "status": "done" + } + ] + } +]''') + + +def bound_sources(repo_root: Path, repo_slug: str): + entries = [entry for entry in BINDINGS if entry["repo"] == repo_slug] + result = [] + for entry in entries: + path = repo_root / entry["path"] + if path.is_symlink() or not path.is_file(): + raise ValueError("Historical source binding missing or symlinked: " + entry["path"]) + data = path.read_bytes() + if sha256(data).hexdigest() != entry["sha256"]: + raise ValueError("Historical source binding changed: " + entry["path"]) + result.append((entry, data.decode("utf-8"))) + return result diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/informed_decision_contract.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/informed_decision_contract.py new file mode 100644 index 00000000..e993ba2f --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/informed_decision_contract.py @@ -0,0 +1,388 @@ +"""Pure contract for informed-decision forwarding and notifications. + +Dev and request validation only. This module does not read a token, open a +socket, register a submitter, or decide a hub decision. ``drafted_by`` is +asserted provenance; it is never an authenticated identity. +""" +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from typing import Any, Mapping + +KIND = "informed-decision.memo-intake" +VERSION = 1 +OVERVIEW_LINE = "The signed-in overview is the authoritative review entry point." +PROVENANCE_NOTE = ( + "drafted_by is asserted provenance, not authenticated identity. " + "A forwarded token proves only the hub submitter." +) +RESOLVING_VERBS = frozenset({"accept", "decline"}) +NOTIFICATION_FIELDS = ("memo_id", "question", "memo_class", "submitter") +MAX_QUESTION = 400 +MAX_DRAFTED_BY = 200 +MAX_PACKET_BYTES = 256 * 1024 +MAX_ENVELOPE_BYTES = 1024 * 1024 +MAX_PACKET_ITEMS = 32 +MAX_NOTIFICATION_BYTES = 8192 + +_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{8,128}$") +_MEMO_ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]{2,62}$") +_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{1,62}$") +_UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") +_SHA_RE = re.compile(r"^sha256:[0-9a-f]{64}$") +_ERROR_RE = re.compile(r"^[a-z0-9_]{1,64}$") + +ENVELOPE_KEYS = { + "kind", "version", "idempotency_key", "memo_class", "review_group", + "drafted_by", "serves", "approval_intent", "memo", "packet", +} +ENVELOPE_REQUIRED = ENVELOPE_KEYS - {"drafted_by", "serves"} +INTENT_KEYS = {"action", "principal", "purpose", "target"} +TARGET_KEYS = {"id", "type", "system"} +PACKET_KEYS = {"item_id", "media_type", "content"} +MEDIA_TYPES = {"text/markdown", "text/plain", "application/json"} + +# Shapes only. A hit is refused by code; the matched text is never echoed. +_SECRET_MARKERS = ( + re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"), + re.compile(r"\bBearer\s+[A-Za-z0-9._~+/=-]{16,}"), + re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}"), + re.compile(r"\b(hvs|hvb|hvr)\.[A-Za-z0-9_-]{20,}"), + re.compile( + r"\b(sk-(or-)?[A-Za-z0-9_-]{20,}|ghp_[A-Za-z0-9]{30,}|" + r"github_pat_[A-Za-z0-9_]{30,}|glpat-[A-Za-z0-9_-]{20,}|" + r"AKIA[0-9A-Z]{16}|xox[abpr]-[A-Za-z0-9-]{10,})" + ), + re.compile( + r"(?i)\b(password|passwd|client_secret|secret|api[_-]?key|access[_-]?token|token)" + r"\s*[:=]\s*['\"]?(?=[A-Za-z0-9_+.=-]*\d)[A-Za-z0-9_+.=-]{12,}(?=['\"\s,;)]|$)" + ), + re.compile(r"://[^/@\s:]+:[^/@\s]+@"), +) + + +class ContractError(ValueError): + def __init__(self, code: str, status: int = 400) -> None: + super().__init__(code) + self.code = code + self.status = status + + +def canonical(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def digest(value: Any) -> str: + return hashlib.sha256(canonical(value).encode("utf-8")).hexdigest() + + +def scan_secrets(value: Any) -> None: + for text in _strings(value): + for pattern in _SECRET_MARKERS: + if pattern.search(text): + raise ContractError("secret_marker") + + +def stable_idempotency_key(decision_id: str, memo_id: str) -> str: + _uuid(decision_id, "hub_decision_id") + _memo_id(memo_id) + return f"state-hub.{decision_id}.{memo_id}" + + +def resolution_applies(verb: str, serves: Mapping | None) -> bool: + """Terminal human accept/decline with resolves=on-answer only.""" + if verb not in RESOLVING_VERBS: + return False + if not isinstance(serves, Mapping): + return False + return serves.get("resolves") == "on-answer" + + +def correlate_resolution( + memo_id: str, *, post_status: int | None, existing: Mapping | None +) -> str: + """Classify a resolve attempt. A 409 is a conflict only after the existing + resolution was read; an unreadable 409 stays retryable.""" + _memo_id(memo_id) + if post_status is None: + return "failed" + if 200 <= post_status < 300: + return "resolved" + if post_status != 409: + return "failed" + if not isinstance(existing, Mapping): + return "failed" + rationale = existing.get("rationale") + if not isinstance(rationale, str): + return "failed" + if f"memo {memo_id} " in rationale: + return "already_resolved" + return "conflict" + + +def safe_error_code(value: Any, fallback: str) -> str: + if isinstance(value, str) and _ERROR_RE.match(value): + return value + return fallback + + +@dataclass(frozen=True) +class Notification: + memo_id: str + question: str + memo_class: str + submitter: str + + def as_body(self) -> dict[str, str]: + return { + "memo_id": self.memo_id, + "question": self.question, + "memo_class": self.memo_class, + "submitter": self.submitter, + } + + @property + def body_digest(self) -> str: + return digest(self.as_body()) + + def message_subject(self) -> str: + return f"Decision memo {self.memo_id}" + + def message_text(self) -> str: + return OVERVIEW_LINE + "\n" + canonical(self.as_body()) + + +def parse_delivery_id(value: str | None) -> str: + if value is None or value == "": + raise ContractError("delivery_id_required") + if not isinstance(value, str) or not _KEY_RE.match(value): + raise ContractError("delivery_id_invalid") + return value + + +def parse_notification(payload: Any) -> Notification: + if not isinstance(payload, dict): + raise ContractError("invalid_notification") + if set(payload) - set(NOTIFICATION_FIELDS): + raise ContractError("unknown_field") + if set(NOTIFICATION_FIELDS) - set(payload): + raise ContractError("missing_field") + scan_secrets(payload) + if any("://" in text for text in _strings(payload)): + raise ContractError("notification_has_link") + return Notification( + memo_id=_memo_id(payload["memo_id"]), + question=_one_question(payload["question"]), + memo_class=_slug(payload["memo_class"], "memo_class"), + submitter=_slug(payload["submitter"], "submitter"), + ) + + +def build_envelope( + *, + decision_id: str, + memo: Mapping, + packet: list, + memo_class: str, + review_group: str, + approval_intent: Mapping, + drafted_by: str | None = None, + resolves: str = "on-answer", +) -> dict: + if not isinstance(memo, Mapping) or not isinstance(memo.get("id"), str): + raise ContractError("invalid_memo") + envelope: dict[str, Any] = { + "kind": KIND, + "version": VERSION, + "idempotency_key": stable_idempotency_key(decision_id, memo["id"]), + "memo_class": memo_class, + "review_group": review_group, + "serves": {"hub_decision_id": decision_id, "resolves": resolves}, + "approval_intent": dict(approval_intent), + "memo": dict(memo), + "packet": list(packet), + } + if drafted_by is not None: + envelope["drafted_by"] = drafted_by + return validate_envelope(envelope) + + +def validate_envelope(envelope: Any) -> dict: + if not isinstance(envelope, dict): + raise ContractError("invalid_envelope") + extra = set(envelope) - ENVELOPE_KEYS + if extra: + raise ContractError("unknown_field") + missing = ENVELOPE_REQUIRED - set(envelope) + if missing: + raise ContractError("missing_field") + if envelope["kind"] != KIND or type(envelope["version"]) is not int or envelope["version"] != VERSION: + raise ContractError("unsupported_kind") + scan_secrets(envelope) + _packet(envelope["packet"]) + _bounded_tree(envelope) + encoded = canonical(envelope).encode("utf-8") + if len(encoded) > MAX_ENVELOPE_BYTES: + raise ContractError("envelope_too_large") + + memo = envelope["memo"] + if not isinstance(memo, dict): + raise ContractError("invalid_memo") + memo_id = _memo_id(memo.get("id")) + if "question" in memo: + _one_question(memo["question"]) + key = _text(envelope["idempotency_key"], pattern=_KEY_RE, code="idempotency_key_invalid") + if "serves" in envelope: + decision_id = _serves(envelope["serves"]) + if key != stable_idempotency_key(decision_id, memo_id): + raise ContractError("idempotency_key_not_stable") + elif not (memo_id.startswith("review-") and _UUID_RE.fullmatch(memo_id[7:]) and key == memo_id): + # Lifecycle service supplies its retained snapshot id/key. Never invent + # a Hub decision relationship or rewrite that immutable envelope. + raise ContractError("lifecycle_handoff_key_invalid") + _slug(envelope["memo_class"], "memo_class") + _slug(envelope["review_group"], "review_group") + drafted_by = envelope.get("drafted_by") + if drafted_by is not None: + _text(drafted_by, max_len=MAX_DRAFTED_BY, code="drafted_by_invalid") + _intent(envelope["approval_intent"]) + if "submitter" in envelope or "actor" in envelope.get("approval_intent", {}): + raise ContractError("unknown_field") + return json.loads(canonical(envelope)) + + +def submission_digest(envelope: Mapping) -> str: + body = {key: value for key, value in envelope.items() if key != "idempotency_key"} + return digest(body) + + +def bounded_receipt(body: Any, expected_memo_id: str) -> dict[str, str]: + if not isinstance(body, dict): + raise ContractError("receipt_incomplete") + memo_id = body.get("memo_id") + approval_id = body.get("approval_id") + binding = body.get("approval_binding_digest") + if memo_id != expected_memo_id or not isinstance(approval_id, str) or not _UUID_RE.match(approval_id): + raise ContractError("receipt_mismatch") + if not isinstance(binding, str) or not _SHA_RE.match(binding): + raise ContractError("receipt_incomplete") + return { + "memo_id": memo_id, + "approval_id": approval_id, + "approval_binding_digest": binding, + } + + +def _serves(serves: Any) -> str: + if not isinstance(serves, dict) or set(serves) != {"hub_decision_id", "resolves"}: + raise ContractError("invalid_serves") + if serves["resolves"] not in ("on-answer", "never"): + raise ContractError("unknown_enum") + return _uuid(serves["hub_decision_id"], "hub_decision_id") + + +def _intent(intent: Any) -> None: + if not isinstance(intent, dict) or set(intent) != INTENT_KEYS: + raise ContractError("invalid_intent") + for field in ("action", "principal", "purpose"): + _text(intent[field], max_len=400, code="invalid_intent") + target = intent["target"] + if not isinstance(target, dict) or set(target) != TARGET_KEYS: + raise ContractError("invalid_intent") + for field in TARGET_KEYS: + _text(target[field], max_len=200, code="invalid_intent") + + +def _packet(packet: Any) -> None: + if not isinstance(packet, list) or not packet or len(packet) > MAX_PACKET_ITEMS: + raise ContractError("packet_required" if not packet else "packet_unbounded") + seen: set[str] = set() + for item in packet: + if not isinstance(item, dict) or set(item) != PACKET_KEYS: + raise ContractError("invalid_packet") + if item["media_type"] not in MEDIA_TYPES: + raise ContractError("unsupported_media_type") + content = item["content"] + if not isinstance(content, str) or len(content.encode("utf-8")) > MAX_PACKET_BYTES: + raise ContractError("packet_too_large") + item_id = _text(item["item_id"], max_len=200, code="invalid_packet") + if item_id in seen: + raise ContractError("duplicate_packet_item") + seen.add(item_id) + + +def _one_question(value: Any) -> str: + text = _text(value, max_len=MAX_QUESTION, code="question_invalid") + if "\n" in text or "\r" in text or "://" in text: + raise ContractError("question_invalid") + if text.count("?") != 1 or not text.endswith("?"): + raise ContractError("question_not_one_question") + return text + + +def _memo_id(value: Any) -> str: + return _text(value, pattern=_MEMO_ID_RE, code="memo_id_invalid") + + +def _slug(value: Any, code: str) -> str: + return _text(value, pattern=_SLUG_RE, code=code + "_invalid") + + +def _uuid(value: Any, code: str) -> str: + return _text(value, pattern=_UUID_RE, code=code + "_invalid") + + +def _text(value: Any, *, max_len: int = 128, pattern: re.Pattern[str] | None = None, code: str) -> str: + if not isinstance(value, str) or not value or len(value) > max_len: + raise ContractError(code) + if pattern is not None and not pattern.match(value): + raise ContractError(code) + return value + + +def _bounded_tree(value: Any) -> None: + total = 0 + + def walk(node: Any, key: str | None = None) -> None: + nonlocal total + if isinstance(node, str): + size = len(node.encode("utf-8")) + if size > MAX_PACKET_BYTES: + raise ContractError("field_too_large") + if key == "question": + _one_question(node) + total += size + elif isinstance(node, dict): + for child_key, child in node.items(): + if not isinstance(child_key, str): + raise ContractError("invalid_type") + walk(child_key, None) + walk(child, child_key) + elif isinstance(node, list): + for child in node: + walk(child, key) + elif isinstance(node, bool) or type(node) in (int, float) or node is None: + # type() excludes bool; bool subclasses int, so it must be named first. + return + else: + raise ContractError("invalid_type") + if total > MAX_ENVELOPE_BYTES: + raise ContractError("envelope_too_large") + + walk(value) + + +def _strings(value: Any): + if isinstance(value, str): + yield value + elif isinstance(value, dict): + for key, child in value.items(): + if isinstance(key, str): + yield key + yield from _strings(child) + elif isinstance(value, list): + for child in value: + yield from _strings(child) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/legacy_compat.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/legacy_compat.py new file mode 100644 index 00000000..11a4d62a --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/legacy_compat.py @@ -0,0 +1,152 @@ +"""Shared legacy REST compat metering (routes and query params).""" +from __future__ import annotations + +import logging + +from fastapi import HTTPException, Request, Response, status +from sqlalchemy.ext.asyncio import AsyncSession + +from api.services.legacy_meter import identity_from_request, record_legacy_usage + +logger = logging.getLogger(__name__) + +_LEGACY_OWNER = "state-hub.api" +# Planning horizon for legacy workstream interfaces; actual removal remains +# legacy-meter-gated per docs/workplan-terminology-transition.md. +_LEGACY_WORKSTREAM_SUNSET = "Wed, 30 Jun 2027 23:59:59 GMT" + + +def legacy_response_headers(replacement_ref: str) -> dict[str, str]: + return { + "Deprecation": "true", + "Sunset": _LEGACY_WORKSTREAM_SUNSET, + "X-StateHub-Replacement": replacement_ref, + "Link": f'<{replacement_ref}>; rel="successor-version"', + } + + +def mark_legacy_response(response: Response | None, replacement_ref: str) -> None: + if response is None: + return + for key, value in legacy_response_headers(replacement_ref).items(): + if key == "Link": + response.headers.append(key, value) + else: + response.headers[key] = value + + +def legacy_query_param_key(method: str, route: str, param: str = "workstream_id") -> str: + return f"rest_api:{method} {route}?{param}" + + +async def meter_legacy_query_param( + *, + session: AsyncSession, + request: Request | None, + response: Response | None, + method: str, + route: str, + replacement_ref: str, + param: str = "workstream_id", +) -> None: + interface_key = legacy_query_param_key(method, route, param) + mark_legacy_response(response, replacement_ref) + try: + await record_legacy_usage( + session, + interface_key=interface_key, + interface_kind="rest_api", + replacement_ref=replacement_ref, + owner_component=_LEGACY_OWNER, + replacement_verified=True, + identity=identity_from_request(request), + ) + except Exception: + await session.rollback() + logger.warning("legacy-meter failed to record %s", interface_key, exc_info=True) + + +def legacy_body_field_key(method: str, route: str, field: str = "workstream_id") -> str: + return f"rest_api:{method} {route} {field}" + + +async def meter_legacy_body_field( + *, + session: AsyncSession, + request: Request | None, + response: Response | None, + method: str, + route: str, + replacement_ref: str, + field: str = "workstream_id", +) -> None: + interface_key = legacy_body_field_key(method, route, field) + mark_legacy_response(response, replacement_ref) + try: + await record_legacy_usage( + session, + interface_key=interface_key, + interface_kind="rest_api", + replacement_ref=replacement_ref, + owner_component=_LEGACY_OWNER, + replacement_verified=True, + identity=identity_from_request(request), + ) + except Exception: + await session.rollback() + logger.warning("legacy-meter failed to record %s", interface_key, exc_info=True) + + +async def retire_legacy_route( + *, + session: AsyncSession, + request: Request | None, + response: Response | None, + interface_key: str, + replacement_ref: str, + detail: str, +) -> None: + """Record final legacy-meter usage for this route, then return 410 Gone.""" + mark_legacy_response(response, replacement_ref) + try: + await record_legacy_usage( + session, + interface_key=interface_key, + interface_kind="rest_api", + replacement_ref=replacement_ref, + owner_component=_LEGACY_OWNER, + replacement_verified=True, + identity=identity_from_request(request), + ) + except Exception: + await session.rollback() + logger.warning("legacy-meter failed to record %s", interface_key, exc_info=True) + raise HTTPException( + status_code=status.HTTP_410_GONE, + detail=detail, + headers=legacy_response_headers(replacement_ref), + ) + + +async def meter_legacy_body_from_model( + body: object, + *, + session: AsyncSession, + request: Request | None, + response: Response | None, + method: str, + route: str, + replacement_ref: str, + field: str = "workstream_id", +) -> None: + if not getattr(body, "used_legacy_workstream_id", False): + return + await meter_legacy_body_field( + session=session, + request=request, + response=response, + method=method, + route=route, + replacement_ref=replacement_ref, + field=field, + ) \ No newline at end of file diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/legacy_meter.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/legacy_meter.py new file mode 100644 index 00000000..de8f1bd4 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/legacy_meter.py @@ -0,0 +1,438 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date, datetime, timedelta, timezone +from typing import Any + +from fastapi import Request +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from api.models.legacy_meter import LegacyInterface, LegacyInterfaceUsageBucket +from api.schemas.legacy_meter import ( + LegacyInterfaceRead, + LegacyInterfaceSummary, + LegacyUsageCounters, + LegacyUsageSummary, + LegacyWeeklyReview, +) + +UNKNOWN_BUCKET = "unknown" +VALID_INTERFACE_KINDS = { + "rest_api", + "mcp_tool", + "procedure_call", + "event_subject", + "cli", + "dashboard_route", + "schema_field", +} +VALID_INTERFACE_STATUSES = {"legacy", "retirement_candidate", "retired"} + + +@dataclass(frozen=True) +class LegacyUsageIdentity: + tenant_key: str = UNKNOWN_BUCKET + user_key: str = UNKNOWN_BUCKET + component_key: str = UNKNOWN_BUCKET + + +def identity_from_request(request: Request | None) -> LegacyUsageIdentity: + if request is None: + return LegacyUsageIdentity() + headers = request.headers + return LegacyUsageIdentity( + tenant_key=_clean_bucket(headers.get("x-statehub-tenant") or headers.get("x-tenant-id")), + user_key=_clean_bucket(headers.get("x-statehub-user") or headers.get("x-user-id")), + component_key=_clean_bucket(headers.get("x-statehub-component") or headers.get("x-component")), + ) + + +async def register_legacy_interface( + session: AsyncSession, + *, + interface_key: str, + interface_kind: str, + replacement_ref: str, + owner_component: str = "state-hub", + replacement_verified: bool = False, + manual_hold: bool = False, + hold_reason: str | None = None, + notes: str | None = None, + status: str = "legacy", + commit: bool = True, + preserve_controls: bool = False, +) -> LegacyInterface: + interface_key = _required(interface_key, "interface_key") + replacement_ref = _required(replacement_ref, "replacement_ref") + interface_kind = _validate_kind(interface_kind) + status = _validate_status(status) + + existing = await _get_by_key(session, interface_key) + if existing is None: + existing = LegacyInterface( + interface_key=interface_key, + interface_kind=interface_kind, + replacement_ref=replacement_ref, + owner_component=_clean_value(owner_component, "state-hub"), + replacement_verified=replacement_verified, + manual_hold=manual_hold, + hold_reason=hold_reason, + notes=notes, + status=status, + ) + session.add(existing) + else: + existing.interface_kind = interface_kind + existing.replacement_ref = replacement_ref + existing.owner_component = _clean_value(owner_component, "state-hub") + existing.replacement_verified = existing.replacement_verified or replacement_verified + if not preserve_controls: + existing.manual_hold = manual_hold + existing.hold_reason = hold_reason + existing.notes = notes if notes is not None else existing.notes + existing.status = status + + if commit: + await session.commit() + await session.refresh(existing) + else: + await session.flush() + return existing + + +async def patch_legacy_interface( + session: AsyncSession, + interface: LegacyInterface, + updates: dict[str, Any], +) -> LegacyInterface: + if "interface_kind" in updates: + updates["interface_kind"] = _validate_kind(updates["interface_kind"]) + if "status" in updates and updates["status"] is not None: + updates["status"] = _validate_status(updates["status"]) + if updates["status"] == "retired" and interface.retired_at is None: + interface.retired_at = datetime.now(tz=timezone.utc) + for field, value in updates.items(): + if value is not None or field in {"manual_hold", "replacement_verified", "hold_reason", "notes"}: + setattr(interface, field, value) + await session.commit() + await session.refresh(interface) + return interface + + +async def record_legacy_usage( + session: AsyncSession, + *, + interface_key: str, + interface_kind: str, + replacement_ref: str, + identity: LegacyUsageIdentity | None = None, + owner_component: str = "state-hub", + replacement_verified: bool = False, + observed_at: datetime | None = None, + call_count: int = 1, + commit: bool = True, +) -> LegacyInterface: + if call_count < 1: + raise ValueError("call_count must be >= 1") + observed_at = observed_at or datetime.now(tz=timezone.utc) + if observed_at.tzinfo is None: + observed_at = observed_at.replace(tzinfo=timezone.utc) + identity = identity or LegacyUsageIdentity() + interface = await register_legacy_interface( + session, + interface_key=interface_key, + interface_kind=interface_kind, + replacement_ref=replacement_ref, + owner_component=owner_component, + replacement_verified=replacement_verified, + preserve_controls=True, + commit=False, + ) + for bucket_kind, bucket_key in ( + ("call", "total"), + ("tenant", identity.tenant_key), + ("user", identity.user_key), + ("component", identity.component_key), + ): + await _increment_bucket( + session, + interface, + period_start=observed_at.date(), + bucket_kind=bucket_kind, + bucket_key=_clean_bucket(bucket_key), + observed_at=observed_at, + call_count=call_count, + ) + if commit: + await session.commit() + await session.refresh(interface) + else: + await session.flush() + return interface + + +async def get_legacy_interface_by_key( + session: AsyncSession, + interface_key: str, +) -> LegacyInterface | None: + return await _get_by_key(session, interface_key) + + +async def list_legacy_interfaces(session: AsyncSession) -> list[LegacyInterface]: + result = await session.execute( + select(LegacyInterface).order_by(LegacyInterface.interface_kind, LegacyInterface.interface_key) + ) + return list(result.scalars().all()) + + +async def legacy_usage_summary( + session: AsyncSession, + *, + window_start: datetime | None = None, + window_end: datetime | None = None, +) -> LegacyUsageSummary: + window_end = _ensure_datetime(window_end) or datetime.now(tz=timezone.utc) + window_start = _ensure_datetime(window_start) or (window_end - timedelta(days=7)) + interfaces = await list_legacy_interfaces(session) + buckets = await _usage_buckets(session) + by_interface: dict[Any, list[LegacyInterfaceUsageBucket]] = {} + for bucket in buckets: + by_interface.setdefault(bucket.legacy_interface_id, []).append(bucket) + + summaries = [ + _summarize_interface(interface, by_interface.get(interface.id, []), window_start, window_end) + for interface in interfaces + ] + return LegacyUsageSummary( + generated_at=datetime.now(tz=timezone.utc), + window_start=window_start, + window_end=window_end, + interfaces=summaries, + ) + + +async def legacy_weekly_review( + session: AsyncSession, + *, + window_start: datetime | None = None, + window_end: datetime | None = None, +) -> LegacyWeeklyReview: + summary = await legacy_usage_summary( + session, + window_start=window_start, + window_end=window_end, + ) + candidates = [item for item in summary.interfaces if item.retirement_candidate] + return LegacyWeeklyReview( + generated_at=summary.generated_at, + window_start=summary.window_start, + window_end=summary.window_end, + activity_core_handoff={ + "activity_id": "statehub-legacy-interface-review", + "cadence": "weekly", + "source_endpoint": "/legacy-meter/weekly-review", + "state_owner": "state-hub", + "scheduler_owner": "activity-core", + }, + interfaces=summary.interfaces, + retirement_candidates=candidates, + ) + + +async def _increment_bucket( + session: AsyncSession, + interface: LegacyInterface, + *, + period_start: date, + bucket_kind: str, + bucket_key: str, + observed_at: datetime, + call_count: int, +) -> LegacyInterfaceUsageBucket: + result = await session.execute( + select(LegacyInterfaceUsageBucket).where( + LegacyInterfaceUsageBucket.legacy_interface_id == interface.id, + LegacyInterfaceUsageBucket.period_start == period_start, + LegacyInterfaceUsageBucket.bucket_kind == bucket_kind, + LegacyInterfaceUsageBucket.bucket_key == bucket_key, + ) + ) + bucket = result.scalar_one_or_none() + if bucket is None: + bucket = LegacyInterfaceUsageBucket( + legacy_interface_id=interface.id, + period_start=period_start, + bucket_kind=bucket_kind, + bucket_key=bucket_key, + call_count=call_count, + first_seen_at=observed_at, + last_seen_at=observed_at, + ) + session.add(bucket) + return bucket + + bucket.call_count += call_count + if observed_at < bucket.first_seen_at: + bucket.first_seen_at = observed_at + if observed_at > bucket.last_seen_at: + bucket.last_seen_at = observed_at + return bucket + + +async def _get_by_key(session: AsyncSession, interface_key: str) -> LegacyInterface | None: + result = await session.execute( + select(LegacyInterface).where(LegacyInterface.interface_key == interface_key) + ) + return result.scalar_one_or_none() + + +async def _usage_buckets(session: AsyncSession) -> list[LegacyInterfaceUsageBucket]: + result = await session.execute(select(LegacyInterfaceUsageBucket)) + return list(result.scalars().all()) + + +def _summarize_interface( + interface: LegacyInterface, + buckets: list[LegacyInterfaceUsageBucket], + window_start: datetime, + window_end: datetime, +) -> LegacyInterfaceSummary: + all_time = _counters(buckets) + window_buckets = [ + bucket for bucket in buckets + if bucket.last_seen_at >= window_start and bucket.first_seen_at < window_end + ] + window = _counters(window_buckets) + last_seen = max((bucket.last_seen_at for bucket in buckets), default=None) + retirement_candidate, reason = _retirement_state( + interface, + window.calls, + all_time_calls=all_time.calls, + last_seen_at=last_seen, + window_end=window_end, + ) + return LegacyInterfaceSummary( + interface=LegacyInterfaceRead.model_validate(interface), + all_time=all_time, + window=window, + last_seen_at=last_seen, + retirement_candidate=retirement_candidate, + retirement_reason=reason, + ) + + +def _counters(buckets: list[LegacyInterfaceUsageBucket]) -> LegacyUsageCounters: + calls = sum(bucket.call_count for bucket in buckets if bucket.bucket_kind == "call") + tenants = _bucket_counts(buckets, "tenant") + users = _bucket_counts(buckets, "user") + components = _bucket_counts(buckets, "component") + return LegacyUsageCounters( + calls=calls, + tenant_count=len(tenants), + user_count=len(users), + component_count=len(components), + tenants=tenants, + users=users, + components=components, + ) + + +def _bucket_counts(buckets: list[LegacyInterfaceUsageBucket], bucket_kind: str) -> dict[str, int]: + counts: dict[str, int] = {} + for bucket in buckets: + if bucket.bucket_kind == bucket_kind: + counts[bucket.bucket_key] = counts.get(bucket.bucket_key, 0) + bucket.call_count + return counts + + +# Minimum quiet period before an interface may be retired, scaled to how much +# traffic it ever carried. A review window only proves the interface was silent +# for that window; for a high-volume interface that is weak evidence, since a +# monthly or quarterly caller can easily fall outside it. Thresholds are +# (all-time calls, minimum days since last_seen_at). +RETIREMENT_QUIET_LADDER: tuple[tuple[int, int], ...] = ( + (10_000, 60), + (100, 30), + (1, 7), +) + + +def _required_quiet_days(all_time_calls: int) -> int: + """Days of silence required before retirement, by all-time call volume.""" + for threshold, days in RETIREMENT_QUIET_LADDER: + if all_time_calls >= threshold: + return days + return 0 + + +def _retirement_state( + interface: LegacyInterface, + window_calls: int, + *, + all_time_calls: int = 0, + last_seen_at: datetime | None = None, + window_end: datetime | None = None, +) -> tuple[bool, str]: + if interface.status == "retired": + return False, "already retired" + if interface.manual_hold: + return False, interface.hold_reason or "manual hold" + if not interface.replacement_ref.strip(): + return False, "missing replacement reference" + if not interface.replacement_verified: + return False, "replacement not verified" + if window_calls > 0: + return False, f"{window_calls} call(s) in review window" + + required_days = _required_quiet_days(all_time_calls) + if required_days and last_seen_at is not None and window_end is not None: + quiet_days = (window_end - _ensure_datetime(last_seen_at)).days + if quiet_days < required_days: + return False, ( + f"quiet {quiet_days}d of {required_days}d required " + f"for {all_time_calls} all-time call(s)" + ) + return True, ( + f"no usage in review window; quiet {quiet_days}d " + f"(>= {required_days}d required for {all_time_calls} all-time call(s))" + ) + return True, "no measured usage in review window" + + +def _ensure_datetime(value: datetime | None) -> datetime | None: + if value is None: + return None + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value + + +def _required(value: str | None, field: str) -> str: + cleaned = _clean_value(value) + if not cleaned: + raise ValueError(f"{field} is required") + return cleaned + + +def _clean_value(value: str | None, default: str = "") -> str: + return str(value or default).strip() + + +def _clean_bucket(value: str | None) -> str: + cleaned = _clean_value(value) + return cleaned or UNKNOWN_BUCKET + + +def _validate_kind(kind: str) -> str: + cleaned = _required(kind, "interface_kind") + if cleaned not in VALID_INTERFACE_KINDS: + raise ValueError(f"interface_kind must be one of {sorted(VALID_INTERFACE_KINDS)}") + return cleaned + + +def _validate_status(status: str) -> str: + cleaned = _required(status, "status") + if cleaned not in VALID_INTERFACE_STATUSES: + raise ValueError(f"status must be one of {sorted(VALID_INTERFACE_STATUSES)}") + return cleaned diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/lifecycle.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/lifecycle.py new file mode 100644 index 00000000..ea770259 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/lifecycle.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from api.task_status import ACTIVE_TASK_STATUSES, normalize_task_status, status_value +from api.workplan_status import normalize_workplan_status + +TASK_STARTED_STATUS = "progress" +TASK_NOT_STARTED_STATUS = "todo" +TASK_ACTIVE_STATUSES = ACTIVE_TASK_STATUSES +PARENT_ACTIVATION_STATUSES = {"proposed", "ready", "backlog"} + +# Legacy alias +normalize_workstream_status = normalize_workplan_status + + +@dataclass(frozen=True) +class LifecycleTransitionResult: + entity_type: str + previous_status: str + target_status: str + changed: bool + parent_activated: bool = False + + +def should_activate_parent_for_task_start( + *, + previous_task_status: Any, + new_task_status: Any, + parent_workplan_status: Any = None, + parent_workstream_status: Any = None, +) -> bool: + """Return whether a task start should move its parent to active.""" + parent_status = parent_workplan_status if parent_workplan_status is not None else parent_workstream_status + return ( + status_value(previous_task_status) == TASK_NOT_STARTED_STATUS + and status_value(new_task_status) == TASK_STARTED_STATUS + and normalize_workplan_status(parent_status) + in PARENT_ACTIVATION_STATUSES + ) + + +def has_active_task_status(task_statuses: list[Any] | tuple[Any, ...]) -> bool: + """Return whether any task status represents currently active work.""" + return any(status_value(status) in TASK_ACTIVE_STATUSES for status in task_statuses) + + +def should_activate_parent_for_active_tasks( + *, + parent_workplan_status: Any = None, + parent_workstream_status: Any = None, + task_statuses: list[Any] | tuple[Any, ...], +) -> bool: + """Return whether existing task state implies an active parent workplan.""" + parent_status = parent_workplan_status if parent_workplan_status is not None else parent_workstream_status + return ( + normalize_workplan_status(parent_status) + in PARENT_ACTIVATION_STATUSES + and has_active_task_status(task_statuses) + ) + + +def activate_parent_for_task_start( + *, + previous_task_status: Any, + new_task_status: Any, + parent_workplan: Any = None, + parent_workstream: Any = None, +) -> bool: + """Activate a planning-state parent workplan when real task work starts.""" + parent = parent_workplan if parent_workplan is not None else parent_workstream + if parent is None: + return False + if not should_activate_parent_for_task_start( + previous_task_status=previous_task_status, + new_task_status=new_task_status, + parent_workplan_status=getattr(parent, "status", None), + parent_workstream_status=getattr(parent, "status", None), + ): + return False + parent.status = "active" + return True + + +def transition_workplan_status( + workplan: Any, + target_status: Any, +) -> LifecycleTransitionResult: + """Apply a canonical workplan status transition.""" + previous_status = normalize_workplan_status(getattr(workplan, "status", None)) + normalised_target = normalize_workplan_status(target_status) + workplan.status = normalised_target + return LifecycleTransitionResult( + entity_type="workplan", + previous_status=previous_status, + target_status=normalised_target, + changed=previous_status != normalised_target, + ) + + +transition_workstream_status = transition_workplan_status + + +def transition_task_status( + task: Any, + target_status: Any, + *, + parent_workplan: Any = None, + parent_workstream: Any = None, + previous_task_status: Any = None, + status_coercer: Any = None, +) -> LifecycleTransitionResult: + """Apply a task status transition and activate the parent when work starts.""" + parent = parent_workplan if parent_workplan is not None else parent_workstream + previous_status = status_value( + getattr(task, "status", None) + if previous_task_status is None + else previous_task_status + ) + normalised_target = normalize_task_status(target_status) + task.status = status_coercer(normalised_target) if status_coercer else normalised_target + parent_activated = activate_parent_for_task_start( + previous_task_status=previous_status, + new_task_status=normalised_target, + parent_workplan=parent, + parent_workstream=parent, + ) + return LifecycleTransitionResult( + entity_type="task", + previous_status=previous_status, + target_status=normalised_target, + changed=previous_status != normalised_target, + parent_activated=parent_activated, + ) \ No newline at end of file diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/markitect_templates.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/markitect_templates.py new file mode 100644 index 00000000..813573c9 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/markitect_templates.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import json +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import Any + +import yaml + +from api.config import settings + + +class MarkitectUnavailable(RuntimeError): + """Raised when neither MarkiTect's Python API nor CLI can render templates.""" + + +class MarkitectRenderError(RuntimeError): + """Raised when MarkiTect rejects a template or data payload.""" + + +def render_markdown_template(template_path: Path, data: dict[str, Any]) -> str: + """Render a Markdown template with MarkiTect. + + Prefer the Python API when available. Fall back to the CLI because local + workstations may have MarkiTect installed as a standalone tool. + """ + + template_text = template_path.read_text(encoding="utf-8") + try: + from markitect_tool.template import render_template + except ImportError: + return _render_with_cli(template_path, data) + + try: + return render_template(template_text, data, strict=True).markdown + except Exception as exc: # MarkiTect raises its own TemplateError hierarchy. + raise MarkitectRenderError(str(exc)) from exc + + +def _render_with_cli(template_path: Path, data: dict[str, Any]) -> str: + cli = _find_markitect_cli() + if cli is None: + raise MarkitectUnavailable( + "MarkiTect is required to render RecentlyOnScope digests. " + "Install markitect_tool, put `markitect` on PATH, or set STATE_HUB_MARKITECT_CLI_PATH." + ) + + with tempfile.TemporaryDirectory(prefix="state-hub-markitect-") as tmp: + data_path = Path(tmp) / "data.yaml" + data_path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8") + result = _run_first_supported( + [ + [cli, "template", "render", str(template_path), "--data", str(data_path)], + [cli, "template-render", str(template_path), str(data_path), "--strict"], + ] + ) + if result.returncode != 0: + detail = (result.stderr or result.stdout or "MarkiTect render failed").strip() + raise MarkitectRenderError(detail) + return result.stdout + + +def inspect_markdown_template(template_path: Path) -> dict[str, Any]: + """Return MarkiTect template analysis data where available.""" + + template_text = template_path.read_text(encoding="utf-8") + try: + from markitect_tool.template import analyze_template + except ImportError: + return _inspect_with_cli(template_path) + + analysis = analyze_template(template_text).to_dict() + if not analysis.get("valid", False): + raise MarkitectRenderError("; ".join(analysis.get("syntax_errors") or [])) + return analysis + + +def _inspect_with_cli(template_path: Path) -> dict[str, Any]: + cli = _find_markitect_cli() + if cli is None: + raise MarkitectUnavailable( + "MarkiTect is required to inspect RecentlyOnScope templates. " + "Install markitect_tool, put `markitect` on PATH, or set STATE_HUB_MARKITECT_CLI_PATH." + ) + result = subprocess.run( + [cli, "template", "inspect", str(template_path), "--format", "json"], + check=False, + capture_output=True, + text=True, + ) + if _is_unsupported_cli_command(result): + with tempfile.TemporaryDirectory(prefix="state-hub-markitect-") as tmp: + data_path = Path(tmp) / "empty.yaml" + data_path.write_text("{}\n", encoding="utf-8") + legacy = subprocess.run( + [cli, "template-render", str(template_path), str(data_path), "--lenient", "--validate"], + check=False, + capture_output=True, + text=True, + ) + if legacy.returncode != 0: + detail = (legacy.stderr or legacy.stdout or "MarkiTect template inspection failed").strip() + raise MarkitectRenderError(detail) + return {"valid": True, "variables": [], "legacy_cli": True} + if result.returncode != 0: + detail = (result.stderr or result.stdout or "MarkiTect template inspection failed").strip() + raise MarkitectRenderError(detail) + try: + return json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise MarkitectRenderError("MarkiTect returned invalid JSON during template inspection") from exc + + +def _find_markitect_cli() -> str | None: + candidates = [ + settings.state_hub_markitect_cli_path, + shutil.which("markitect"), + "/home/worsch/bin/markitect", + ] + for candidate in candidates: + if not candidate: + continue + path = Path(candidate).expanduser() + if path.is_file(): + return str(path) + if shutil.which(str(candidate)): + return str(candidate) + return None + + +def _run_first_supported(commands: list[list[str]]) -> subprocess.CompletedProcess[str]: + last_result: subprocess.CompletedProcess[str] | None = None + for command in commands: + result = subprocess.run(command, check=False, capture_output=True, text=True) + if not _is_unsupported_cli_command(result): + return result + last_result = result + return last_result if last_result is not None else subprocess.CompletedProcess([], 1, "", "") + + +def _is_unsupported_cli_command(result: subprocess.CompletedProcess[str]) -> bool: + output = f"{result.stdout}\n{result.stderr}" + return result.returncode != 0 and ( + "No such command" in output + or "Missing command" in output + or "Got unexpected extra argument" in output + ) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/memo_dispatch.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/memo_dispatch.py new file mode 100644 index 00000000..0037f45e --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/memo_dispatch.py @@ -0,0 +1,174 @@ +"""Private, explicitly queued State Hub memo dispatch. No API route or human act.""" +from contextlib import contextmanager +import argparse +import fcntl +import json +import os +from pathlib import Path +import signal +import stat +import threading +import urllib.error +import urllib.request + +from api.services.informed_decision_contract import MAX_ENVELOPE_BYTES, validate_envelope +from api.services.memo_forwarding import MemoForwarder + +ORIGIN = 'http://informed-decision-workload.informed-decision.svc.cluster.local:8081' +TOKEN = '/var/run/secrets/state-hub/informed-decision/intake-token' +MAX_RESPONSE = 65536 + + +class Refused(RuntimeError): + pass + + +def read_json(path, limit): + with Path(path).open('rb') as stream: + raw = stream.read(limit + 1) + if len(raw) > limit: + raise Refused('input_too_large') + return json.loads(raw) + + +def token_file(path): + # Reopen on every attempt: projected files rotate by symlink replacement. + with Path(path).open('rb') as stream: + raw = stream.read(16385) + value = raw.decode('ascii').strip() + if not value or len(raw) > 16384 or any(c.isspace() for c in value): + raise Refused('token_unavailable') + return value + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + + +def transport(method, url, body, headers): + if method != 'POST' or url != ORIGIN + '/v1/memos': + raise Refused('route_refused') + raw = json.dumps(body, allow_nan=False, separators=(',', ':')).encode() + if len(raw) > MAX_ENVELOPE_BYTES: + raise Refused('input_too_large') + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()) + request = urllib.request.Request(url, data=raw, headers=headers, method=method) + try: + try: + response = opener.open(request, timeout=5) + except urllib.error.HTTPError as error: + response = error + with response: + status = response.code + raw = response.read(MAX_RESPONSE + 1) + if len(raw) > MAX_RESPONSE: + raise OSError('transport_unavailable') + try: + body = json.loads(raw) + except (ValueError, UnicodeError): + body = {} + return status, body + except Exception: + # Never propagate URLs, response bodies, credentials or transport errors. + raise OSError('transport_unavailable') from None + + +class Dispatcher: + def __init__(self, directory, *, send=transport, read_token=lambda: token_file(TOKEN), clock=None): + self.directory = Path(directory) + if not self.directory.is_absolute(): + raise Refused('absolute_queue_path_required') + self.directory.mkdir(mode=0o700, parents=False, exist_ok=True) + info = self.directory.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700: + raise Refused('private_queue_required') + # SQLite files inherit 0600; reject existing unsafe custody, including links. + self.path = self.directory / 'forward.sqlite' + for name in ('forward.sqlite', 'forward.sqlite-wal', 'forward.sqlite-shm', 'dispatch.lock'): + path = self.directory / name + if path.exists() or path.is_symlink(): + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o600: + raise Refused('private_queue_required') + self.send, self.read_token, self.clock = send, read_token, clock + with self.lock(): + previous = os.umask(0o077) + try: + self.ledger = MemoForwarder(self.path, transport=send, + intake_base_url=ORIGIN, hub_base_url='http://state-hub.state-hub.svc.cluster.local:8000', + token_reader=read_token, **({'clock': clock} if clock else {})) + finally: + os.umask(previous) + + @contextmanager + def lock(self): + fd = os.open(self.directory / 'dispatch.lock', os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + previous = os.umask(0o077) + try: + yield + finally: + os.umask(previous) + except BlockingIOError: + raise Refused('queue_busy') from None + finally: + os.close(fd) + + def enqueue(self, envelope): + validate_envelope(envelope) + if envelope['memo_class'] != 'decision-record' or envelope['review_group'] != 'net-kingdom-admins': + raise Refused('routing_refused') + with self.lock(): + return self.ledger.submit(envelope, dispatch=False) + + def once(self): + # Hold across remote attempt + local receipt to serialize other processes. + # If killed after POST, restart replays the same envelope/key. Intake owns + # native create uncertainty and refuses automatic native-approval repost. + with self.lock(): + return self.ledger.run_due() + + def counts(self): + with self.lock(), self.ledger._connect() as db: + return dict(db.execute('SELECT state,COUNT(*) FROM pose_ledger GROUP BY state')) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--queue', required=True, type=Path) + commands = parser.add_subparsers(dest='command', required=True) + commands.add_parser('enqueue').add_argument('--envelope', required=True, type=Path) + for name in ('once', 'run', 'status'): + commands.add_parser(name) + args = parser.parse_args(argv) + try: + dispatcher = Dispatcher(args.queue) + if args.command == 'enqueue': + result = dispatcher.enqueue(read_json(args.envelope, MAX_ENVELOPE_BYTES)) + print(json.dumps({'status': result.status, 'detail': result.detail})) + elif args.command == 'once': + result = dispatcher.once() + print(json.dumps({'status': result.status if result else 'idle'})) + elif args.command == 'status': + print(json.dumps(dispatcher.counts())) + else: + stopped = threading.Event() + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: stopped.set()) + while not stopped.is_set(): + try: + dispatcher.once() + except Refused as exc: + if str(exc) != 'queue_busy': + raise + stopped.wait(2) + return 0 + except Exception: + print('{"error":"dispatch_refused"}') + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/memo_forwarding.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/memo_forwarding.py new file mode 100644 index 00000000..7cb9ed84 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/memo_forwarding.py @@ -0,0 +1,447 @@ +"""Pose, replay, and conflict ledger with injected transport. + +Not imported by the API runtime. The private dispatcher composes this ledger +under a process lock; dev tests retain the injectable adapter. It never reads a token +file, mints a credential, or dials a default host. Tests inject the transport +and, when a pose should carry a header, a token callable. The callable's +value is not retained. + +Choosing the workload identity that projects an intake-audience token, and +reviewing that identity into the submitter registry, is an owner step. This +fixture does not name or admit an account. +""" +from __future__ import annotations + +import json +import sqlite3 +import time +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Callable, Iterator + +from api.services.informed_decision_contract import ( + PROVENANCE_NOTE, + ContractError, + bounded_receipt, + correlate_resolution, + resolution_applies, + safe_error_code, + submission_digest, + validate_envelope, +) + +Transport = Callable[[str, str, dict, dict], tuple[int, dict]] +TokenReader = Callable[[], str] + +_RETRYABLE = frozenset({408, 429}) + + +@dataclass(frozen=True) +class ForwardResult: + status: str + idempotency_key: str + receipt: dict | None + drafted_by: str | None + drafted_by_provenance: str + provenance_note: str + detail: str = "" + + +class MemoForwarder: + def __init__( + self, + path: str | Path, + *, + transport: Transport, + intake_base_url: str, + hub_base_url: str, + token_reader: TokenReader | None = None, + clock: Callable[[], float] = time.time, + ) -> None: + self.path = str(path) + self.transport = transport + self.intake_base_url = _base_url(intake_base_url) + self.hub_base_url = _base_url(hub_base_url) + self.token_reader = token_reader + self.clock = clock + Path(self.path).parent.mkdir(parents=True, exist_ok=True) + with self._connect() as db: + db.executescript( + """ + CREATE TABLE IF NOT EXISTS pose_ledger ( + idempotency_key TEXT PRIMARY KEY, + submission_digest TEXT NOT NULL, + envelope_json TEXT NOT NULL, + drafted_by TEXT, + drafted_by_provenance TEXT NOT NULL + CHECK (drafted_by_provenance IN ('asserted', 'absent')), + provenance_note TEXT NOT NULL, + state TEXT NOT NULL + CHECK (state IN ('pending', 'delivered', 'blocked')), + receipt_json TEXT, + attempts INTEGER NOT NULL DEFAULT 0, + next_attempt REAL NOT NULL DEFAULT 0, + last_error TEXT, + blocked_reason TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS resolution_ledger ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + decision_id TEXT NOT NULL, + memo_id TEXT NOT NULL, + verb TEXT NOT NULL, + resolves TEXT, + status TEXT NOT NULL, + error_type TEXT, + payload_json TEXT, + next_attempt REAL, + created_at TEXT NOT NULL + ); + """ + ) + + def submit(self, envelope: dict, *, dispatch: bool = True) -> ForwardResult: + checked = validate_envelope(envelope) + key = checked["idempotency_key"] + body_digest = submission_digest(checked) + drafted_by = checked.get("drafted_by") + provenance = "asserted" if drafted_by else "absent" + now = self.clock() + early: tuple[str, str] | None = None + with self._connect() as db: + row = db.execute( + "SELECT * FROM pose_ledger WHERE idempotency_key=?", (key,) + ).fetchone() + if row is None: + db.execute( + """INSERT INTO pose_ledger ( + idempotency_key, submission_digest, envelope_json, drafted_by, + drafted_by_provenance, provenance_note, state, attempts, + next_attempt, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, 'pending', 0, 0, ?, ?)""", + ( + key, body_digest, _dump(checked), drafted_by, provenance, + PROVENANCE_NOTE, _iso(now), _iso(now), + ), + ) + elif row["submission_digest"] != body_digest: + db.execute( + """UPDATE pose_ledger SET blocked_reason='idempotency_conflict', + updated_at=? WHERE idempotency_key=?""", + (_iso(now), key), + ) + if row["state"] != "delivered": + db.execute( + "UPDATE pose_ledger SET state='blocked' WHERE idempotency_key=?", + (key,), + ) + early = ("blocked", "idempotency_conflict") + elif row["state"] == "delivered": + early = ("replay", "replay") + elif row["state"] == "blocked": + early = ("blocked", row["blocked_reason"] or "blocked") + elif row["next_attempt"] > now: + early = ("pending", row["last_error"] or "waiting") + if early is not None: + return self._result(key, *early) + if not dispatch: + return self._result(key, "pending", "queued") + return self._attempt(key) + + def run_due(self) -> ForwardResult | None: + now = self.clock() + with self._connect() as db: + row = db.execute( + """SELECT idempotency_key FROM pose_ledger + WHERE state='pending' AND next_attempt<=? + ORDER BY created_at LIMIT 1""", + (now,), + ).fetchone() + key = None if row is None else row["idempotency_key"] + if key is None: + return None + return self._attempt(key) + + def pose(self, key: str) -> dict | None: + with self._connect() as db: + row = db.execute( + "SELECT * FROM pose_ledger WHERE idempotency_key=?", (key,) + ).fetchone() + return dict(row) if row is not None else None + + def observe_resolution( + self, *, decision_id: str, memo_id: str, verb: str, serves: dict | None, payload: dict | None + ) -> str: + resolves = serves.get("resolves") if isinstance(serves, dict) else None + if not resolution_applies(verb, serves): + self._append_resolution( + decision_id=decision_id, memo_id=memo_id, verb=verb, resolves=resolves, + status="not_applicable", error_type=None, payload=None, retry_at=None, + ) + return "not_applicable" + prior = self._latest_resolution(decision_id, memo_id) + if prior and prior["status"] in {"resolved", "already_resolved", "conflict"}: + return prior["status"] + if prior and prior["status"] == "failed": + if prior["next_attempt"] is not None and prior["next_attempt"] > self.clock(): + return "failed" + if payload is None and prior["payload_json"]: + payload = json.loads(prior["payload_json"]) + if not isinstance(payload, dict): + raise ValueError("resolution payload is required") + encoded = _dump(payload) + if len(encoded.encode("utf-8")) > 8192: + self._append_resolution( + decision_id=decision_id, memo_id=memo_id, verb=verb, resolves=resolves, + status="conflict", error_type="payload_unbounded", payload=None, retry_at=None, + ) + return "conflict" + try: + status, _body = self.transport( + "POST", + f"{self.hub_base_url}/decisions/{decision_id}/resolve", + payload, + {}, + ) + status = int(status) + except OSError as exc: + self._append_resolution( + decision_id=decision_id, memo_id=memo_id, verb=verb, resolves=resolves, + status="failed", error_type=type(exc).__name__, payload=payload, + retry_at=self.clock() + _backoff(1), + ) + return "failed" + except (TypeError, ValueError): + self._append_resolution( + decision_id=decision_id, memo_id=memo_id, verb=verb, resolves=resolves, + status="failed", error_type="transport_invalid", payload=payload, + retry_at=self.clock() + _backoff(1), + ) + return "failed" + existing = None + if status == 409: + try: + read_status, existing = self.transport( + "GET", f"{self.hub_base_url}/decisions/{decision_id}", {}, {} + ) + if int(read_status) != 200 or not isinstance(existing, dict): + existing = None + raise OSError("resolution_unreadable") + except OSError as exc: + error = exc.__class__.__name__ if str(exc) != "resolution_unreadable" else "resolution_unreadable" + self._append_resolution( + decision_id=decision_id, memo_id=memo_id, verb=verb, resolves=resolves, + status="failed", error_type=error, payload=payload, + retry_at=self.clock() + _backoff(1), + ) + return "failed" + outcome = correlate_resolution(memo_id, post_status=status, existing=existing) + self._append_resolution( + decision_id=decision_id, memo_id=memo_id, verb=verb, resolves=resolves, + status=outcome, error_type=None if outcome != "failed" else f"http_{status}", + payload=payload, retry_at=None if outcome != "failed" else self.clock() + _backoff(1), + ) + return outcome + + def run_due_resolution(self) -> str | None: + now = self.clock() + with self._connect() as db: + row = db.execute( + """SELECT * FROM resolution_ledger + WHERE status='failed' AND next_attempt IS NOT NULL AND next_attempt<=? + ORDER BY id DESC LIMIT 1""", + (now,), + ).fetchone() + pending = dict(row) if row is not None else None + if pending is None: + return None + payload = json.loads(pending["payload_json"]) if pending["payload_json"] else None + serves = {"hub_decision_id": pending["decision_id"], "resolves": pending["resolves"]} + return self.observe_resolution( + decision_id=pending["decision_id"], + memo_id=pending["memo_id"], + verb=pending["verb"], + serves=serves, + payload=payload, + ) + + def resolutions(self) -> list[dict]: + with self._connect() as db: + rows = db.execute( + "SELECT * FROM resolution_ledger ORDER BY id" + ).fetchall() + return [dict(row) for row in rows] + + def _attempt(self, key: str) -> ForwardResult: + with self._connect() as db: + row = db.execute( + "SELECT * FROM pose_ledger WHERE idempotency_key=?", (key,) + ).fetchone() + current = dict(row) if row is not None else None + if current is None or current["state"] != "pending": + return self._result(key, current["state"] if current else "blocked", "not_pending") + envelope = json.loads(current["envelope_json"]) + token, token_error = self._read_token() + if token_error: + self._mark_retry(key, token_error) + return self._result(key, "pending", token_error) + headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + try: + status, body = self.transport( + "POST", f"{self.intake_base_url}/v1/memos", envelope, headers + ) + status = int(status) + except OSError as exc: + self._mark_retry(key, type(exc).__name__) + return self._result(key, "pending", type(exc).__name__) + except (TypeError, ValueError): + self._mark_retry(key, "transport_invalid") + return self._result(key, "pending", "transport_invalid") + if status in (200, 201): + try: + receipt = bounded_receipt(body, envelope["memo"]["id"]) + except ContractError as exc: + code = safe_error_code(exc.code, "receipt_mismatch") + self._mark_blocked(key, code) + return self._result(key, "blocked", code) + self._mark_delivered(key, receipt) + return self._result(key, "delivered", "delivered") + if status == 409 or (status not in _RETRYABLE and 400 <= status < 500): + code = safe_error_code( + body.get("error") if isinstance(body, dict) else None, + "conflict" if status == 409 else f"http_{status}", + ) + self._mark_blocked(key, code) + return self._result(key, "blocked", code) + self._mark_retry(key, f"http_{status}") + return self._result(key, "pending", f"http_{status}") + + def _read_token(self) -> tuple[str | None, str | None]: + if self.token_reader is None: + return None, "projected_token_absent" + try: + token = self.token_reader() + except Exception as exc: + return None, type(exc).__name__ + if not isinstance(token, str) or not token.strip() or token != token.strip(): + return None, "projected_token_absent" + if len(token) > 16384: + return None, "projected_token_absent" + return token, None + + def _mark_retry(self, key: str, error: str) -> None: + now = self.clock() + with self._connect() as db: + attempts = db.execute( + "SELECT attempts FROM pose_ledger WHERE idempotency_key=?", (key,) + ).fetchone()["attempts"] + 1 + db.execute( + """UPDATE pose_ledger SET state='pending', attempts=?, last_error=?, + next_attempt=?, updated_at=? WHERE idempotency_key=?""", + (attempts, error, now + _backoff(attempts), _iso(now), key), + ) + + def _mark_blocked(self, key: str, reason: str) -> None: + now = self.clock() + with self._connect() as db: + db.execute( + """UPDATE pose_ledger SET state='blocked', blocked_reason=?, last_error=?, + updated_at=? WHERE idempotency_key=?""", + (reason, reason, _iso(now), key), + ) + + def _mark_delivered(self, key: str, receipt: dict) -> None: + now = self.clock() + with self._connect() as db: + db.execute( + """UPDATE pose_ledger SET state='delivered', receipt_json=?, last_error=NULL, + blocked_reason=NULL, updated_at=? WHERE idempotency_key=?""", + (_dump(receipt), _iso(now), key), + ) + + def _result(self, key: str, status: str, detail: str) -> ForwardResult: + row = self.pose(key) + receipt = json.loads(row["receipt_json"]) if row and row["receipt_json"] else None + return ForwardResult( + status=status, + idempotency_key=key, + receipt=receipt, + drafted_by=row["drafted_by"] if row else None, + drafted_by_provenance=row["drafted_by_provenance"] if row else "absent", + provenance_note=row["provenance_note"] if row else PROVENANCE_NOTE, + detail=detail, + ) + + def _append_resolution( + self, *, decision_id, memo_id, verb, resolves, status, error_type, payload, retry_at + ) -> None: + with self._connect() as db: + db.execute( + """UPDATE resolution_ledger SET next_attempt=NULL + WHERE decision_id=? AND memo_id=? AND status='failed'""", + (decision_id, memo_id), + ) + db.execute( + """INSERT INTO resolution_ledger ( + decision_id, memo_id, verb, resolves, status, error_type, + payload_json, next_attempt, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + decision_id, memo_id, verb, resolves, status, error_type, + _dump(payload) if payload is not None else None, + retry_at, _iso(self.clock()), + ), + ) + + def _latest_resolution(self, decision_id: str, memo_id: str) -> dict | None: + with self._connect() as db: + row = db.execute( + """SELECT * FROM resolution_ledger + WHERE decision_id=? AND memo_id=? ORDER BY id DESC LIMIT 1""", + (decision_id, memo_id), + ).fetchone() + return dict(row) if row is not None else None + + @contextmanager + def _connect(self) -> Iterator[sqlite3.Connection]: + db = sqlite3.connect(self.path) + db.row_factory = sqlite3.Row + try: + db.execute("PRAGMA journal_mode=WAL") + yield db + db.commit() + except Exception: + db.rollback() + raise + finally: + db.close() + + +def _base_url(value: str) -> str: + if ( + not isinstance(value, str) + or not (value.startswith("http://") or value.startswith("https://")) + ): + raise ValueError("base_url is required") + if "127.0.0.1" in value or "localhost" in value: + raise ValueError("base_url must be an injected test host") + return value.rstrip("/") + + +def _backoff(attempts: int) -> float: + return float(min(3600, 2 ** min(attempts, 11))) + + +def _dump(value: dict) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def _iso(timestamp: float) -> str: + return datetime.fromtimestamp(timestamp, timezone.utc).isoformat() + + +# Compatibility for the original injectable development fixture. +DevIntakeFixture = MemoForwarder diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/memo_notification.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/memo_notification.py new file mode 100644 index 00000000..7f2d0ea2 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/memo_notification.py @@ -0,0 +1,67 @@ +"""Deduplicated informed-decision notification receipts. + +The signed-in overview is the review entry point. A receipt stores and +announces only memo id, question, class, and submitter. Replays of the same +delivery id return the retained receipt; a different body conflicts and leaves +the original readable. +""" +from __future__ import annotations + +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from api.models.agent_message import AgentMessage +from api.models.memo_notification import MemoNotificationReceipt +from api.services.informed_decision_contract import Notification +from api.services.repository_aliases import canonicalize_repository_slug + +NOTIFICATION_SENDER = "informed-decision" +NOTIFICATION_RECIPIENT = "bernd" + + +class NotificationConflict(Exception): + def __init__(self, receipt: MemoNotificationReceipt) -> None: + self.receipt = receipt + + +async def accept_notification( + session: AsyncSession, delivery_id: str, notice: Notification +) -> tuple[MemoNotificationReceipt, bool]: + existing = await session.get(MemoNotificationReceipt, delivery_id) + if existing is not None: + if existing.body_digest != notice.body_digest: + raise NotificationConflict(existing) + return existing, True + + message = AgentMessage( + from_agent=await canonicalize_repository_slug(session, NOTIFICATION_SENDER), + to_agent=await canonicalize_repository_slug(session, NOTIFICATION_RECIPIENT), + subject=notice.message_subject(), + body=notice.message_text(), + kind="message", + ) + session.add(message) + await session.flush() + receipt = MemoNotificationReceipt( + delivery_id=delivery_id, + body_digest=notice.body_digest, + memo_id=notice.memo_id, + question=notice.question, + memo_class=notice.memo_class, + submitter=notice.submitter, + message_id=message.id, + authoritative_review="signed-in-overview", + ) + session.add(receipt) + try: + await session.commit() + except IntegrityError: + await session.rollback() + existing = await session.get(MemoNotificationReceipt, delivery_id) + if existing is None: + raise + if existing.body_digest != notice.body_digest: + raise NotificationConflict(existing) + return existing, True + await session.refresh(receipt) + return receipt, False diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/message_receipts.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/message_receipts.py new file mode 100644 index 00000000..8e120805 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/message_receipts.py @@ -0,0 +1,212 @@ +"""Per-recipient broadcast receipts and standing notices (STATE-WP-0093). + +Broadcast visibility is per reader: a ``message_receipts`` row per +(message, agent) records delivery, read and acknowledgement. Direct messages +never get receipts and keep the global ``AgentMessage.read_at``. + +- ``news`` broadcasts stay in a reader's unread inbox until that reader has + any receipt (delivery at orientation is enough). +- ``standing`` broadcasts stay until the reader acknowledges, the notice + expires, or it is superseded (which archives it). +""" +from __future__ import annotations + +import uuid +from collections.abc import Iterable, Sequence +from datetime import datetime, timedelta, timezone + +from sqlalchemy import and_, exists, func, not_, or_, select +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncSession + +from api.models.agent_message import BROADCAST, AgentMessage, MessageReceipt +from api.models.managed_repo import ManagedRepo +from api.schemas.agent_message import NoticeStatus, StandingNoticeDigest + +NEWS_DEFAULT_TTL = timedelta(days=30) + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def is_broadcast(message: AgentMessage) -> bool: + return message.to_agent == BROADCAST + + +def not_expired(now: datetime): + return or_(AgentMessage.expires_at.is_(None), AgentMessage.expires_at > now) + + +def broadcast_unread_clause(reader_values: Sequence[str], now: datetime): + """SQL clause: broadcast is still unread for a reader (any of its slugs).""" + has_any_receipt = exists().where( + and_( + MessageReceipt.message_id == AgentMessage.id, + MessageReceipt.agent.in_(reader_values), + ) + ) + has_ack = exists().where( + and_( + MessageReceipt.message_id == AgentMessage.id, + MessageReceipt.agent.in_(reader_values), + MessageReceipt.acknowledged_at.is_not(None), + ) + ) + return and_( + AgentMessage.to_agent == BROADCAST, + not_expired(now), + or_( + and_(AgentMessage.kind == "standing", not_(has_ack)), + and_(AgentMessage.kind != "standing", not_(has_any_receipt)), + ), + ) + + +async def upsert_receipt( + session: AsyncSession, + message_ids: Iterable[uuid.UUID], + agent: str, + *, + delivered: bool = False, + read: bool = False, + acknowledged: bool = False, + at: datetime | None = None, +) -> None: + """Idempotently stamp receipt fields; the first timestamp wins.""" + ids = list(message_ids) + if not ids or not agent or agent == BROADCAST: + return + at = at or utcnow() + rows = [ + { + "message_id": mid, + "agent": agent, + "delivered_at": at if delivered else None, + "read_at": at if (read or acknowledged) else None, + "acknowledged_at": at if acknowledged else None, + } + for mid in ids + ] + stmt = pg_insert(MessageReceipt).values(rows) + table = MessageReceipt.__table__ + stmt = stmt.on_conflict_do_update( + index_elements=[table.c.message_id, table.c.agent], + set_={ + col: func.coalesce(table.c[col], stmt.excluded[col]) + for col in ("delivered_at", "read_at", "acknowledged_at") + }, + ) + await session.execute(stmt) + + +async def receipts_for( + session: AsyncSession, + message_ids: Iterable[uuid.UUID], + reader_values: Sequence[str], +) -> dict[uuid.UUID, dict[str, datetime | None]]: + """Merged receipt state per message across a reader's historical slugs.""" + ids = list(message_ids) + if not ids: + return {} + result = await session.execute( + select(MessageReceipt).where( + MessageReceipt.message_id.in_(ids), + MessageReceipt.agent.in_(reader_values), + ) + ) + merged: dict[uuid.UUID, dict[str, datetime | None]] = {} + for receipt in result.scalars().all(): + state = merged.setdefault( + receipt.message_id, + {"delivered_at": None, "read_at": None, "acknowledged_at": None}, + ) + for field in state: + value = getattr(receipt, field) + if value is not None and (state[field] is None or value < state[field]): + state[field] = value + return merged + + +async def _active_repo_slugs(session: AsyncSession) -> list[str]: + result = await session.execute( + select(ManagedRepo.slug).where(ManagedRepo.status == "active").order_by(ManagedRepo.slug) + ) + return list(result.scalars().all()) + + +async def live_standing_notices( + session: AsyncSession, now: datetime | None = None +) -> list[AgentMessage]: + now = now or utcnow() + result = await session.execute( + select(AgentMessage) + .where( + AgentMessage.to_agent == BROADCAST, + AgentMessage.kind == "standing", + AgentMessage.archived_at.is_(None), + not_expired(now), + ) + .order_by(AgentMessage.created_at.desc()) + ) + return list(result.scalars().all()) + + +async def notice_statuses(session: AsyncSession) -> list[NoticeStatus]: + notices = await live_standing_notices(session) + if not notices: + return [] + repos = await _active_repo_slugs(session) + result = await session.execute( + select(MessageReceipt).where( + MessageReceipt.message_id.in_([n.id for n in notices]) + ) + ) + by_notice: dict[uuid.UUID, list[MessageReceipt]] = {} + for receipt in result.scalars().all(): + by_notice.setdefault(receipt.message_id, []).append(receipt) + statuses: list[NoticeStatus] = [] + for notice in notices: + receipts = by_notice.get(notice.id, []) + acked = sorted({r.agent for r in receipts if r.acknowledged_at is not None}) + seen = {r.agent for r in receipts} + delivered_only = sorted(seen - set(acked)) + unreached = [slug for slug in repos if slug not in seen] + statuses.append( + NoticeStatus( + id=notice.id, + from_agent=notice.from_agent, + subject=notice.subject, + body=notice.body, + created_at=notice.created_at, + expires_at=notice.expires_at, + supersedes_id=notice.supersedes_id, + acknowledged=acked, + delivered_only=delivered_only, + unreached=unreached, + acked_count=len(acked), + delivered_only_count=len(delivered_only), + unreached_count=len(unreached), + ) + ) + return statuses + + +async def standing_notice_digests(session: AsyncSession) -> list[StandingNoticeDigest]: + statuses = await notice_statuses(session) + if not statuses: + return [] + repos = set(await _active_repo_slugs(session)) + digests = [] + for status in statuses: + acked_repos = len(repos.intersection(status.acknowledged)) + digests.append( + StandingNoticeDigest( + id=status.id, + subject=status.subject, + expires_at=status.expires_at, + acked=acked_repos, + pending=len(repos) - acked_repos, + ) + ) + return digests diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/ops_run_projection.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/ops_run_projection.py new file mode 100644 index 00000000..5632aa60 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/ops_run_projection.py @@ -0,0 +1,130 @@ +"""Read-only activity-core ops_run projection (STATE-WP-0078).""" + +from __future__ import annotations + +import asyncio +import time +from datetime import datetime, timedelta, timezone +from typing import Any + +import httpx + +from api.config import settings +from api.schemas.ops_run import OpsRunProjection, OpsRunProjectionItem + + +_cache: OpsRunProjection | None = None +_cache_at: float = 0.0 +_lock = asyncio.Lock() + + +def reset_ops_run_projection_cache() -> None: + global _cache, _cache_at + _cache = None + _cache_at = 0.0 + + +def _parse_datetime(value: Any) -> datetime | None: + if not value or not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +async def _fetch_ops_runs() -> dict[str, Any]: + if not settings.activity_core_url: + raise RuntimeError("ACTIVITY_CORE_URL is not configured") + headers = {} + if settings.activity_core_worker_token: + headers["X-Worker-Token"] = settings.activity_core_worker_token + async with httpx.AsyncClient( + base_url=settings.activity_core_url.rstrip("/"), + timeout=10.0, + follow_redirects=True, + trust_env=False, + headers=headers, + ) as client: + response = await client.get("/ops-runs", params={"limit": 200}) + response.raise_for_status() + return response.json() + + +def _project(payload: dict[str, Any], now: datetime) -> OpsRunProjection: + raw_items = payload.get("items") or [] + counts = payload.get("counts") or {} + failed_cutoff = now - timedelta(hours=24) + stuck_cutoff = now - timedelta(hours=max(0.1, settings.ops_run_sla_hours)) + failed_24h = 0 + stuck = 0 + items: list[OpsRunProjectionItem] = [] + + for raw in raw_items: + state = str(raw.get("state") or "unknown") + updated_at = _parse_datetime(raw.get("updated_at")) + created_at = _parse_datetime(raw.get("created_at")) + if state == "failed" and updated_at and updated_at >= failed_cutoff: + failed_24h += 1 + if state in {"open", "claimed"} and (created_at or updated_at): + if (created_at or updated_at) < stuck_cutoff: + stuck += 1 + + result = raw.get("result") if isinstance(raw.get("result"), dict) else {} + last_error = result.get("error") or raw.get("last_error") + if state in {"open", "claimed", "failed"}: + items.append(OpsRunProjectionItem( + id=str(raw.get("id")), + definition=str(raw.get("activity_definition_id")) if raw.get("activity_definition_id") else None, + target_repo=raw.get("target_repo"), + state=state, + lease={ + "owner": raw.get("claim_owner"), + "until": raw.get("lease_until"), + "attempt": raw.get("attempt", 0), + } if raw.get("claim_owner") or raw.get("lease_until") else None, + updated_at=updated_at, + last_error=str(last_error) if last_error else None, + )) + + items.sort(key=lambda item: item.updated_at or datetime.min.replace(tzinfo=timezone.utc), reverse=True) + return OpsRunProjection( + available=True, + generated_at=now, + open=int(counts.get("open", sum(item.state == "open" for item in items))), + claimed=int(counts.get("claimed", sum(item.state == "claimed" for item in items))), + failed_24h=failed_24h, + stuck_open_or_claimed=stuck, + items=items[:50], + ) + + +async def get_ops_run_projection(*, refresh: bool = False) -> OpsRunProjection: + global _cache, _cache_at + now_mono = time.monotonic() + ttl = max(1.0, settings.ops_run_projection_ttl_seconds) + if not refresh and _cache is not None and now_mono - _cache_at < ttl: + return _cache + + async with _lock: + now_mono = time.monotonic() + if not refresh and _cache is not None and now_mono - _cache_at < ttl: + return _cache + now = datetime.now(timezone.utc) + try: + projection = _project(await _fetch_ops_runs(), now) + except Exception as exc: + if _cache is not None and _cache.available: + projection = _cache.model_copy(update={ + "stale": True, + "error": f"activity-core refresh failed: {exc}", + }) + else: + projection = OpsRunProjection( + generated_at=now, + error=f"activity-core unavailable: {exc}", + ) + _cache = projection + _cache_at = now_mono + return projection diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/recently_on_scope.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/recently_on_scope.py new file mode 100644 index 00000000..359ec42e --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/recently_on_scope.py @@ -0,0 +1,632 @@ +from __future__ import annotations + +import os +import re +import uuid +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import yaml +from fastapi import HTTPException +from sqlalchemy import false, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from api.config import settings +from api.models.decision import Decision +from api.models.domain import Domain +from api.models.managed_repo import ManagedRepo +from api.models.progress_event import ProgressEvent +from api.models.task import Task, TaskStatus +from api.models.topic import Topic +from api.models.workplan import Workplan +from api.schemas.recently_on_scope import ( + RecentlyOnScopeFailedDomain, + RecentlyOnScopeHourlyRun, + RecentlyOnScopeReportMetadata, + RecentlyOnScopeSourceCounts, + RecentlyOnScopeSkippedDomain, +) +from api.services.markitect_templates import inspect_markdown_template, render_markdown_template + +_DURATION_RE = re.compile(r"^(?P[1-9][0-9]*)(?P[mhd])$") +_REPORT_ID_RE = re.compile(r"^[0-9]{8}T[0-9]{6}Z--(?:[0-9]+[mhd]|[0-9]{8}T[0-9]{6}Z)$") +_TEMPLATE_PATH = Path("templates/recently-on-scope/domain-digest.md") +_TEMPLATE_VERSION = "1" + + +@dataclass(frozen=True) +class DigestWindow: + range: str + since: datetime + until: datetime + exact: bool = False + + +def parse_duration(value: str) -> timedelta: + match = _DURATION_RE.fullmatch(value.strip()) + if match is None: + raise ValueError("range must use a duration such as 15m, 1h, 6h, or 1d") + count = int(match.group("count")) + unit = match.group("unit") + if unit == "m": + return timedelta(minutes=count) + if unit == "h": + return timedelta(hours=count) + return timedelta(days=count) + + +def resolve_window( + range_value: str = "1h", + since: datetime | None = None, + until: datetime | None = None, + *, + now: datetime | None = None, +) -> DigestWindow: + duration = parse_duration(range_value) + end = _as_utc(until or now or datetime.now(tz=UTC)) + start = _as_utc(since) if since is not None else end - duration + if start >= end: + raise ValueError("since must be earlier than until") + return DigestWindow(range=range_value, since=start, until=end, exact=since is not None or until is not None) + + +def report_root() -> Path: + root = Path(settings.state_hub_report_dir).expanduser() + if not root.is_absolute(): + root = Path.cwd() / root + return root + + +def template_path() -> Path: + path = _TEMPLATE_PATH + if not path.is_absolute(): + path = Path.cwd() / path + return path + + +def report_id_for(window: DigestWindow) -> str: + if window.exact: + return f"{_stamp(window.since)}--{_stamp(window.until)}" + return f"{_stamp(window.until)}--{window.range}" + + +def report_path_for(domain_slug: str, window: DigestWindow) -> Path: + return report_root() / domain_slug / f"{report_id_for(window)}.md" + + +async def collect_domain_activity( + session: AsyncSession, + domain_slug: str, + window: DigestWindow, +) -> dict[str, Any]: + domain = await _get_domain(domain_slug, session) + topics = await _list_topics(domain.id, session) + topic_ids = [topic.id for topic in topics] + workstreams = await _list_workstreams(topic_ids, session) + workstream_ids = [workstream.id for workstream in workstreams] + tasks = await _list_tasks(workstream_ids, session) + task_ids = [task.id for task in tasks] + decisions = await _list_recent_decisions(topic_ids, workstream_ids, window, session) + decision_ids = [decision.id for decision in decisions] + progress_events = await _list_recent_progress(topic_ids, workstream_ids, task_ids, decision_ids, window, session) + repos = await _list_repos(domain.id, session) + + recent_workstreams = [ + workstream for workstream in workstreams + if _in_window(workstream.created_at, window) or _in_window(workstream.updated_at, window) + ] + recent_tasks = [ + task for task in tasks + if _in_window(task.created_at, window) or _in_window(task.updated_at, window) + ] + attention_tasks = [ + task for task in tasks + if task.needs_human and _enum_value(task.status) not in {TaskStatus.done.value, TaskStatus.cancel.value} + ] + + data = { + "domain": _domain_data(domain), + "window": { + "range": window.range, + "since": _iso(window.since), + "until": _iso(window.until), + }, + "generated_at": _iso(datetime.now(tz=UTC)), + "template_version": _TEMPLATE_VERSION, + "source_counts": { + "progress_events": len(progress_events), + "decisions": len(decisions), + "workstreams": len(recent_workstreams), + "tasks": len(recent_tasks), + "repos": len(repos), + "attention_items": len(attention_tasks), + }, + "progress_events": [_progress_data(event) for event in progress_events], + "decisions": [_decision_data(decision) for decision in decisions], + "workstreams": [_workstream_data(workstream) for workstream in recent_workstreams], + "tasks": [_task_data(task) for task in recent_tasks], + "repos": [_repo_data(repo) for repo in repos], + "attention_items": [_attention_task_data(task) for task in attention_tasks], + } + data |= _section_text(data) + return data + + +async def generate_report( + session: AsyncSession, + domain_slug: str, + window: DigestWindow, +) -> tuple[RecentlyOnScopeReportMetadata, str]: + data = await collect_domain_activity(session, domain_slug, window) + return _render_report_from_data(data, window) + + +async def generate_hourly_reports( + session: AsyncSession, + window: DigestWindow, + *, + active_only: bool = True, + include_attention: bool = False, +) -> RecentlyOnScopeHourlyRun: + generated: list[RecentlyOnScopeReportMetadata] = [] + skipped: list[RecentlyOnScopeSkippedDomain] = [] + failed: list[RecentlyOnScopeFailedDomain] = [] + + domains = await _list_domains(session, active_only=active_only) + for domain in domains: + try: + data = await collect_domain_activity(session, domain.slug, window) + counts = RecentlyOnScopeSourceCounts(**data["source_counts"]) + if not _has_qualifying_activity(counts, include_attention=include_attention): + skipped.append( + RecentlyOnScopeSkippedDomain( + domain_slug=domain.slug, + reason="no qualifying activity in window", + source_counts=counts, + ) + ) + continue + metadata, _markdown = _render_report_from_data(data, window) + generated.append(metadata) + except Exception as exc: # pragma: no cover - exercised via router tests + failed.append(RecentlyOnScopeFailedDomain(domain_slug=domain.slug, error=str(exc))) + + generated_at = datetime.now(tz=UTC) + progress_event_id = await _log_hourly_progress( + session, + window, + generated_at=generated_at, + active_only=active_only, + include_attention=include_attention, + generated=generated, + skipped=skipped, + failed=failed, + ) + return RecentlyOnScopeHourlyRun( + range=window.range, + since=window.since, + until=window.until, + generated_at=generated_at, + active_only=active_only, + include_attention=include_attention, + generated=generated, + skipped=skipped, + failed=failed, + progress_event_id=progress_event_id, + ) + + +def _render_report_from_data( + data: dict[str, Any], + window: DigestWindow, +) -> tuple[RecentlyOnScopeReportMetadata, str]: + tmpl = template_path() + inspect_markdown_template(tmpl) + markdown = render_markdown_template(tmpl, data) + path = report_path_for(data["domain"]["slug"], window) + _write_report(path, markdown) + return _metadata_from_data(report_id_for(window), path, data, window), markdown + + +def list_reports(domain_slug: str) -> list[RecentlyOnScopeReportMetadata]: + directory = report_root() / domain_slug + if not directory.is_dir(): + return [] + reports = [] + for path in sorted(directory.glob("*.md"), reverse=True): + metadata = metadata_from_report(path) + if metadata is not None: + reports.append(metadata) + return reports + + +def read_report(domain_slug: str, report_id: str) -> str: + if _REPORT_ID_RE.fullmatch(report_id) is None: + raise FileNotFoundError(report_id) + path = report_root() / domain_slug / f"{report_id}.md" + if not path.is_file(): + raise FileNotFoundError(report_id) + return path.read_text(encoding="utf-8") + + +def metadata_from_report(path: Path) -> RecentlyOnScopeReportMetadata | None: + text = path.read_text(encoding="utf-8") + frontmatter = _frontmatter(text) + if not frontmatter: + return None + source_counts = frontmatter.get("source_counts") or {} + try: + return RecentlyOnScopeReportMetadata( + id=path.stem, + domain_slug=str(frontmatter["domain_slug"]), + range=str(frontmatter["range"]), + since=_as_utc(frontmatter["since"]), + until=_as_utc(frontmatter["until"]), + generated_at=_as_utc(frontmatter["generated_at"]), + path=str(path), + source_counts=RecentlyOnScopeSourceCounts(**source_counts), + ) + except (KeyError, TypeError, ValueError): + return None + + +async def _list_domains(session: AsyncSession, *, active_only: bool) -> list[Domain]: + stmt = select(Domain).order_by(Domain.slug) + if active_only: + stmt = stmt.where(Domain.status == "active") + result = await session.execute(stmt) + return list(result.scalars().all()) + + +def _has_qualifying_activity( + counts: RecentlyOnScopeSourceCounts, + *, + include_attention: bool, +) -> bool: + if ( + counts.progress_events + or counts.decisions + or counts.workstreams + or counts.tasks + ): + return True + return include_attention and counts.attention_items > 0 + + +async def _log_hourly_progress( + session: AsyncSession, + window: DigestWindow, + *, + generated_at: datetime, + active_only: bool, + include_attention: bool, + generated: list[RecentlyOnScopeReportMetadata], + skipped: list[RecentlyOnScopeSkippedDomain], + failed: list[RecentlyOnScopeFailedDomain], +) -> uuid.UUID: + event = ProgressEvent( + event_type="recently_on_scope_hourly", + summary=( + "RecentlyOnScope hourly batch completed: " + f"{len(generated)} generated, {len(skipped)} skipped, {len(failed)} failed" + ), + detail={ + "range": window.range, + "since": _iso(window.since), + "until": _iso(window.until), + "generated_at": _iso(generated_at), + "active_only": active_only, + "include_attention": include_attention, + "generated": [item.model_dump(mode="json") for item in generated], + "skipped": [item.model_dump(mode="json") for item in skipped], + "failed": [item.model_dump(mode="json") for item in failed], + }, + author="state-hub", + ) + session.add(event) + await session.commit() + await session.refresh(event) + return event.id + + +async def _get_domain(domain_slug: str, session: AsyncSession) -> Domain: + result = await session.execute(select(Domain).where(Domain.slug == domain_slug)) + domain = result.scalar_one_or_none() + if domain is None: + raise HTTPException(status_code=404, detail=f"Domain '{domain_slug}' not found") + return domain + + +async def _list_topics(domain_id: uuid.UUID, session: AsyncSession) -> list[Topic]: + result = await session.execute(select(Topic).where(Topic.domain_id == domain_id).order_by(Topic.slug)) + return list(result.scalars().all()) + + +async def _list_workstreams(topic_ids: list[uuid.UUID], session: AsyncSession) -> list[Workplan]: + result = await session.execute( + select(Workplan) + .where(_in(Workplan.topic_id, topic_ids)) + .order_by(Workplan.updated_at.desc(), Workplan.created_at.desc()) + ) + return list(result.scalars().all()) + + +async def _list_tasks(workstream_ids: list[uuid.UUID], session: AsyncSession) -> list[Task]: + result = await session.execute( + select(Task) + .where(_in(Task.workplan_id, workstream_ids)) + .order_by(Task.updated_at.desc(), Task.created_at.desc()) + ) + return list(result.scalars().all()) + + +async def _list_recent_decisions( + topic_ids: list[uuid.UUID], + workstream_ids: list[uuid.UUID], + window: DigestWindow, + session: AsyncSession, +) -> list[Decision]: + result = await session.execute( + select(Decision) + .where(or_(_in(Decision.topic_id, topic_ids), _in(Decision.workplan_id, workstream_ids))) + .where( + or_( + _between(Decision.created_at, window), + _between(Decision.updated_at, window), + _between(Decision.decided_at, window), + ) + ) + .order_by(Decision.updated_at.desc(), Decision.created_at.desc()) + ) + return list(result.scalars().all()) + + +async def _list_recent_progress( + topic_ids: list[uuid.UUID], + workstream_ids: list[uuid.UUID], + task_ids: list[uuid.UUID], + decision_ids: list[uuid.UUID], + window: DigestWindow, + session: AsyncSession, +) -> list[ProgressEvent]: + result = await session.execute( + select(ProgressEvent) + .where(_between(ProgressEvent.created_at, window)) + .where( + or_( + _in(ProgressEvent.topic_id, topic_ids), + _in(ProgressEvent.workplan_id, workstream_ids), + _in(ProgressEvent.task_id, task_ids), + _in(ProgressEvent.decision_id, decision_ids), + ) + ) + .order_by(ProgressEvent.created_at.desc()) + ) + return list(result.scalars().all()) + + +async def _list_repos(domain_id: uuid.UUID, session: AsyncSession) -> list[ManagedRepo]: + result = await session.execute( + select(ManagedRepo) + .where(ManagedRepo.domain_id == domain_id) + .where(ManagedRepo.status == "active") + .order_by(ManagedRepo.slug) + ) + return list(result.scalars().all()) + + +def _in(column, values: list[uuid.UUID]): + return column.in_(values) if values else false() + + +def _between(column, window: DigestWindow): + return column.is_not(None) & (column >= window.since) & (column <= window.until) + + +def _in_window(value: datetime | None, window: DigestWindow) -> bool: + if value is None: + return False + value = _as_utc(value) + return window.since <= value <= window.until + + +def _metadata_from_data( + report_id: str, + path: Path, + data: dict[str, Any], + window: DigestWindow, +) -> RecentlyOnScopeReportMetadata: + return RecentlyOnScopeReportMetadata( + id=report_id, + domain_slug=data["domain"]["slug"], + range=window.range, + since=window.since, + until=window.until, + generated_at=_as_utc(data["generated_at"]), + path=str(path), + source_counts=RecentlyOnScopeSourceCounts(**data["source_counts"]), + ) + + +def _write_report(path: Path, markdown: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".{path.name}.tmp") + tmp.write_text(markdown, encoding="utf-8") + os.replace(tmp, path) + + +def _frontmatter(text: str) -> dict[str, Any] | None: + if not text.startswith("---\n"): + return None + end = text.find("\n---", 4) + if end == -1: + return None + data = yaml.safe_load(text[4:end].strip()) or {} + return data if isinstance(data, dict) else None + + +def _section_text(data: dict[str, Any]) -> dict[str, str]: + return { + "progress_section": _progress_section(data["progress_events"]), + "decisions_section": _decisions_section(data["decisions"]), + "workstreams_section": _workstreams_section(data["workstreams"]), + "tasks_section": _tasks_section(data["tasks"]), + "attention_section": _attention_section(data["attention_items"]), + "repos_section": _repos_section(data["repos"]), + } + + +def _progress_section(items: list[dict[str, Any]]) -> str: + if not items: + return "_No progress events in this window._" + return "\n".join( + f"- `{item['created_at']}` `{item['event_type']}` {item['summary']}" + + (f" - {item['author']}" if item.get("author") else "") + for item in items + ) + + +def _decisions_section(items: list[dict[str, Any]]) -> str: + if not items: + return "_No decisions changed in this window._" + return "\n".join( + f"- `{item['updated_at']}` **{item['title']}** - {item['status']}" + + (f"; decided by {item['decided_by']}" if item.get("decided_by") else "") + for item in items + ) + + +def _workstreams_section(items: list[dict[str, Any]]) -> str: + if not items: + return "_No workstreams changed in this window._" + return "\n".join( + f"- `{item['updated_at']}` **{item['title']}** (`{item['slug']}`) - {item['status']}" + for item in items + ) + + +def _tasks_section(items: list[dict[str, Any]]) -> str: + if not items: + return "_No tasks changed in this window._" + return "\n".join( + f"- `{item['updated_at']}` **{item['title']}** - {item['status']} / {item['priority']}" + for item in items + ) + + +def _attention_section(items: list[dict[str, Any]]) -> str: + if not items: + return "_No open human-intervention items for this domain._" + return "\n".join( + f"- **{item['title']}** - {item.get('intervention_note') or item.get('blocking_reason') or 'needs human attention'}" + for item in items + ) + + +def _repos_section(items: list[dict[str, Any]]) -> str: + if not items: + return "_No active repositories are registered for this domain._" + return "\n".join( + f"- `{item['slug']}` - {item['name']}" + (f" ({item['remote_url']})" if item.get("remote_url") else "") + for item in items + ) + + +def _domain_data(domain: Domain) -> dict[str, Any]: + return { + "id": str(domain.id), + "slug": domain.slug, + "name": domain.name, + "description": domain.description or "", + } + + +def _progress_data(event: ProgressEvent) -> dict[str, Any]: + return { + "id": str(event.id), + "created_at": _iso(event.created_at), + "event_type": event.event_type, + "summary": event.summary, + "author": event.author, + "workplan_id": str(event.workplan_id) if event.workplan_id else None, + "workstream_id": str(event.workplan_id) if event.workplan_id else None, + "task_id": str(event.task_id) if event.task_id else None, + "decision_id": str(event.decision_id) if event.decision_id else None, + } + + +def _decision_data(decision: Decision) -> dict[str, Any]: + return { + "id": str(decision.id), + "title": decision.title, + "status": _enum_value(decision.status), + "decision_type": _enum_value(decision.decision_type), + "decided_by": decision.decided_by, + "decided_at": _iso(decision.decided_at) if decision.decided_at else "", + "created_at": _iso(decision.created_at), + "updated_at": _iso(decision.updated_at), + } + + +def _workstream_data(workstream: Workplan) -> dict[str, Any]: + return { + "id": str(workstream.id), + "slug": workstream.slug, + "title": workstream.title, + "status": workstream.status, + "owner": workstream.owner or "", + "created_at": _iso(workstream.created_at), + "updated_at": _iso(workstream.updated_at), + } + + +def _task_data(task: Task) -> dict[str, Any]: + return { + "id": str(task.id), + "workstream_id": str(task.workplan_id), + "title": task.title, + "status": _enum_value(task.status), + "priority": _enum_value(task.priority), + "assignee": task.assignee or "", + "blocking_reason": task.blocking_reason or "", + "created_at": _iso(task.created_at), + "updated_at": _iso(task.updated_at), + } + + +def _attention_task_data(task: Task) -> dict[str, Any]: + data = _task_data(task) + data["intervention_note"] = task.intervention_note or "" + return data + + +def _repo_data(repo: ManagedRepo) -> dict[str, Any]: + return { + "id": str(repo.id), + "slug": repo.slug, + "name": repo.name, + "remote_url": repo.remote_url or "", + "local_path": repo.local_path or "", + } + + +def _enum_value(value: Any) -> Any: + return getattr(value, "value", value) + + +def _as_utc(value: datetime | str) -> datetime: + if isinstance(value, str): + value = datetime.fromisoformat(value.replace("Z", "+00:00")) + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + + +def _iso(value: datetime) -> str: + return _as_utc(value).isoformat().replace("+00:00", "Z") + + +def _stamp(value: datetime) -> str: + return _as_utc(value).strftime("%Y%m%dT%H%M%SZ") diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/reconciliation.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/reconciliation.py new file mode 100644 index 00000000..af953e2b --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/reconciliation.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any + +from api.services.lifecycle import status_value +from api.task_status import CANONICAL_TASK_STATUSES +from api.workplan_status import CLOSED_WORKPLAN_STATUSES, normalize_workplan_status + + +class ReconciliationClass(str, Enum): + WRITE_THROUGH = "write_through" + DEFERRED = "deferred" + HUMAN_CONFIRMATION = "human_confirmation" + + +@dataclass(frozen=True) +class StateChangeClassification: + reconciliation_class: ReconciliationClass + reason: str + follow_up: str + + +WRITE_THROUGH_WORKPLAN_STATUSES = {"proposed", "ready", "active", "backlog"} +TASK_STATUSES = set(CANONICAL_TASK_STATUSES) + + +def classify_workplan_status_change( + *, + current_status: Any, + target_status: Any, + file_backed: bool, + archived_file: bool = False, + tasks_terminal: bool | None = None, +) -> StateChangeClassification: + """Classify a UI-originated workstream status transition.""" + current = normalize_workplan_status(current_status) + target = normalize_workplan_status(target_status) + + if not file_backed: + return StateChangeClassification( + ReconciliationClass.DEFERRED, + "workstream is not backed by a local workplan file", + "create a reconciliation record for operator follow-up", + ) + if archived_file: + return StateChangeClassification( + ReconciliationClass.HUMAN_CONFIRMATION, + "workplan file is archived", + "confirm whether to restore the file or leave the DB state unchanged", + ) + if current == target: + return StateChangeClassification( + ReconciliationClass.WRITE_THROUGH, + "status is unchanged", + "no file update required", + ) + if target in WRITE_THROUGH_WORKPLAN_STATUSES and current not in CLOSED_WORKPLAN_STATUSES: + return StateChangeClassification( + ReconciliationClass.WRITE_THROUGH, + "open lifecycle transition can be represented in workplan frontmatter", + "patch workplan status and sync the DB from file", + ) + if target == "finished": + if tasks_terminal is True: + return StateChangeClassification( + ReconciliationClass.WRITE_THROUGH, + "all known tasks are terminal", + "patch workplan status to finished and sync the DB from file", + ) + return StateChangeClassification( + ReconciliationClass.HUMAN_CONFIRMATION, + "finishing with non-terminal or unknown task state can hide open work", + "ask for confirmation or finish/cancel remaining tasks first", + ) + if target == "archived": + return StateChangeClassification( + ReconciliationClass.HUMAN_CONFIRMATION, + "archiving may move files and hide active context", + "ask for confirmation and run the archive workflow", + ) + if target == "blocked": + return StateChangeClassification( + ReconciliationClass.HUMAN_CONFIRMATION, + "blocked workstreams need an explicit blocker or dependency reason", + "capture the blocker before writing status", + ) + return StateChangeClassification( + ReconciliationClass.DEFERRED, + f"unsupported workstream status transition {current!r} -> {target!r}", + "create a reconciliation record for operator review", + ) + + +classify_workstream_status_change = classify_workplan_status_change + + +def classify_task_status_change( + *, + current_status: Any, + target_status: Any, + file_backed: bool, + task_linked: bool, + archived_file: bool = False, + blocking_reason: str | None = None, +) -> StateChangeClassification: + """Classify a UI-originated task status transition.""" + current = status_value(current_status) + target = status_value(target_status) + + if not file_backed: + return StateChangeClassification( + ReconciliationClass.DEFERRED, + "task belongs to a workstream without a local workplan file", + "create a reconciliation record for operator follow-up", + ) + if archived_file: + return StateChangeClassification( + ReconciliationClass.HUMAN_CONFIRMATION, + "task belongs to an archived workplan file", + "confirm whether to restore or edit the archived file", + ) + if not task_linked: + return StateChangeClassification( + ReconciliationClass.DEFERRED, + "task is not linked to a task block in the workplan file", + "create or repair the task block link before writing status", + ) + if current == target: + return StateChangeClassification( + ReconciliationClass.WRITE_THROUGH, + "status is unchanged", + "no file update required", + ) + if target == "wait" and not (blocking_reason or "").strip(): + return StateChangeClassification( + ReconciliationClass.HUMAN_CONFIRMATION, + "waiting tasks should explain the wait condition", + "capture the wait reason before writing status", + ) + if target in TASK_STATUSES: + return StateChangeClassification( + ReconciliationClass.WRITE_THROUGH, + "task status can be represented in the workplan task block", + "patch task block status and sync the DB from file", + ) + return StateChangeClassification( + ReconciliationClass.DEFERRED, + f"unsupported task status transition {current!r} -> {target!r}", + "create a reconciliation record for operator review", + ) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repo_manager_dual_run.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repo_manager_dual_run.py new file mode 100644 index 00000000..4cc98f27 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repo_manager_dual_run.py @@ -0,0 +1,544 @@ +"""State Hub dual-run adapter → Repo Manager (RMGR-WP-0002/0003 Stage B). + +Config file (same as repo-manager): ~/.repo-manager/dual-run.yaml +or RM_DUAL_RUN_CONFIG. Env overrides when set: + + RM_WRITEBACK, RM_RECONCILE, RM_WRITEBACK_PUSH, RM_PILOT_REPOS, RM_METER_PATH + +Rollback: disable in config / unset env → native State Hub path. +""" + +from __future__ import annotations + +import json +import logging +import os +import subprocess +import sys +import uuid +from functools import lru_cache +from pathlib import Path +from typing import Any + +import yaml + +logger = logging.getLogger(__name__) + +_TRUE = frozenset({"1", "true", "yes", "on"}) +_FALSE = frozenset({"0", "false", "no", "off"}) + + +def config_path() -> Path: + raw = os.environ.get("RM_DUAL_RUN_CONFIG", "~/.repo-manager/dual-run.yaml") + return Path(raw).expanduser() + + +@lru_cache(maxsize=1) +def _load_file_config() -> dict[str, Any]: + path = config_path() + if not path.is_file(): + return {} + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError): + return {} + return data if isinstance(data, dict) else {} + + +def reload_config() -> None: + _load_file_config.cache_clear() + + +def _env_bool(name: str) -> bool | None: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return None + v = raw.strip().lower() + if v in _TRUE: + return True + if v in _FALSE: + return False + return None + + +def _cfg_bool(key: str, env_name: str, default: bool = False) -> bool: + ev = _env_bool(env_name) + if ev is not None: + return ev + cfg = _load_file_config() + if key in cfg: + val = cfg[key] + if isinstance(val, bool): + return val + if isinstance(val, str): + return val.strip().lower() in _TRUE + return default + + +def writeback_enabled() -> bool: + return _cfg_bool("writeback", "RM_WRITEBACK", False) + + +def reconcile_enabled() -> bool: + return _cfg_bool("reconcile", "RM_RECONCILE", False) + + +def writeback_push_enabled() -> bool: + return _cfg_bool("writeback_push", "RM_WRITEBACK_PUSH", False) + + +def pilot_slugs() -> set[str] | None: + raw = os.environ.get("RM_PILOT_REPOS") + if raw is not None: + raw = raw.strip() + if not raw: + return set() + return {s.strip() for s in raw.split(",") if s.strip()} + cfg = _load_file_config() + if "pilot_repos" not in cfg: + return None + val = cfg["pilot_repos"] + if val is None: + return None + if isinstance(val, str): + if not val.strip(): + return set() + return {s.strip() for s in val.split(",") if s.strip()} + if isinstance(val, list): + return {str(s).strip() for s in val if str(s).strip()} + return None + + +def slug_allowed(slug: str | None) -> bool: + if not slug: + return False + pilots = pilot_slugs() + if pilots is None: + return True + return slug in pilots + + +def writeback_for_repo(slug: str | None) -> bool: + return writeback_enabled() and slug_allowed(slug) + + +def reconcile_for_repo(slug: str | None) -> bool: + return reconcile_enabled() and slug_allowed(slug) + + +def meter_path() -> Path: + raw = os.environ.get("RM_METER_PATH") + if raw: + return Path(raw).expanduser() + cfg = _load_file_config() + if cfg.get("meter_path"): + return Path(str(cfg["meter_path"])).expanduser() + return Path("~/.repo-manager/mutation-meter.jsonl").expanduser() + + +def record_mutation( + *, + source: str, + kind: str, + repo_slug: str | None, + detail: dict[str, Any] | None = None, +) -> None: + try: + from datetime import UTC, datetime + + path = meter_path() + path.parent.mkdir(parents=True, exist_ok=True) + row = { + "ts": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "source": source, + "kind": kind, + "repo_slug": repo_slug, + "detail": detail or {}, + } + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(row, default=str) + "\n") + except OSError: + pass + + +def run_rmgr(args: list[str], *, timeout: int = 120) -> tuple[int, str, str]: + env = os.environ.copy() + custom = os.environ.get("RMGR_BIN", "").strip() + if custom: + cmd = custom.split() + args + else: + src = os.environ.get("REPO_MANAGER_SRC", "").strip() + sibling = Path(src) if src else (Path.home() / "repo-manager" / "src") + if not sibling.is_dir() and (Path.home() / "repo-manager" / "src").is_dir(): + sibling = Path.home() / "repo-manager" / "src" + if sibling.is_dir(): + env["PYTHONPATH"] = str(sibling) + ( + os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "" + ) + cmd = [ + sys.executable, + "-c", + "from repo_manager.cli import main; import sys; raise SystemExit(main(sys.argv[1:]))", + *args, + ] + else: + cmd = ["rmgr", *args] + + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + check=False, + env=env, + ) + return proc.returncode, proc.stdout, proc.stderr + + +def rm_update_task_status( + *, + repo_path: str | Path, + task_id: str, + status: str, + repo_slug: str | None = None, + reason: str = "state-hub dual-run", + push: bool | None = None, + correlation_id: str | None = None, +) -> dict[str, Any]: + correlation_id = correlation_id or str(uuid.uuid4()) + if push is None: + push = writeback_push_enabled() + args = [ + "update-task-status", + "--path", + str(repo_path), + "--task-id", + str(task_id), + "--status", + status, + "--reason", + reason, + "--correlation-id", + correlation_id, + "--idempotency-key", + f"sh-dual-{task_id}-{status}-{correlation_id}", + ] + if repo_slug: + args.extend(["--slug", repo_slug]) + if push: + args.append("--push") + + code, out, err = run_rmgr(args) + try: + result = json.loads(out.strip() or "{}") + except json.JSONDecodeError: + result = { + "status": "failed", + "error": { + "code": "internal", + "message": f"rmgr non-json exit={code} stderr={err!r} stdout={out[:500]!r}", + }, + "correlation_id": correlation_id, + } + if code != 0 and result.get("status") not in ("applied", "rejected"): + result.setdefault("status", "failed") + result.setdefault( + "error", + {"code": "internal", "message": f"rmgr exit={code} stderr={err!r}"}, + ) + return result + + +def rm_update_workplan( + *, + repo_path: str | Path, + workplan_id: str, + operation: str = "update", + title: str | None = None, + goal: str | None = None, + status: str | None = None, + owner: str | None = None, + domain: str | None = None, + topic_slug: str | None = None, + repo_slug: str | None = None, + reason: str = "state-hub dual-run", + push: bool | None = None, + correlation_id: str | None = None, + confirm_archive: bool = False, +) -> dict[str, Any]: + """Delegate a file-backed workplan mutation to Repo Manager. + + ``operation=archive`` is the recoverable counterpart of State Hub's + DELETE workplan route; it never erases repository history. + """ + correlation_id = correlation_id or str(uuid.uuid4()) + if operation not in {"create", "update", "archive"}: + return { + "status": "rejected", + "correlation_id": correlation_id, + "error": {"code": "validation_error", "message": f"invalid operation {operation!r}"}, + } + if push is None: + push = writeback_push_enabled() + cli_operation = "delete" if operation == "archive" else operation + args = [ + "workplan", + cli_operation, + "--path", + str(repo_path), + "--workplan-id", + str(workplan_id), + "--reason", + reason, + "--correlation-id", + correlation_id, + "--idempotency-key", + f"sh-dual-workplan-{operation}-{workplan_id}-{correlation_id}", + ] + for flag, value in ( + ("--title", title), + ("--goal", goal), + ("--status", status), + ("--owner", owner), + ("--domain", domain), + ("--topic-slug", topic_slug), + ("--slug", repo_slug), + ): + if value is not None: + args.extend([flag, value]) + if operation == "archive" and confirm_archive: + args.append("--confirm") + if push: + args.append("--push") + + code, out, err = run_rmgr(args) + try: + result = json.loads(out.strip() or "{}") + except json.JSONDecodeError: + result = { + "status": "failed", + "error": { + "code": "internal", + "message": f"rmgr non-json exit={code} stderr={err!r} stdout={out[:500]!r}", + }, + "correlation_id": correlation_id, + } + if code != 0 and result.get("status") not in ("applied", "rejected"): + result.setdefault("status", "failed") + result.setdefault( + "error", + {"code": "internal", "message": f"rmgr exit={code} stderr={err!r}"}, + ) + return result + + +def rm_update_register_entry( + *, + repo_path: str | Path, + kind: str, + entry_id: str, + operation: str = "put", + title: str | None = None, + status: str | None = None, + data: dict[str, Any] | None = None, + note: str | None = None, + author: str | None = None, + repo_slug: str | None = None, + reason: str = "state-hub retirement adapter", + push: bool | None = None, + correlation_id: str | None = None, +) -> dict[str, Any]: + """Delegate register create/update/defer/note to Repo Manager.""" + correlation_id = correlation_id or str(uuid.uuid4()) + if operation not in {"put", "defer", "note"}: + return { + "status": "rejected", + "correlation_id": correlation_id, + "error": {"code": "validation_error", "message": f"invalid operation {operation!r}"}, + } + if push is None: + push = writeback_push_enabled() + args = [ + "register", + operation, + "--path", + str(repo_path), + "--kind", + kind, + "--entry-id", + entry_id, + "--reason", + reason, + "--correlation-id", + correlation_id, + "--idempotency-key", + f"sh-dual-register-{operation}-{kind}-{entry_id}-{correlation_id}", + ] + for flag, value in ( + ("--title", title), + ("--status", status), + ("--note", note), + ("--author", author), + ("--slug", repo_slug), + ): + if value is not None: + args.extend([flag, value]) + if data is not None: + args.extend(["--data-json", json.dumps(data, separators=(",", ":"))]) + if push: + args.append("--push") + + code, out, err = run_rmgr(args) + try: + result = json.loads(out.strip() or "{}") + except json.JSONDecodeError: + result = { + "status": "failed", + "error": { + "code": "internal", + "message": f"rmgr non-json exit={code} stderr={err!r} stdout={out[:500]!r}", + }, + "correlation_id": correlation_id, + } + if code != 0 and result.get("status") not in ("applied", "rejected"): + result.setdefault("status", "failed") + result.setdefault( + "error", + {"code": "internal", "message": f"rmgr exit={code} stderr={err!r}"}, + ) + return result + + +def rm_scan_sbom( + *, + repo_path: str | Path, + repo_slug: str | None = None, +) -> dict[str, Any]: + """Derive a versioned SBOM snapshot from repository-owned sources.""" + args = ["sbom", "scan", "--path", str(repo_path)] + if repo_slug: + args.extend(["--slug", repo_slug]) + + code, out, err = run_rmgr(args) + try: + result = json.loads(out.strip() or "{}") + except json.JSONDecodeError: + result = { + "ok": False, + "error": f"rmgr non-json exit={code} stderr={err!r} stdout={out[:500]!r}", + } + if code != 0: + result.setdefault("ok", False) + result.setdefault("error", f"rmgr exit={code} stderr={err!r}") + result["exit_code"] = code + return result + + +def rm_scaffold( + *, + repo_path: str | Path, + flavor: str, + wp_prefix: str | None = None, + slug: str | None = None, + domain: str = "infotech", + force: bool = False, + commit: bool = False, +) -> dict[str, Any]: + """Delegate repository scaffolding to ``rmgr scaffold`` (STATE-WP-0080-T02).""" + args = [ + "scaffold", + "--path", + str(repo_path), + "--flavor", + flavor, + "--domain", + domain, + ] + if slug: + args.extend(["--slug", slug]) + if wp_prefix: + args.extend(["--wp-prefix", wp_prefix]) + if force: + args.append("--force") + if not commit: + args.append("--no-commit") + code, out, err = run_rmgr(args) + try: + result = json.loads(out.strip() or "{}") + except json.JSONDecodeError: + result = { + "status": "failed", + "error": { + "code": "internal", + "message": f"rmgr non-json exit={code} stderr={err!r} stdout={out[:500]!r}", + }, + } + if code != 0 and result.get("status") not in ("applied", "rejected"): + result.setdefault("status", "failed") + result.setdefault( + "error", + {"code": "internal", "message": f"rmgr exit={code} stderr={err!r}"}, + ) + result["exit_code"] = code + return result + + +def rm_reconcile(*, repo_path: str | Path, repo_slug: str | None = None) -> dict[str, Any]: + args = ["reconcile", "--path", str(repo_path)] + if repo_slug: + args.extend(["--slug", repo_slug]) + code, out, err = run_rmgr(args) + try: + result = json.loads(out.strip() or "{}") + except json.JSONDecodeError: + result = {"ok": False, "error": err or out, "exit_code": code} + result["exit_code"] = code + return result + + +def try_writeback_for_task( + *, + repo_path: str | Path | None, + repo_slug: str | None, + task_id: str, + status: str, + reason: str = "state-hub dual-run PATCH /tasks", +) -> dict[str, Any] | None: + if not writeback_for_repo(repo_slug): + return None + if not repo_path: + logger.warning("RM dual-run writeback skipped: no repo_path for %s", repo_slug) + return None + path = Path(repo_path) + if not path.is_dir(): + logger.warning("RM dual-run writeback skipped: missing path %s", path) + return None + + result = rm_update_task_status( + repo_path=path, + task_id=task_id, + status=status, + repo_slug=repo_slug, + reason=reason, + push=writeback_push_enabled(), + ) + if result.get("status") == "applied": + record_mutation( + source="repo-manager", + kind="task_status_writeback_via_sh_facade", + repo_slug=repo_slug, + detail={ + "task_id": task_id, + "status": status, + "git_sha": (result.get("evidence") or {}).get("git_sha"), + "push_ok": (result.get("evidence") or {}).get("push_ok"), + }, + ) + else: + logger.warning( + "RM dual-run writeback failed for %s task=%s: %s", + repo_slug, + task_id, + result.get("error") or result, + ) + return result diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repository_aliases.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repository_aliases.py new file mode 100644 index 00000000..a0e6e2d3 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repository_aliases.py @@ -0,0 +1,229 @@ +"""Canonical repository identity resolution across current and prior slugs. + +The slug registry is the lookup boundary. Historical records deliberately keep +the slug they recorded; callers use ``slug_values`` when they need an identity- +wide read and ``canonical_slug`` when they create a new reference. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass + +from fastapi import HTTPException +from sqlalchemy import func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import noload + +from api.models.fabric_graph import FabricGraphEdge, FabricGraphImport, FabricGraphNode +from api.models.managed_repo import ManagedRepo +from api.models.repository_rename import RepositorySlug + + +@dataclass(frozen=True) +class RepositorySlugResolution: + repo: ManagedRepo + requested_slug: str + canonical_slug: str + slug_status: str + aliases: tuple[str, ...] + source_operation_id: uuid.UUID | None = None + + @property + def slug_values(self) -> tuple[str, ...]: + return (self.canonical_slug, *self.aliases) + + +async def resolve_repository_slug( + session: AsyncSession, + slug: str, + *, + required: bool = True, +) -> RepositorySlugResolution | None: + """Resolve a current or protected prior slug to one repository UUID.""" + + record = ( + await session.execute( + select(RepositorySlug).where(RepositorySlug.slug == slug) + ) + ).scalar_one_or_none() + + # Compatibility for databases upgraded before the identity backfill has run. + if record is None: + repo = ( + await session.execute(select(ManagedRepo).where(ManagedRepo.slug == slug)) + ).scalar_one_or_none() + if repo is None: + if required: + raise HTTPException(status_code=404, detail=f"Repo '{slug}' not found") + return None + return RepositorySlugResolution( + repo=repo, + requested_slug=slug, + canonical_slug=repo.slug, + slug_status="canonical", + aliases=(), + ) + + repo = await session.get(ManagedRepo, record.repo_id) + if repo is None: + raise HTTPException( + status_code=409, + detail=f"Repository slug registry entry '{slug}' has no repository", + ) + records = list( + ( + await session.execute( + select(RepositorySlug) + .where(RepositorySlug.repo_id == repo.id) + .order_by(RepositorySlug.kind.desc(), RepositorySlug.slug) + ) + ).scalars() + ) + canonicals = [item for item in records if item.kind == "canonical"] + if len(canonicals) != 1 or canonicals[0].slug != repo.slug: + raise HTTPException( + status_code=409, + detail=f"Repository '{repo.id}' has inconsistent canonical slug state", + ) + return RepositorySlugResolution( + repo=repo, + requested_slug=slug, + canonical_slug=repo.slug, + slug_status=record.kind, + aliases=tuple(item.slug for item in records if item.kind == "alias"), + source_operation_id=record.source_operation_id, + ) + + +async def repository_resolutions_for_repositories( + session: AsyncSession, + repositories: list[ManagedRepo], +) -> dict[uuid.UUID, RepositorySlugResolution]: + """Resolve a repository collection with one slug-registry query. + + Collection callers already hold canonical ``ManagedRepo`` rows. Resolving + every row through :func:`resolve_repository_slug` would issue two identity + queries per repository. Load the complete alias sets once instead while + retaining the pre-registry compatibility behavior. + """ + + if not repositories: + return {} + + repository_ids = [repository.id for repository in repositories] + records = list( + ( + await session.execute( + select(RepositorySlug) + .options( + noload(RepositorySlug.repo), + noload(RepositorySlug.source_operation), + ) + .where(RepositorySlug.repo_id.in_(repository_ids)) + .order_by(RepositorySlug.repo_id, RepositorySlug.kind.desc(), RepositorySlug.slug) + ) + ).scalars() + ) + records_by_repository: dict[uuid.UUID, list[RepositorySlug]] = { + repository_id: [] for repository_id in repository_ids + } + for record in records: + records_by_repository[record.repo_id].append(record) + + resolutions: dict[uuid.UUID, RepositorySlugResolution] = {} + for repository in repositories: + repository_records = records_by_repository[repository.id] + requested_record = next( + (record for record in repository_records if record.slug == repository.slug), + None, + ) + if requested_record is None: + # Compatibility for databases upgraded before the identity backfill. + resolutions[repository.id] = RepositorySlugResolution( + repo=repository, + requested_slug=repository.slug, + canonical_slug=repository.slug, + slug_status="canonical", + aliases=(), + ) + continue + + canonicals = [record for record in repository_records if record.kind == "canonical"] + if len(canonicals) != 1 or canonicals[0].slug != repository.slug: + raise HTTPException( + status_code=409, + detail=f"Repository '{repository.id}' has inconsistent canonical slug state", + ) + resolutions[repository.id] = RepositorySlugResolution( + repo=repository, + requested_slug=repository.slug, + canonical_slug=repository.slug, + slug_status=requested_record.kind, + aliases=tuple( + record.slug for record in repository_records if record.kind == "alias" + ), + source_operation_id=requested_record.source_operation_id, + ) + + return resolutions + + +async def canonicalize_repository_slug(session: AsyncSession, value: str) -> str: + """Canonicalize a value only when it is a registered repository identity.""" + + resolution = await resolve_repository_slug(session, value, required=False) + return resolution.canonical_slug if resolution is not None else value + + +async def repository_resolution_for_id( + session: AsyncSession, repo: ManagedRepo +) -> RepositorySlugResolution: + return await resolve_repository_slug(session, repo.slug) # type: ignore[return-value] + + +async def stale_external_references( + session: AsyncSession, + resolution: RepositorySlugResolution, +) -> list[dict[str, object]]: + """Name external projections that still contain an historical slug. + + These are handoffs to the projection owner, never implicit rewrite targets. + """ + + if not resolution.aliases: + return [] + aliases = list(resolution.aliases) + checks = ( + (FabricGraphImport, "source_repo_slug"), + (FabricGraphNode, "source_repo_slug"), + (FabricGraphNode, "repo_slug"), + (FabricGraphEdge, "source_repo_slug"), + ) + stale: list[dict[str, object]] = [] + for model, field_name in checks: + field = getattr(model, field_name) + rows = ( + await session.execute( + select(field, func.count()).where(field.in_(aliases)).group_by(field) + ) + ).all() + for value, count in rows: + stale.append( + { + "owner": "railiance-fabric", + "surface": model.__tablename__, + "field": field_name, + "value": value, + "count": count, + "status": "stale", + "handoff": "owner update or re-ingest required", + } + ) + return stale + + +def affected_slug_predicate(field: object, slugs: tuple[str, ...]): + """Match a JSONB slug-list against any name in one repository lineage.""" + + return or_(*(field.contains([slug]) for slug in slugs)) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repository_identity.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repository_identity.py new file mode 100644 index 00000000..3eb2e092 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repository_identity.py @@ -0,0 +1,23 @@ +from sqlalchemy.ext.asyncio import AsyncSession + +from api.models.managed_repo import ManagedRepo +from api.models.repository_rename import RepositoryForgeIdentity, RepositorySlug + + +def stage_initial_repository_identity( + session: AsyncSession, repo: ManagedRepo +) -> None: + """Attach non-guessed identity state to a newly flushed repository row.""" + if repo.id is None: + raise ValueError("repository must be flushed before identity records are staged") + session.add_all( + [ + RepositoryForgeIdentity(repo_id=repo.id), + RepositorySlug( + repo_id=repo.id, + slug=repo.slug, + kind="canonical", + protected=True, + ), + ] + ) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repository_rename.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repository_rename.py new file mode 100644 index 00000000..21c4f56e --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/repository_rename.py @@ -0,0 +1,1845 @@ +"""Fail-closed, UUID-addressed repository rename lifecycle (STATE-WP-0085-T03).""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import uuid +from copy import deepcopy +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Any +from urllib.parse import quote + +from sqlalchemy import or_, select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from api.config import settings +from api.models.agent_message import AgentMessage +from api.models.capability_catalog import CapabilityCatalog +from api.models.decision import Decision +from api.models.fabric_graph import FabricGraphNode +from api.models.interface_change import InterfaceChange +from api.models.managed_repo import ManagedRepo +from api.models.progress_event import ProgressEvent +from api.models.repository_rename import ( + RepositoryForgeIdentity, + RepositoryRenameOperation, + RepositorySlug, +) +from api.models.sbom_entry import SBOMEntry +from api.models.sbom_snapshot import SBOMSnapshot +from api.models.service_catalog import ServiceFirstParty +from api.models.task import Task +from api.models.token_event import TokenEvent +from api.models.workplan import Workplan +from api.schemas.repository_rename import ( + ForgeIdentityVerifyRequest, + RepositoryRenameOperationCreate, + RepositoryRenamePhaseApply, + RepositoryRenamePreflightRequest, +) +from api.services.forge_repository import ( + ForgeRepositoryConflict, + ForgeRepositoryGateway, + ForgeRepositoryRedirected, + ForgeRepositorySnapshot, + ForgeRepositoryError, + ForgeRepositoryUnreadable, +) + + +ACTIVE_OPERATION_PHASES = { + "draft", + "preflighted", + "forge-renamed", + "statehub-rebound", + "source-synced", + "consumers-verified", + "rollback-preflight", +} +FORWARD_PHASES = [ + "preflighted", + "forge-renamed", + "statehub-rebound", + "source-synced", + "consumers-verified", + "completed", +] + + +class RenameLifecycleError(RuntimeError): + status_code = 409 + code = "repository_rename_conflict" + + def __init__(self, message: str, *, details: dict[str, Any] | None = None): + super().__init__(message) + self.details = details or {} + + +class RenameNotFound(RenameLifecycleError): + status_code = 404 + code = "repository_rename_not_found" + + +class RenamePreconditionFailed(RenameLifecycleError): + status_code = 412 + code = "repository_rename_precondition_failed" + + +class RenameServiceUnavailable(RenameLifecycleError): + status_code = 503 + code = "repository_rename_service_unavailable" + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def _json_default(value: Any) -> str: + if isinstance(value, datetime): + return value.astimezone(timezone.utc).isoformat() + if isinstance(value, uuid.UUID): + return str(value) + if hasattr(value, "value"): + return str(value.value) + return str(value) + + +def canonical_json(value: Any) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + default=_json_default, + ).encode("utf-8") + + +def checksum(value: Any) -> str: + return hashlib.sha256(canonical_json(value)).hexdigest() + + +def _b64encode(value: bytes) -> str: + return base64.urlsafe_b64encode(value).rstrip(b"=").decode("ascii") + + +def _b64decode(value: str) -> bytes: + return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + + +def _sign_preflight(payload: dict[str, Any]) -> str: + secret = settings.repository_rename_preflight_secret + if not secret: + raise RenameServiceUnavailable( + "Repository rename preflight signing is not configured" + ) + encoded = _b64encode(canonical_json(payload)) + signature = hmac.new(secret.encode(), encoded.encode(), hashlib.sha256).digest() + return f"{encoded}.{_b64encode(signature)}" + + +def _verify_preflight_token(token: str) -> dict[str, Any]: + secret = settings.repository_rename_preflight_secret + if not secret: + raise RenameServiceUnavailable( + "Repository rename preflight signing is not configured" + ) + try: + encoded, supplied = token.split(".", 1) + expected = hmac.new(secret.encode(), encoded.encode(), hashlib.sha256).digest() + if not hmac.compare_digest(expected, _b64decode(supplied)): + raise ValueError("signature") + payload = json.loads(_b64decode(encoded)) + expires_at = datetime.fromisoformat(payload["expires_at"]) + except (ValueError, KeyError, json.JSONDecodeError) as exc: + raise RenamePreconditionFailed("Invalid repository rename preflight token") from exc + if expires_at <= utcnow(): + raise RenamePreconditionFailed("Repository rename preflight token has expired") + return payload + + +def confirmation_for(repo_id: uuid.UUID, old_slug: str, new_slug: str) -> str: + return f"rename:{repo_id}:{old_slug}:{new_slug}" + + +def rollback_confirmation_for(operation_id: uuid.UUID) -> str: + return f"rollback:{operation_id}" + + +async def _id_rows(session: AsyncSession, query) -> list[str]: + result = await session.execute(query) + return sorted(str(value) for value in result.scalars().all()) + + +def _record_summary(ids: list[str]) -> dict[str, Any]: + return {"count": len(ids), "ids": ids, "checksum": checksum(ids)} + + +async def _relationship_rows( + session: AsyncSession, query, fields: tuple[str, ...] +) -> list[dict[str, Any]]: + def normalize(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, datetime): + return value.astimezone(timezone.utc).isoformat() + if isinstance(value, uuid.UUID): + return str(value) + if hasattr(value, "value"): + return value.value + return str(value) + + result = await session.execute(query) + rows = [ + {field: normalize(getattr(row, field)) for field in fields} + for row in result.all() + ] + return sorted(rows, key=lambda row: canonical_json(row)) + + +def _relationship_summary( + rows: list[dict[str, Any]], *, totals: dict[str, int] | None = None +) -> dict[str, Any]: + ids = sorted(row["id"] for row in rows) + result = { + "count": len(rows), + "ids": ids, + "checksum": checksum(ids), + "relationships": rows, + "relationship_checksum": checksum(rows), + } + if totals is not None: + result["totals"] = totals + result["totals_checksum"] = checksum(totals) + return result + + +async def collect_continuity_baseline( + session: AsyncSession, repo_id: uuid.UUID +) -> dict[str, Any]: + workplans = await _relationship_rows( + session, + select( + Workplan.id.label("id"), + Workplan.repo_id.label("repo_id"), + Workplan.slug.label("slug"), + ).where(Workplan.repo_id == repo_id), + ("id", "repo_id", "slug"), + ) + workplan_uuids = [uuid.UUID(row["id"]) for row in workplans] + tasks = await _relationship_rows( + session, + select( + Task.id.label("id"), + Task.workplan_id.label("workplan_id"), + Task.record_id.label("record_id"), + ).where( + Task.workplan_id.in_(workplan_uuids) if workplan_uuids else False + ), + ("id", "workplan_id", "record_id"), + ) + task_uuids = [uuid.UUID(row["id"]) for row in tasks] + progress = await _relationship_rows( + session, + select( + ProgressEvent.id.label("id"), + ProgressEvent.workplan_id.label("workplan_id"), + ProgressEvent.task_id.label("task_id"), + ProgressEvent.decision_id.label("decision_id"), + ).where( + or_( + ProgressEvent.workplan_id.in_(workplan_uuids) + if workplan_uuids + else False, + ProgressEvent.task_id.in_(task_uuids) if task_uuids else False, + ) + ), + ("id", "workplan_id", "task_id", "decision_id"), + ) + decisions = await _relationship_rows( + session, + select( + Decision.id.label("id"), + Decision.workplan_id.label("workplan_id"), + ).where( + Decision.workplan_id.in_(workplan_uuids) if workplan_uuids else False + ), + ("id", "workplan_id"), + ) + token_events = await _relationship_rows( + session, + select( + TokenEvent.id.label("id"), + TokenEvent.repo_id.label("repo_id"), + TokenEvent.workplan_id.label("workplan_id"), + TokenEvent.task_id.label("task_id"), + TokenEvent.tokens_in.label("tokens_in"), + TokenEvent.tokens_out.label("tokens_out"), + TokenEvent.cached_input_tokens.label("cached_input_tokens"), + TokenEvent.reasoning_output_tokens.label("reasoning_output_tokens"), + TokenEvent.raw_total_tokens.label("raw_total_tokens"), + ).where( + or_( + TokenEvent.repo_id == repo_id, + TokenEvent.workplan_id.in_(workplan_uuids) + if workplan_uuids + else False, + TokenEvent.task_id.in_(task_uuids) if task_uuids else False, + ) + ), + ( + "id", + "repo_id", + "workplan_id", + "task_id", + "tokens_in", + "tokens_out", + "cached_input_tokens", + "reasoning_output_tokens", + "raw_total_tokens", + ), + ) + token_totals = { + field: sum(int(row[field]) for row in token_events if row[field] is not None) + for field in ( + "tokens_in", + "tokens_out", + "cached_input_tokens", + "reasoning_output_tokens", + "raw_total_tokens", + ) + } + sbom_snapshots = await _relationship_rows( + session, + select( + SBOMSnapshot.id.label("id"), + SBOMSnapshot.repo_id.label("repo_id"), + ).where(SBOMSnapshot.repo_id == repo_id), + ("id", "repo_id"), + ) + sbom_entries = await _relationship_rows( + session, + select( + SBOMEntry.id.label("id"), + SBOMEntry.repo_id.label("repo_id"), + SBOMEntry.snapshot_id.label("snapshot_id"), + ).where(SBOMEntry.repo_id == repo_id), + ("id", "repo_id", "snapshot_id"), + ) + services = await _relationship_rows( + session, + select( + ServiceFirstParty.service_id.label("id"), + ServiceFirstParty.repo_id.label("repo_id"), + ).where(ServiceFirstParty.repo_id == repo_id), + ("id", "repo_id"), + ) + capabilities = await _relationship_rows( + session, + select( + CapabilityCatalog.id.label("id"), + CapabilityCatalog.repo_id.label("repo_id"), + ).where(CapabilityCatalog.repo_id == repo_id), + ("id", "repo_id"), + ) + interface_changes = await _relationship_rows( + session, + select( + InterfaceChange.id.label("id"), + InterfaceChange.repo_id.label("repo_id"), + ).where(InterfaceChange.repo_id == repo_id), + ("id", "repo_id"), + ) + bindings = await _relationship_rows( + session, + select( + Workplan.id.label("id"), + Workplan.repo_id.label("repo_id"), + Workplan.backing_relative_path.label("backing_relative_path"), + Workplan.backing_filename.label("backing_filename"), + ).where( + Workplan.repo_id == repo_id, + Workplan.backing_relative_path.is_not(None), + ), + ("id", "repo_id", "backing_relative_path", "backing_filename"), + ) + slug_rows = await _relationship_rows( + session, + select( + RepositorySlug.id.label("id"), + RepositorySlug.repo_id.label("repo_id"), + RepositorySlug.slug.label("slug"), + RepositorySlug.kind.label("kind"), + ).where(RepositorySlug.repo_id == repo_id), + ("id", "repo_id", "slug", "kind"), + ) + known_slugs = [row["slug"] for row in slug_rows] + messages = await _relationship_rows( + session, + select( + AgentMessage.id.label("id"), + AgentMessage.from_agent.label("from_agent"), + AgentMessage.to_agent.label("to_agent"), + AgentMessage.thread_id.label("thread_id"), + ).where( + or_( + AgentMessage.from_agent.in_(known_slugs), + AgentMessage.to_agent.in_(known_slugs), + ) + if known_slugs + else False + ), + ("id", "from_agent", "to_agent", "thread_id"), + ) + active_work = await _active_work(session, repo_id) + active_dispatch = [ + {"id": row["id"], "repo_id": str(repo_id), "kind": "workplan"} + for row in active_work["workplans"] + ] + [ + { + "id": row["id"], + "workplan_id": row["workplan_id"], + "kind": "task", + } + for row in active_work["tasks"] + ] + active_dispatch = sorted(active_dispatch, key=lambda row: canonical_json(row)) + records = { + "repository": _relationship_summary( + [{"id": str(repo_id), "repo_id": str(repo_id)}] + ), + "workplans": _relationship_summary(workplans), + "tasks": _relationship_summary(tasks), + "progress_events": _relationship_summary(progress), + "decisions": _relationship_summary(decisions), + "token_events": _relationship_summary(token_events, totals=token_totals), + "sbom_snapshots": _relationship_summary(sbom_snapshots), + "sbom_entries": _relationship_summary(sbom_entries), + "services": _relationship_summary(services), + "capabilities": _relationship_summary(capabilities), + "interface_changes": _relationship_summary(interface_changes), + "workplan_bindings": _relationship_summary(bindings), + "active_dispatch": _relationship_summary(active_dispatch), + "aliases": _relationship_summary(slug_rows), + "messages": _relationship_summary(messages), + } + records["continuity_checksum"] = checksum(records) + return records + + +async def _active_work(session: AsyncSession, repo_id: uuid.UUID) -> dict[str, Any]: + workplans_result = await session.execute( + select(Workplan.id, Workplan.slug, Workplan.status) + .where( + Workplan.repo_id == repo_id, + Workplan.status.not_in(("finished", "archived")), + ) + .order_by(Workplan.slug) + ) + workplans = [ + {"id": str(row.id), "slug": row.slug, "status": row.status} + for row in workplans_result.all() + ] + workplan_ids = [uuid.UUID(row["id"]) for row in workplans] + tasks_result = await session.execute( + select(Task.id, Task.workplan_id, Task.record_id, Task.status) + .where( + Task.workplan_id.in_(workplan_ids) if workplan_ids else False, + Task.status.not_in(("done", "cancel")), + ) + .order_by(Task.id) + ) + tasks = [ + { + "id": str(row.id), + "workplan_id": str(row.workplan_id), + "record_id": row.record_id, + "status": getattr(row.status, "value", str(row.status)), + } + for row in tasks_result.all() + ] + return {"workplans": workplans, "tasks": tasks} + + +async def _affected_records( + session: AsyncSession, repo_id: uuid.UUID, old_slug: str +) -> dict[str, Any]: + messages = await _id_rows( + session, + select(AgentMessage.id).where( + or_(AgentMessage.from_agent == old_slug, AgentMessage.to_agent == old_slug) + ), + ) + interface_mentions = await _id_rows( + session, + select(InterfaceChange.id).where( + InterfaceChange.affected_repo_slugs.contains([old_slug]) + ), + ) + fabric_nodes = await _id_rows( + session, + select(FabricGraphNode.id).where( + or_( + FabricGraphNode.repo_slug == old_slug, + FabricGraphNode.source_repo_slug == old_slug, + ) + ), + ) + return { + "messages": _record_summary(messages), + "interface_change_mentions": _record_summary(interface_mentions), + "fabric_nodes": _record_summary(fabric_nodes), + "external_handoffs": [ + item + for item, ids in ( + ("interface-change-consumers", interface_mentions), + ("fabric-graph-projections", fabric_nodes), + ) + if ids + ], + } + + +def _new_remote_url(repo: ManagedRepo, identity: RepositoryForgeIdentity, new_slug: str) -> str: + base = str(identity.forge_instance).rstrip("/") + suffix = ".git" if (repo.remote_url or "").endswith(".git") else "" + return f"{base}/{identity.forge_owner}/{new_slug}{suffix}" + + +async def _load_repo_identity( + session: AsyncSession, repo_id: uuid.UUID +) -> tuple[ManagedRepo, RepositoryForgeIdentity | None]: + repo = await session.get(ManagedRepo, repo_id) + if repo is None: + raise RenameNotFound(f"Repository {repo_id} was not found") + result = await session.execute( + select(RepositoryForgeIdentity).where( + RepositoryForgeIdentity.repo_id == repo_id + ) + ) + return repo, result.scalar_one_or_none() + + +async def verify_forge_identity( + session: AsyncSession, + gateway: ForgeRepositoryGateway, + repo_id: uuid.UUID, + body: ForgeIdentityVerifyRequest, +) -> RepositoryForgeIdentity: + repo, identity = await _load_repo_identity(session, repo_id) + if identity is None: + raise RenamePreconditionFailed("Repository has no Forge identity row") + try: + snapshot = await gateway.inspect( + instance=body.forge_instance, + owner=body.forge_owner, + name=repo.slug, + ) + except ForgeRepositoryUnreadable as exc: + raise RenamePreconditionFailed(str(exc)) from exc + if snapshot is None: + raise RenamePreconditionFailed("Forge repository is absent or unreadable") + if snapshot.name != repo.slug: + raise RenamePreconditionFailed( + "Forge identity verification followed a repository redirect", + details={"expected_name": repo.slug, "actual_name": snapshot.name}, + ) + if snapshot.repository_id != body.forge_repository_id: + raise RenamePreconditionFailed( + "Forge repository ID does not match the asserted immutable ID", + details={ + "expected_forge_repository_id": body.forge_repository_id, + "actual_forge_repository_id": snapshot.repository_id, + }, + ) + if identity.verification_state == "verified": + actual = ( + identity.provider, + identity.forge_instance, + identity.forge_owner, + identity.forge_repository_id, + ) + supplied = ( + body.provider, + body.forge_instance.rstrip("/"), + body.forge_owner, + body.forge_repository_id, + ) + if actual != supplied: + raise RenamePreconditionFailed("Verified Forge identity is immutable") + return identity + identity.provider = body.provider + identity.forge_instance = body.forge_instance.rstrip("/") + identity.forge_owner = body.forge_owner + identity.forge_repository_id = body.forge_repository_id + identity.verification_state = "verified" + identity.verified_at = utcnow() + identity.verified_by = body.verified_by + identity.verification_evidence = { + "forge_snapshot": snapshot.as_dict(), + "verified_repo_slug": repo.slug, + } + await session.commit() + await session.refresh(identity) + return identity + + +async def build_preflight( + session: AsyncSession, + gateway: ForgeRepositoryGateway, + repo_id: uuid.UUID, + body: RepositoryRenamePreflightRequest, + *, + issue_token: bool = True, +) -> dict[str, Any]: + repo, identity = await _load_repo_identity(session, repo_id) + now = utcnow() + blockers: list[dict[str, Any]] = [] + warnings: list[dict[str, Any]] = [] + old_snapshot: ForgeRepositorySnapshot | None = None + target_snapshot: ForgeRepositorySnapshot | None = None + + if issue_token and not settings.repository_rename_preflight_secret: + blockers.append( + { + "code": "preflight_signing_unavailable", + "message": "Repository rename preflight signing is not configured", + } + ) + + if repo.slug == body.new_slug: + blockers.append({"code": "same_slug", "message": "Target slug equals canonical slug"}) + claimed = await session.execute( + select(RepositorySlug).where(RepositorySlug.slug == body.new_slug) + ) + claimed_row = claimed.scalar_one_or_none() + old_claim = await session.execute( + select(RepositorySlug).where(RepositorySlug.slug == repo.slug) + ) + old_claim_row = old_claim.scalar_one_or_none() + if ( + old_claim_row is None + or old_claim_row.repo_id != repo.id + or old_claim_row.kind != "canonical" + ): + blockers.append( + { + "code": "statehub_canonical_inconsistent", + "message": "Managed repository slug is not its protected canonical route", + } + ) + if claimed_row is not None: + blockers.append( + { + "code": "statehub_slug_claimed", + "message": "Target slug is already protected in State Hub", + "repo_id": str(claimed_row.repo_id), + "kind": claimed_row.kind, + } + ) + active_result = await session.execute( + select(RepositoryRenameOperation).where( + RepositoryRenameOperation.repo_id == repo_id, + RepositoryRenameOperation.phase.in_(ACTIVE_OPERATION_PHASES), + ) + ) + active_operations = list(active_result.scalars().all()) + if active_operations: + blockers.append( + { + "code": "active_rename_operation", + "message": "Repository already has an active rename operation", + "operation_ids": [str(item.id) for item in active_operations], + } + ) + target_operation_result = await session.execute( + select(RepositoryRenameOperation.id, RepositoryRenameOperation.repo_id).where( + RepositoryRenameOperation.new_slug == body.new_slug, + RepositoryRenameOperation.phase.in_(ACTIVE_OPERATION_PHASES), + RepositoryRenameOperation.repo_id != repo_id, + ) + ) + target_operations = list(target_operation_result.all()) + if target_operations: + blockers.append( + { + "code": "target_held_by_active_operation", + "message": "Target slug is reserved by another active rename operation", + "operations": [ + {"operation_id": str(row.id), "repo_id": str(row.repo_id)} + for row in target_operations + ], + } + ) + if body.queued_edge_writes: + blockers.append( + { + "code": "queued_edge_writes", + "message": "Caller reported edge writes that must be replayed or resolved first", + "count": len(body.queued_edge_writes), + } + ) + if identity is None or identity.verification_state != "verified": + blockers.append( + { + "code": "forge_identity_unverified", + "message": "Repository Forge identity is not verified", + } + ) + else: + try: + old_snapshot = await gateway.inspect( + instance=str(identity.forge_instance), + owner=str(identity.forge_owner), + name=repo.slug, + ) + target_snapshot = await gateway.inspect( + instance=str(identity.forge_instance), + owner=str(identity.forge_owner), + name=body.new_slug, + ) + except ForgeRepositoryUnreadable as exc: + blockers.append( + {"code": "forge_unreadable", "message": str(exc)} + ) + if old_snapshot is None: + blockers.append( + {"code": "forge_source_absent", "message": "Canonical Forge repository is absent or unreadable"} + ) + else: + if old_snapshot.name != repo.slug: + blockers.append( + { + "code": "forge_redirected", + "message": "Forge source lookup resolved to another coordinate", + "expected": repo.slug, + "actual": old_snapshot.name, + } + ) + if old_snapshot.repository_id != identity.forge_repository_id: + blockers.append( + { + "code": "wrong_forge_repository_id", + "message": "Forge repository ID differs from verified identity", + "expected": identity.forge_repository_id, + "actual": old_snapshot.repository_id, + } + ) + if not old_snapshot.projection_readable or not old_snapshot.projection_source_present: + blockers.append( + { + "code": "projection_unreadable", + "message": "Forge work-record projection source is not readable", + } + ) + if target_snapshot is not None: + blockers.append( + { + "code": "forge_target_claimed", + "message": "Target Forge repository name is already claimed", + "forge_repository_id": target_snapshot.repository_id, + } + ) + + baselines = await collect_continuity_baseline(session, repo_id) + active_work = await _active_work(session, repo_id) + affected = await _affected_records(session, repo_id, repo.slug) + affected.update( + { + "workplan_bindings": baselines["workplan_bindings"], + "workplans": baselines["workplans"], + "tasks": baselines["tasks"], + "progress_events": baselines["progress_events"], + "decisions": baselines["decisions"], + "token_events": baselines["token_events"], + "interface_changes": baselines["interface_changes"], + "sbom_snapshots": baselines["sbom_snapshots"], + "sbom_entries": baselines["sbom_entries"], + "services": baselines["services"], + "capability_entries": baselines["capabilities"], + } + ) + if active_work["workplans"]: + warnings.append( + { + "code": "active_work_present", + "message": "Active work must be quiesced or explicitly coordinated during cutover", + "workplan_count": len(active_work["workplans"]), + "task_count": len(active_work["tasks"]), + } + ) + + identity_dict = ( + { + "id": str(identity.id), + "verification_state": identity.verification_state, + "provider": identity.provider, + "forge_instance": identity.forge_instance, + "forge_owner": identity.forge_owner, + "forge_repository_id": identity.forge_repository_id, + } + if identity is not None + else None + ) + new_remote = _new_remote_url(repo, identity, body.new_slug) if identity else None + proposed = [ + {"phase": "forge-renamed", "system": "forgejo", "field": "name", "from": repo.slug, "to": body.new_slug}, + {"phase": "statehub-rebound", "system": "state-hub", "field": "managed_repos.slug", "from": repo.slug, "to": body.new_slug}, + {"phase": "statehub-rebound", "system": "state-hub", "field": "managed_repos.remote_url", "from": repo.remote_url, "to": new_remote}, + {"phase": "statehub-rebound", "system": "state-hub", "field": "repository_slugs", "from": f"{repo.slug}:canonical", "to": f"{repo.slug}:alias,{body.new_slug}:canonical"}, + ] + retained = [ + {"field": "managed_repos.id", "value": str(repo.id)}, + {"field": "repository_forge_identities.forge_repository_id", "value": identity.forge_repository_id if identity else None}, + {"field": "repository_slugs.alias", "value": repo.slug}, + {"field": "managed_repos.local_path", "value": repo.local_path}, + {"field": "managed_repos.host_paths", "value": repo.host_paths}, + {"field": "telemetry_and_work_record_ids", "value": baselines["continuity_checksum"]}, + ] + stable_report = { + "schema_version": "state-hub.repository-rename-preflight.v1", + "repo_id": str(repo.id), + "old_slug": repo.slug, + "new_slug": body.new_slug, + "safe_to_apply": not blockers, + "blockers": blockers, + "warnings": warnings, + "current": { + "statehub": { + "repo_id": str(repo.id), + "slug": repo.slug, + "name": repo.name, + "remote_url": repo.remote_url, + "local_path": repo.local_path, + "host_paths": repo.host_paths, + "canonical_available": bool( + old_claim_row + and old_claim_row.repo_id == repo.id + and old_claim_row.kind == "canonical" + ), + "slug_registry": ( + { + "id": str(old_claim_row.id), + "kind": old_claim_row.kind, + "protected": old_claim_row.protected, + } + if old_claim_row + else None + ), + }, + "forge_identity": identity_dict, + "forge_available": old_snapshot is not None, + "forge": old_snapshot.as_dict() if old_snapshot else None, + }, + "target": { + "slug": body.new_slug, + "statehub_available": claimed_row is None, + "forge_available": target_snapshot is None, + "forge": target_snapshot.as_dict() if target_snapshot else None, + }, + "baselines": baselines, + "active_work": active_work, + "affected": affected, + "queued_edge_writes": [item.model_dump(mode="json") for item in body.queued_edge_writes], + "proposed_mutations": proposed, + "retained_history": retained, + } + report_checksum = checksum(stable_report) + expires_at = now + timedelta(seconds=settings.repository_rename_preflight_ttl_seconds) + token = None + if issue_token and not blockers: + token = _sign_preflight( + { + "version": 1, + "repo_id": str(repo.id), + "old_slug": repo.slug, + "new_slug": body.new_slug, + "forge_repository_id": identity.forge_repository_id if identity else None, + "source_commit": old_snapshot.head_commit if old_snapshot else None, + "report_checksum": report_checksum, + "issued_at": now.isoformat(), + "expires_at": expires_at.isoformat(), + } + ) + return { + **stable_report, + "report_checksum": report_checksum, + "preflight_token": token, + "preflighted_at": now, + "expires_at": expires_at if token else None, + } + + +async def create_operation( + session: AsyncSession, + gateway: ForgeRepositoryGateway, + repo_id: uuid.UUID, + body: RepositoryRenameOperationCreate, +) -> tuple[RepositoryRenameOperation, bool]: + if body.operation_id is not None: + existing = await session.get(RepositoryRenameOperation, body.operation_id) + if existing is not None: + if ( + existing.repo_id != repo_id + or existing.new_slug != body.new_slug + or existing.actor != body.actor + or body.confirmation + != confirmation_for(repo_id, existing.old_slug, existing.new_slug) + ): + raise RenamePreconditionFailed( + "Repository rename operation ID is already bound to another request" + ) + return existing, True + + token = _verify_preflight_token(body.preflight_token) + if token.get("repo_id") != str(repo_id) or token.get("new_slug") != body.new_slug: + raise RenamePreconditionFailed("Preflight token does not address this rename") + preflight = await build_preflight( + session, + gateway, + repo_id, + RepositoryRenamePreflightRequest( + new_slug=body.new_slug, queued_edge_writes=body.queued_edge_writes + ), + issue_token=False, + ) + if not preflight["safe_to_apply"]: + raise RenamePreconditionFailed( + "Repository rename preflight is no longer safe", + details={"blockers": preflight["blockers"]}, + ) + if preflight["report_checksum"] != token.get("report_checksum"): + raise RenamePreconditionFailed( + "Repository rename preflight evidence is stale", + details={ + "expected_report_checksum": token.get("report_checksum"), + "actual_report_checksum": preflight["report_checksum"], + }, + ) + old_slug = preflight["old_slug"] + if body.confirmation != confirmation_for(repo_id, old_slug, body.new_slug): + raise RenamePreconditionFailed("Explicit repository rename confirmation is incorrect") + repo, identity = await _load_repo_identity(session, repo_id) + if identity is None or identity.verification_state != "verified": + raise RenamePreconditionFailed("Repository Forge identity is not verified") + forge = preflight["current"]["forge"] + now = utcnow() + operation = RepositoryRenameOperation( + id=body.operation_id or uuid.uuid4(), + repo_id=repo.id, + forge_identity_id=identity.id, + forge_identity_state="verified", + expected_provider=str(identity.provider), + expected_forge_instance=str(identity.forge_instance), + expected_forge_owner=str(identity.forge_owner), + expected_forge_repository_id=int(identity.forge_repository_id), + expected_source_commit=forge["head_commit"], + expected_default_branch=forge["default_branch"], + old_slug=repo.slug, + new_slug=body.new_slug, + old_coordinates={ + "slug": repo.slug, + "name": repo.name, + "remote_url": repo.remote_url, + "local_path": repo.local_path, + "host_paths": repo.host_paths, + }, + new_coordinates={ + "slug": body.new_slug, + "name": body.new_slug, + "remote_url": _new_remote_url(repo, identity, body.new_slug), + "local_path": repo.local_path, + "host_paths": repo.host_paths, + }, + phase="preflighted", + actor=body.actor, + phase_changed_at=now, + preflighted_at=datetime.fromisoformat(token["issued_at"]), + preflight_expires_at=datetime.fromisoformat(token["expires_at"]), + evidence={ + "preflight": { + "report_checksum": preflight["report_checksum"], + "baselines": preflight["baselines"], + "affected": preflight["affected"], + "warnings": preflight["warnings"], + }, + "phases": { + "preflighted": {"at": now.isoformat(), "resumed": False} + }, + "telemetry": { + "started_at": now.isoformat(), + "phase_attempts": {"preflighted": 1}, + "phase_durations_ms": {"preflighted": 0}, + "retries": 0, + "failures": 0, + "rollback_attempts": 0, + "verification_outcome": "pending", + }, + }, + ) + session.add(operation) + try: + await session.commit() + except IntegrityError as exc: + await session.rollback() + if body.operation_id is not None: + existing = await session.get(RepositoryRenameOperation, body.operation_id) + if ( + existing is not None + and existing.repo_id == repo_id + and existing.new_slug == body.new_slug + and existing.actor == body.actor + and body.confirmation + == confirmation_for(repo_id, existing.old_slug, existing.new_slug) + ): + return existing, True + raise RenamePreconditionFailed( + "A conflicting repository rename operation was created" + ) from exc + await session.refresh(operation) + return operation, False + + +async def load_operation_by_id( + session: AsyncSession, + operation_id: uuid.UUID, +) -> RepositoryRenameOperation: + operation = await session.get(RepositoryRenameOperation, operation_id) + if operation is None: + raise RenameNotFound( + f"Repository rename operation {operation_id} was not found" + ) + return operation + + +async def load_operation( + session: AsyncSession, + repo_id: uuid.UUID, + operation_id: uuid.UUID, + *, + for_update: bool = False, +) -> RepositoryRenameOperation: + query = select(RepositoryRenameOperation).where( + RepositoryRenameOperation.id == operation_id, + RepositoryRenameOperation.repo_id == repo_id, + ) + if for_update: + query = query.with_for_update() + result = await session.execute(query) + operation = result.scalar_one_or_none() + if operation is None: + raise RenameNotFound(f"Repository rename operation {operation_id} was not found") + return operation + + +async def list_operations( + session: AsyncSession, repo_id: uuid.UUID, *, active_only: bool = False +) -> list[RepositoryRenameOperation]: + repo = await session.get(ManagedRepo, repo_id) + if repo is None: + raise RenameNotFound(f"Repository {repo_id} was not found") + query = ( + select(RepositoryRenameOperation) + .where(RepositoryRenameOperation.repo_id == repo_id) + .order_by(RepositoryRenameOperation.created_at.desc()) + ) + if active_only: + query = query.where(RepositoryRenameOperation.phase.in_(ACTIVE_OPERATION_PHASES)) + result = await session.execute(query) + return list(result.scalars().all()) + + +def _expected_head(operation: RepositoryRenameOperation) -> str: + """Head the Forge default branch must show at this point in the journal. + + The preflight baseline anchors every phase up to ``statehub-rebound``. + ``source-synced`` legitimately advances the head (the source-synchronization + revision and fix-consistency writeback are pushed from the fresh clone), so + once that phase is journalled its recorded Forge head is the anchor. + """ + journal, _ = _telemetry(operation) + synced = (journal.get("phases") or {}).get("source-synced") or {} + head = (synced.get("forge") or {}).get("head_commit") + return head or operation.expected_source_commit + + +def _assert_snapshot( + operation: RepositoryRenameOperation, + snapshot: ForgeRepositorySnapshot, + *, + expected_name: str | None = None, + expected_head: str | None = None, +) -> None: + expected_head = expected_head or _expected_head(operation) + if snapshot.repository_id != operation.expected_forge_repository_id: + raise RenamePreconditionFailed( + "Forge repository ID changed", + details={ + "expected": operation.expected_forge_repository_id, + "actual": snapshot.repository_id, + }, + ) + if snapshot.default_branch != operation.expected_default_branch: + raise RenamePreconditionFailed("Forge default branch changed") + if snapshot.head_commit != expected_head: + raise RenamePreconditionFailed( + "Forge default-branch head moved", + details={ + "expected": expected_head, + "actual": snapshot.head_commit, + }, + ) + if not snapshot.projection_readable or not snapshot.projection_source_present: + raise RenamePreconditionFailed("Forge work-record projection is unreadable") + if expected_name is not None and snapshot.name != expected_name: + raise RenamePreconditionFailed( + "Forge response resolved to a different repository coordinate", + details={"expected_name": expected_name, "actual_name": snapshot.name}, + ) + + +async def _forge_at( + gateway: ForgeRepositoryGateway, + operation: RepositoryRenameOperation, + name: str, +) -> ForgeRepositorySnapshot | None: + try: + return await gateway.inspect( + instance=operation.expected_forge_instance, + owner=operation.expected_forge_owner, + name=name, + ) + except ForgeRepositoryUnreadable as exc: + raise RenamePreconditionFailed(str(exc)) from exc + + +async def _apply_forge_rename( + gateway: ForgeRepositoryGateway, operation: RepositoryRenameOperation +) -> dict[str, Any]: + new = await _forge_at(gateway, operation, operation.new_slug) + if new is not None: + # A redirect is compatibility evidence only. Establish immutable identity + # at the requested new coordinate before accepting the old-name alias. + _assert_snapshot(operation, new, expected_name=operation.new_slug) + try: + old = await gateway.inspect( + instance=operation.expected_forge_instance, + owner=operation.expected_forge_owner, + name=operation.old_slug, + ) + except ForgeRepositoryRedirected as exc: + canonical_api = ( + operation.expected_forge_instance.rstrip("/") + + "/api/v1/repos/" + + quote(operation.expected_forge_owner, safe="") + + "/" + + quote(operation.new_slug, safe="") + ) + if exc.target_url != canonical_api: + raise RenamePreconditionFailed( + "Old Forge name redirects outside the verified new coordinate" + ) from exc + old = None + except ForgeRepositoryUnreadable as exc: + raise RenamePreconditionFailed(str(exc)) from exc + if old is not None: + raise RenamePreconditionFailed("Both old and new Forge names are claimed") + return {"resumed": True, "forge": new.as_dict()} + old = await _forge_at(gateway, operation, operation.old_slug) + if old is None: + raise RenamePreconditionFailed("Forge repository is absent at both expected names") + _assert_snapshot(operation, old, expected_name=operation.old_slug) + try: + renamed = await gateway.rename( + instance=operation.expected_forge_instance, + owner=operation.expected_forge_owner, + old_name=operation.old_slug, + new_name=operation.new_slug, + ) + except (ForgeRepositoryUnreadable, ForgeRepositoryConflict) as exc: + raise RenamePreconditionFailed(str(exc)) from exc + _assert_snapshot(operation, renamed, expected_name=operation.new_slug) + return {"resumed": False, "forge": renamed.as_dict()} + + +async def _apply_statehub_rebind( + session: AsyncSession, operation: RepositoryRenameOperation +) -> dict[str, Any]: + repo = await session.get(ManagedRepo, operation.repo_id, with_for_update=True) + if repo is None: + raise RenameNotFound("Managed repository disappeared") + if repo.id != operation.repo_id: + raise RenamePreconditionFailed("Managed repository UUID changed") + old_result = await session.execute( + select(RepositorySlug) + .where(RepositorySlug.slug == operation.old_slug) + .with_for_update() + ) + new_result = await session.execute( + select(RepositorySlug) + .where(RepositorySlug.slug == operation.new_slug) + .with_for_update() + ) + old_slug = old_result.scalar_one_or_none() + new_slug = new_result.scalar_one_or_none() + if repo.slug == operation.new_slug: + if ( + old_slug is None + or old_slug.repo_id != repo.id + or old_slug.kind != "alias" + or new_slug is None + or new_slug.repo_id != repo.id + or new_slug.kind != "canonical" + ): + raise RenamePreconditionFailed("State Hub rename state is internally inconsistent") + return {"resumed": True, "repo_id": str(repo.id)} + if repo.slug != operation.old_slug: + raise RenamePreconditionFailed("Managed repository slug changed outside this operation") + if old_slug is None or old_slug.repo_id != repo.id or old_slug.kind != "canonical": + raise RenamePreconditionFailed("Old canonical slug record is missing") + if new_slug is not None: + raise RenamePreconditionFailed("Target State Hub slug became claimed") + old_slug.kind = "alias" + old_slug.protected = True + old_slug.source_operation_id = operation.id + await session.flush() + session.add( + RepositorySlug( + repo_id=repo.id, + slug=operation.new_slug, + kind="canonical", + protected=True, + source_operation_id=operation.id, + ) + ) + repo.slug = operation.new_slug + repo.name = operation.new_coordinates["name"] + repo.remote_url = operation.new_coordinates["remote_url"] + await session.flush() + return {"resumed": False, "repo_id": str(repo.id)} + + +async def verify_operation( + session: AsyncSession, + gateway: ForgeRepositoryGateway, + operation: RepositoryRenameOperation, +) -> dict[str, Any]: + checks: list[dict[str, Any]] = [] + + def add(name: str, ok: bool, expected: Any, actual: Any) -> None: + checks.append({"name": name, "ok": ok, "expected": expected, "actual": actual}) + + repo = await session.get(ManagedRepo, operation.repo_id) + add("managed_repository_exists", repo is not None, str(operation.repo_id), str(repo.id) if repo else None) + add("managed_repository_uuid", bool(repo and repo.id == operation.repo_id), str(operation.repo_id), str(repo.id) if repo else None) + expected_name = operation.old_slug if operation.phase == "preflighted" else operation.new_slug + if operation.phase == "rolled-back": + expected_name = operation.old_slug + try: + forge = await _forge_at(gateway, operation, expected_name) + except RenameLifecycleError as exc: + forge = None + add("forge_readable", False, True, str(exc)) + else: + add("forge_readable", forge is not None, True, forge is not None) + if forge is not None: + add("forge_coordinate", forge.name == expected_name, expected_name, forge.name) + add("forge_repository_id", forge.repository_id == operation.expected_forge_repository_id, operation.expected_forge_repository_id, forge.repository_id) + anchor = _expected_head(operation) + head_ok = forge.head_commit == anchor + if not head_ok and "source-synced" in (_telemetry(operation)[0].get("phases") or {}): + # After source-synced the source repository keeps moving (its own + # commits and sync writebacks); the recorded head must stay contained. + try: + head_ok = await gateway.descends_from( + instance=operation.expected_forge_instance, + owner=operation.expected_forge_owner, + name=expected_name, + base=anchor, + head=forge.head_commit, + ) + except (ForgeRepositoryError, AttributeError): + head_ok = False + add("source_commit", head_ok, anchor, forge.head_commit) + add("default_branch", forge.default_branch == operation.expected_default_branch, operation.expected_default_branch, forge.default_branch) + add("projection_readable", forge.projection_readable and forge.projection_source_present, True, forge.projection_readable and forge.projection_source_present) + expected_canonical = operation.old_slug if operation.phase in {"preflighted", "forge-renamed", "rolled-back"} else operation.new_slug + canonical_result = await session.execute( + select(RepositorySlug).where( + RepositorySlug.repo_id == operation.repo_id, + RepositorySlug.kind == "canonical", + ) + ) + canonical = canonical_result.scalar_one_or_none() + add("canonical_route", bool(canonical and canonical.slug == expected_canonical), expected_canonical, canonical.slug if canonical else None) + current = await collect_continuity_baseline(session, operation.repo_id) + baseline = (operation.evidence.get("preflight") or {}).get("baselines") or {} + expected_checksum = baseline.get("continuity_checksum", "") + missing: dict[str, list[str]] = {} + changed_relationships: dict[str, list[str]] = {} + preserved_relationship_checksums: dict[str, str] = {} + counts: dict[str, dict[str, int]] = {} + for record_type, baseline_record in baseline.items(): + if record_type == "continuity_checksum" or not isinstance(baseline_record, dict): + continue + current_record = current.get(record_type) or {"ids": [], "count": 0} + absent = sorted(set(baseline_record.get("ids") or []) - set(current_record.get("ids") or [])) + if absent: + missing[record_type] = absent + if record_type != "aliases": + baseline_rows = { + row["id"]: row + for row in baseline_record.get("relationships") or [] + } + current_rows = { + row["id"]: row + for row in current_record.get("relationships") or [] + } + changed = sorted( + record_id + for record_id, baseline_row in baseline_rows.items() + if current_rows.get(record_id) != baseline_row + ) + if changed: + changed_relationships[record_type] = changed + preserved_rows = [ + current_rows[record_id] + for record_id in baseline_rows + if record_id in current_rows + ] + preserved_relationship_checksums[record_type] = checksum( + sorted(preserved_rows, key=lambda row: canonical_json(row)) + ) + counts[record_type] = { + "baseline": int(baseline_record.get("count") or 0), + "current": int(current_record.get("count") or 0), + } + # Active dispatch is a status-derived view: finishing a task during the + # operation retires it from dispatch without losing the record. Only a + # dispatch id whose task/workplan no longer exists counts as missing. + retired_dispatch: list[str] = [] + live_ids = set((current.get("tasks") or {}).get("ids") or []) | set( + (current.get("workplans") or {}).get("ids") or [] + ) + for bucket in (missing, changed_relationships): + ids = bucket.get("active_dispatch") + if not ids: + continue + gone = [item for item in ids if item not in live_ids] + retired_dispatch = sorted(set(retired_dispatch) | (set(ids) - set(gone))) + if gone: + bucket["active_dispatch"] = gone + else: + del bucket["active_dispatch"] + if retired_dispatch: + counts.pop("active_dispatch", None) + # New telemetry may legitimately arrive while a phased operation is being + # executed. Continuity means every baseline identity still exists; it does + # not freeze the repository's append-only history at the preflight count. + add( + "relationship_continuity", + not missing and not changed_relationships, + {"missing": {}, "changed": {}}, + {"missing": missing, "changed": changed_relationships, "retired_dispatch": retired_dispatch}, + ) + add( + "record_counts_non_decreasing", + all(item["current"] >= item["baseline"] for item in counts.values()), + {name: item["baseline"] for name, item in counts.items()}, + {name: item["current"] for name, item in counts.items()}, + ) + baseline_token_totals = (baseline.get("token_events") or {}).get("totals") or {} + current_token_totals = (current.get("token_events") or {}).get("totals") or {} + add( + "token_totals_non_decreasing", + all( + int(current_token_totals.get(name) or 0) >= int(value or 0) + for name, value in baseline_token_totals.items() + ), + baseline_token_totals, + current_token_totals, + ) + slug_result = await session.execute( + select(RepositorySlug).where( + RepositorySlug.repo_id == operation.repo_id, + RepositorySlug.slug.in_((operation.old_slug, operation.new_slug)), + ) + ) + slug_routes = {row.slug: row.kind for row in slug_result.scalars().all()} + if operation.phase in {"preflighted", "forge-renamed"}: + expected_routes = {operation.old_slug: "canonical"} + elif operation.phase == "rolled-back": + expected_routes = { + operation.old_slug: "canonical", + operation.new_slug: "alias", + } + else: + expected_routes = { + operation.old_slug: "alias", + operation.new_slug: "canonical", + } + add( + "slug_routes", + all(slug_routes.get(slug) == kind for slug, kind in expected_routes.items()), + expected_routes, + slug_routes, + ) + return { + "operation_id": operation.id, + "repo_id": operation.repo_id, + "phase": operation.phase, + "ok": all(item["ok"] for item in checks), + "checks": checks, + "baseline_checksum": expected_checksum, + "current_checksum": current["continuity_checksum"], + "relationship_checksums": { + "baseline": { + name: value.get("relationship_checksum") + for name, value in baseline.items() + if isinstance(value, dict) and name != "aliases" + }, + "preserved": preserved_relationship_checksums, + }, + } + + +def _telemetry(operation: RepositoryRenameOperation) -> tuple[dict[str, Any], dict[str, Any]]: + journal = deepcopy(operation.evidence or {}) + telemetry = dict(journal.get("telemetry") or {}) + telemetry.setdefault("started_at", operation.preflighted_at.isoformat()) + telemetry.setdefault("phase_attempts", {}) + telemetry.setdefault("phase_durations_ms", {}) + telemetry.setdefault("retries", 0) + telemetry.setdefault("failures", 0) + telemetry.setdefault("rollback_attempts", 0) + telemetry.setdefault("verification_outcome", "pending") + return journal, telemetry + + +def _record_retry(operation: RepositoryRenameOperation, phase: str) -> None: + journal, telemetry = _telemetry(operation) + attempts = dict(telemetry["phase_attempts"]) + attempts[phase] = int(attempts.get(phase) or 0) + 1 + telemetry["phase_attempts"] = attempts + telemetry["retries"] = int(telemetry["retries"]) + 1 + telemetry["last_retry"] = {"phase": phase, "at": utcnow().isoformat()} + journal["telemetry"] = telemetry + operation.evidence = journal + + +def _record_failure( + operation: RepositoryRenameOperation, phase: str, exc: RenameLifecycleError +) -> None: + journal, telemetry = _telemetry(operation) + failures_by_phase = dict(telemetry.get("failures_by_phase") or {}) + failures_by_phase[phase] = int(failures_by_phase.get(phase) or 0) + 1 + telemetry["failures_by_phase"] = failures_by_phase + telemetry["failures"] = int(telemetry["failures"]) + 1 + telemetry["last_failure"] = { + "phase": phase, + "code": exc.code, + "at": utcnow().isoformat(), + } + if phase.startswith("rollback"): + telemetry["rollback_outcome"] = "failed" + journal["telemetry"] = telemetry + operation.evidence = journal + + +def _record_phase(operation: RepositoryRenameOperation, phase: str, evidence: dict[str, Any]) -> None: + now = utcnow() + journal, telemetry = _telemetry(operation) + phases = dict(journal.get("phases") or {}) + # Verification payloads use native UUID/datetime values for API response + # typing. The operation journal is JSONB, so normalize at the boundary + # instead of leaking storage concerns through every verifier. + safe_evidence = json.loads(canonical_json(evidence)) + phases[phase] = {"at": now.isoformat(), **safe_evidence} + journal["phases"] = phases + attempts = dict(telemetry["phase_attempts"]) + attempts[phase] = int(attempts.get(phase) or 0) + 1 + telemetry["phase_attempts"] = attempts + durations = dict(telemetry["phase_durations_ms"]) + durations[phase] = max( + 0, int((now - operation.phase_changed_at).total_seconds() * 1000) + ) + telemetry["phase_durations_ms"] = durations + telemetry["last_phase"] = phase + if phase == "rollback-preflight": + telemetry["rollback_attempts"] = int(telemetry["rollback_attempts"]) + 1 + telemetry["rollback_outcome"] = "approved" + elif phase == "rolled-back": + telemetry["rollback_outcome"] = "completed" + verification = evidence.get("verification") + if isinstance(verification, dict): + telemetry["verification_outcome"] = ( + "passed" if verification.get("ok") else "failed" + ) + journal["telemetry"] = telemetry + operation.evidence = journal + operation.phase = phase + operation.phase_changed_at = now + operation.error_code = None + operation.error_message = None + operation.error_details = None + operation.error_at = None + if phase == "completed": + operation.completed_at = now + if phase == "rolled-back": + operation.rolled_back_at = now + + +def _phase_index(phase: str) -> int: + try: + return FORWARD_PHASES.index(phase) + except ValueError: + return -1 + + +async def apply_phase( + session: AsyncSession, + gateway: ForgeRepositoryGateway, + repo_id: uuid.UUID, + operation_id: uuid.UUID, + requested_phase: str, + body: RepositoryRenamePhaseApply, +) -> tuple[RepositoryRenameOperation, bool]: + if requested_phase not in FORWARD_PHASES[1:]: + raise RenamePreconditionFailed(f"Unsupported forward phase '{requested_phase}'") + operation = await load_operation(session, repo_id, operation_id, for_update=True) + if body.confirmation != confirmation_for(repo_id, operation.old_slug, operation.new_slug): + raise RenamePreconditionFailed("Explicit repository rename confirmation is incorrect") + current_index = _phase_index(operation.phase) + requested_index = _phase_index(requested_phase) + if current_index >= requested_index >= 0: + _record_retry(operation, requested_phase) + await session.commit() + await session.refresh(operation) + return operation, True + if operation.phase != body.expected_phase: + raise RenamePreconditionFailed( + "Repository rename phase compare-and-set failed", + details={"expected": body.expected_phase, "actual": operation.phase}, + ) + if requested_index != current_index + 1: + raise RenamePreconditionFailed( + "Repository rename phases cannot be skipped", + details={"current": operation.phase, "requested": requested_phase}, + ) + try: + if requested_phase == "forge-renamed": + evidence = await _apply_forge_rename(gateway, operation) + elif requested_phase == "statehub-rebound": + forge = await _forge_at(gateway, operation, operation.new_slug) + if forge is None: + raise RenamePreconditionFailed("Forge rename is not observable") + _assert_snapshot(operation, forge, expected_name=operation.new_slug) + evidence = await _apply_statehub_rebind(session, operation) + elif requested_phase == "source-synced": + if not body.evidence or body.evidence.get("fresh_clone") is not True: + raise RenamePreconditionFailed( + "Source synchronization requires fresh-clone operator evidence" + ) + clone_repository_id = body.evidence.get("forge_repository_id") + if clone_repository_id != operation.expected_forge_repository_id: + raise RenamePreconditionFailed( + "Fresh clone points at the wrong Forge repository ID", + details={ + "expected": operation.expected_forge_repository_id, + "actual": clone_repository_id, + }, + ) + clone_head = body.evidence.get("head_commit") + forge = await _forge_at(gateway, operation, operation.new_slug) + if forge is None: + raise RenamePreconditionFailed("Renamed Forge repository is absent") + if clone_head != forge.head_commit: + raise RenamePreconditionFailed( + "Fresh clone head does not match the Forge default-branch head", + details={ + "expected": forge.head_commit, + "actual": clone_head, + "baseline": operation.expected_source_commit, + }, + ) + # The head may have advanced past the baseline through the + # source-synchronization revision; it becomes the new anchor. + _assert_snapshot( + operation, forge, expected_name=operation.new_slug, expected_head=clone_head + ) + evidence = { + "forge": forge.as_dict(), + "baseline_commit": operation.expected_source_commit, + "head_advanced": forge.head_commit != operation.expected_source_commit, + "operator_evidence": body.evidence, + } + elif requested_phase == "consumers-verified": + if not body.checks or not all(body.checks.values()): + raise RenamePreconditionFailed( + "Consumer verification requires a non-empty set of passing checks", + details={"checks": body.checks}, + ) + verification = await verify_operation(session, gateway, operation) + if not verification["ok"]: + raise RenamePreconditionFailed( + "Continuity verification failed before consumer acceptance", + details=verification, + ) + evidence = {"checks": body.checks, "verification": verification, "operator_evidence": body.evidence} + else: # completed + verification = await verify_operation(session, gateway, operation) + if not verification["ok"]: + raise RenamePreconditionFailed( + "Repository rename verification failed", details=verification + ) + evidence = {"verification": verification} + _record_phase(operation, requested_phase, evidence) + await session.commit() + await session.refresh(operation) + return operation, False + except RenameLifecycleError as exc: + # Roll back every State Hub write attempted by the phase. A Forge + # rename may already have committed externally; the unchanged journal + # phase is precisely what lets the next request discover and resume it. + await session.rollback() + failed = await load_operation(session, repo_id, operation_id, for_update=True) + _record_failure(failed, requested_phase, exc) + failed.error_code = exc.code + failed.error_message = str(exc) + failed.error_details = json.loads(canonical_json(exc.details)) + failed.error_at = utcnow() + await session.commit() + raise + except IntegrityError as exc: + await session.rollback() + failure = RenamePreconditionFailed( + "Repository rename phase lost a database compare-and-set race" + ) + failed = await load_operation(session, repo_id, operation_id, for_update=True) + _record_failure(failed, requested_phase, failure) + failed.error_code = failure.code + failed.error_message = str(failure) + failed.error_details = json.loads(canonical_json(failure.details)) + failed.error_at = utcnow() + await session.commit() + raise failure from exc + + +async def rollback_preflight( + session: AsyncSession, + gateway: ForgeRepositoryGateway, + repo_id: uuid.UUID, + operation_id: uuid.UUID, + *, + expected_phase: str, + confirmation: str, +) -> tuple[RepositoryRenameOperation, dict[str, Any]]: + operation = await load_operation(session, repo_id, operation_id, for_update=True) + if operation.phase == "rolled-back": + return operation, {"rollback_from_phase": "rolled-back", "safe_to_rollback": True, "blockers": [], "irreversible": []} + if confirmation != rollback_confirmation_for(operation.id): + raise RenamePreconditionFailed("Explicit rollback confirmation is incorrect") + if operation.phase != expected_phase: + raise RenamePreconditionFailed( + "Rollback preflight phase compare-and-set failed", + details={"expected": expected_phase, "actual": operation.phase}, + ) + if operation.phase == "rollback-preflight": + rollback_data = (operation.evidence or {}).get("rollback_preflight") or {} + return operation, rollback_data + competing_result = await session.execute( + select(RepositoryRenameOperation.id).where( + RepositoryRenameOperation.repo_id == repo_id, + RepositoryRenameOperation.id != operation.id, + RepositoryRenameOperation.phase.in_(ACTIVE_OPERATION_PHASES), + ) + ) + competing = [str(value) for value in competing_result.scalars().all()] + old = await _forge_at(gateway, operation, operation.old_slug) + new = await _forge_at(gateway, operation, operation.new_slug) + blockers: list[dict[str, Any]] = [] + if competing: + blockers.append( + { + "code": "newer_active_rename_operation", + "operation_ids": competing, + } + ) + rollback_from_phase = operation.phase + if operation.phase == "preflighted": + if old is not None and new is None: + try: + _assert_snapshot(operation, old, expected_name=operation.old_slug) + except RenameLifecycleError as exc: + blockers.append({"code": exc.code, "message": str(exc), **exc.details}) + elif old is None and new is not None: + # Forge committed but the phase journal did not. This is the + # principal interruption case the operation ID must recover from. + rollback_from_phase = "forge-renamed-unrecorded" + try: + _assert_snapshot(operation, new, expected_name=operation.new_slug) + except RenameLifecycleError as exc: + blockers.append({"code": exc.code, "message": str(exc), **exc.details}) + elif old is None: + blockers.append({"code": "forge_repository_absent_at_both_names"}) + else: + blockers.append({"code": "both_forge_slugs_claimed"}) + else: + if old is not None: + blockers.append({"code": "old_forge_slug_claimed", "forge_repository_id": old.repository_id}) + if new is None: + blockers.append({"code": "renamed_forge_repository_absent"}) + else: + try: + _assert_snapshot(operation, new, expected_name=operation.new_slug) + except RenameLifecycleError as exc: + blockers.append({"code": exc.code, "message": str(exc), **exc.details}) + irreversible = [ + {"code": "external_caches", "message": "External caches and redirects may retain the new coordinate."}, + {"code": "consumer_changes", "message": "Consumer commits made after cutover are not rewritten automatically."}, + {"code": "telemetry_history", "message": "The attempted operation and phase evidence remain durable."}, + ] + data = { + "rollback_from_phase": rollback_from_phase, + "safe_to_rollback": not blockers, + "blockers": blockers, + "irreversible": irreversible, + } + if blockers: + failure = RenamePreconditionFailed( + "Repository rename rollback is unsafe", details=data + ) + _record_failure(operation, "rollback-preflight", failure) + operation.error_code = failure.code + operation.error_message = str(failure) + operation.error_details = json.loads(canonical_json(failure.details)) + operation.error_at = utcnow() + await session.commit() + raise failure + journal = deepcopy(operation.evidence or {}) + journal["rollback_preflight"] = data + operation.evidence = journal + _record_phase(operation, "rollback-preflight", data) + await session.commit() + await session.refresh(operation) + return operation, data + + +async def _rollback_statehub( + session: AsyncSession, operation: RepositoryRenameOperation +) -> dict[str, Any]: + repo = await session.get(ManagedRepo, operation.repo_id, with_for_update=True) + if repo is None: + raise RenameNotFound("Managed repository disappeared") + if repo.slug == operation.old_slug: + return {"resumed": True, "repo_id": str(repo.id)} + if repo.slug != operation.new_slug: + raise RenamePreconditionFailed("Managed repository slug changed outside this operation") + old_result = await session.execute( + select(RepositorySlug).where(RepositorySlug.slug == operation.old_slug).with_for_update() + ) + new_result = await session.execute( + select(RepositorySlug).where(RepositorySlug.slug == operation.new_slug).with_for_update() + ) + old_slug = old_result.scalar_one_or_none() + new_slug = new_result.scalar_one_or_none() + if not old_slug or old_slug.repo_id != repo.id or old_slug.kind != "alias": + raise RenamePreconditionFailed("Protected old alias is unavailable for rollback") + if not new_slug or new_slug.repo_id != repo.id or new_slug.kind != "canonical": + raise RenamePreconditionFailed("New canonical slug is unavailable for rollback") + new_slug.kind = "alias" + new_slug.protected = True + new_slug.source_operation_id = operation.id + await session.flush() + old_slug.kind = "canonical" + old_slug.protected = True + old_slug.source_operation_id = operation.id + repo.slug = operation.old_slug + repo.name = operation.old_coordinates["name"] + repo.remote_url = operation.old_coordinates["remote_url"] + await session.flush() + return {"resumed": False, "repo_id": str(repo.id)} + + +async def apply_rollback( + session: AsyncSession, + gateway: ForgeRepositoryGateway, + repo_id: uuid.UUID, + operation_id: uuid.UUID, + *, + expected_phase: str, + confirmation: str, +) -> tuple[RepositoryRenameOperation, bool]: + operation = await load_operation(session, repo_id, operation_id, for_update=True) + if operation.phase == "rolled-back": + _record_retry(operation, "rolled-back") + await session.commit() + await session.refresh(operation) + return operation, True + if operation.phase != expected_phase or operation.phase != "rollback-preflight": + raise RenamePreconditionFailed( + "Rollback phase compare-and-set failed", + details={"expected": expected_phase, "actual": operation.phase}, + ) + if confirmation != rollback_confirmation_for(operation.id): + raise RenamePreconditionFailed("Explicit rollback confirmation is incorrect") + rollback_from = ((operation.evidence or {}).get("rollback_preflight") or {}).get("rollback_from_phase") + if not rollback_from: + raise RenamePreconditionFailed("Rollback preflight evidence is missing") + try: + old = await _forge_at(gateway, operation, operation.old_slug) + new = await _forge_at(gateway, operation, operation.new_slug) + forge_resumed = False + if rollback_from != "preflighted": + if old is not None: + _assert_snapshot(operation, old, expected_name=operation.old_slug) + if new is not None: + raise RenamePreconditionFailed( + "Both Forge coordinates are claimed during rollback" + ) + forge_resumed = True + else: + if new is None: + raise RenamePreconditionFailed( + "Forge repository is absent during rollback" + ) + _assert_snapshot(operation, new, expected_name=operation.new_slug) + try: + restored = await gateway.rename( + instance=operation.expected_forge_instance, + owner=operation.expected_forge_owner, + old_name=operation.new_slug, + new_name=operation.old_slug, + ) + except (ForgeRepositoryUnreadable, ForgeRepositoryConflict) as exc: + raise RenamePreconditionFailed(str(exc)) from exc + _assert_snapshot( + operation, restored, expected_name=operation.old_slug + ) + statehub = await _rollback_statehub(session, operation) + _record_phase( + operation, + "rolled-back", + { + "rollback_from_phase": rollback_from, + "forge_resumed": forge_resumed, + "statehub": statehub, + }, + ) + await session.commit() + await session.refresh(operation) + return operation, False + except RenameLifecycleError as exc: + await session.rollback() + failed = await load_operation(session, repo_id, operation_id, for_update=True) + _record_failure(failed, "rolled-back", exc) + failed.error_code = exc.code + failed.error_message = str(exc) + failed.error_details = json.loads(canonical_json(exc.details)) + failed.error_at = utcnow() + await session.commit() + raise diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/review_contracts.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/review_contracts.py new file mode 100644 index 00000000..0db02beb --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/review_contracts.py @@ -0,0 +1,336 @@ +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from typing import Any + +from fastapi import HTTPException +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from api.models.review_contract import ReviewContract, ReviewReceipt +from api.schemas.review_contract import ( + GateReviewState, + OwnerReviewState, + ReviewAggregateRead, +) + + +def canonical_json(value: Any) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode() + + +def digest_document(value: Any) -> str: + return hashlib.sha256(canonical_json(value)).hexdigest() + + +def _assertion_id(text: str) -> str: + return "assertion:" + hashlib.sha256(text.encode()).hexdigest()[:20] + + +def normalize_contract(raw: dict[str, Any]) -> tuple[dict[str, Any], str]: + """Validate v1 or adapt the WP-0024 prototype without executing its checks.""" + digest = digest_document(raw) + if raw.get("interface") == "railiance.owner-review" and raw.get("version") == 1: + owners_raw = raw.get("owners") + task_owners = raw.get("task_owners") + hashes = raw.get("artifact_sha256") + if ( + not isinstance(owners_raw, dict) + or not owners_raw + or not isinstance(task_owners, dict) + or not task_owners + ): + raise HTTPException( + 422, "legacy contract requires non-empty owners and task_owners" + ) + if not isinstance(hashes, dict): + raise HTTPException(422, "legacy contract requires artifact_sha256") + owners = [] + for owner_id, scope in owners_raw.items(): + artifacts = scope.get("artifacts") or [] + assertions = scope.get("assertions") or [] + checks = scope.get("checks") or [] + if ( + not artifacts + or not assertions + or not checks + or any(path not in hashes for path in artifacts) + ): + raise HTTPException(422, f"{owner_id}: incomplete legacy owner scope") + owners.append( + { + "id": owner_id, + "artifact_ids": artifacts, + "assertions": [ + {"id": _assertion_id(text), "statement": text} + for text in assertions + ], + "check_ids": checks, + } + ) + owner_ids = {item["id"] for item in owners} + gates = [] + for gate_id, required in task_owners.items(): + if ( + not required + or len(required) != len(set(required)) + or not set(required) <= owner_ids + ): + raise HTTPException(422, f"{gate_id}: invalid legacy owner set") + gates.append({"id": gate_id, "policy": "all_required", "owners": required}) + normalized = { + "schema_version": "review-contract/v1", + "contract_key": raw.get("workplan_id"), + "subject": {"kind": "workplan", "id": raw.get("workplan_id")}, + "owners": owners, + "gates": gates, + "artifacts": { + path: {"algorithm": "sha256", "digest": value} + for path, value in hashes.items() + }, + "allowed_dispositions": [ + item.replace("-", "_") for item in (raw.get("decisions") or []) + ], + "legacy": { + "interface": raw["interface"], + "version": 1, + "source_digest": digest, + }, + } + return validate_v1(normalized), digest + if raw.get("schema_version") != "review-contract/v1": + raise HTTPException(422, "schema_version must be review-contract/v1") + return validate_v1(raw), digest + + +def validate_v1(doc: dict[str, Any]) -> dict[str, Any]: + contract_key, subject = doc.get("contract_key"), doc.get("subject") + owners, gates, artifacts = doc.get("owners"), doc.get("gates"), doc.get("artifacts") + dispositions = doc.get("allowed_dispositions") + if not isinstance(contract_key, str) or not contract_key: + raise HTTPException(422, "contract_key is required") + if ( + not isinstance(subject, dict) + or not subject.get("kind") + or not subject.get("id") + ): + raise HTTPException(422, "typed subject kind and id are required") + if ( + not isinstance(owners, list) + or not owners + or not isinstance(gates, list) + or not gates + ): + raise HTTPException(422, "non-empty owners and gates are required") + if not isinstance(artifacts, dict) or not artifacts: + raise HTTPException(422, "revision-pinned artifacts are required") + if ( + set(dispositions or []) != {"approve", "request_changes"} + or len(dispositions) != 2 + ): + raise HTTPException( + 422, "allowed_dispositions must contain approve and request_changes only" + ) + owner_ids: set[str] = set() + for owner in owners: + if not isinstance(owner, dict): + raise HTTPException(422, "owner entries must be objects") + owner_id = owner.get("id") + artifact_ids = owner.get("artifact_ids") or [] + assertions = owner.get("assertions") or [] + checks = owner.get("check_ids") or [] + assertion_ids = [ + item.get("id") for item in assertions if isinstance(item, dict) + ] + if ( + not owner_id + or owner_id in owner_ids + or not artifact_ids + or not checks + or not assertion_ids + ): + raise HTTPException( + 422, + "each owner needs a unique id and non-empty artifact/assertion/check scope", + ) + if len(assertion_ids) != len(set(assertion_ids)) or any( + path not in artifacts for path in artifact_ids + ): + raise HTTPException(422, f"{owner_id}: invalid assertion or artifact scope") + owner_ids.add(owner_id) + for artifact_id, artifact in artifacts.items(): + if ( + not isinstance(artifact, dict) + or artifact.get("algorithm") != "sha256" + or not _is_sha256(artifact.get("digest")) + ): + raise HTTPException( + 422, f"{artifact_id}: only sha256 artifacts are supported" + ) + gate_ids: set[str] = set() + for gate in gates: + if not isinstance(gate, dict): + raise HTTPException(422, "gate entries must be objects") + required = gate.get("owners") or [] + if ( + gate.get("policy") != "all_required" + or not gate.get("id") + or gate["id"] in gate_ids + ): + raise HTTPException( + 422, "v1 gates require unique ids and all_required policy" + ) + if ( + not required + or len(required) != len(set(required)) + or not set(required) <= owner_ids + ): + raise HTTPException(422, f"{gate.get('id')}: gate owner set is invalid") + gate_ids.add(gate["id"]) + return doc + + +def _is_sha256(value: Any) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and all(c in "0123456789abcdef" for c in value) + ) + + +def owner_scope(contract: ReviewContract, owner_id: str) -> dict[str, Any]: + for owner in contract.document["owners"]: + if owner["id"] == owner_id: + return owner + raise HTTPException(422, f"unknown owner: {owner_id}") + + +def validate_receipt(contract: ReviewContract, body: Any) -> dict[str, Any]: + if body.contract_digest != contract.contract_digest: + raise HTTPException( + 409, "contract digest mismatch; re-review the active contract" + ) + scope = owner_scope(contract, body.owner_id) + if body.disposition == "request_changes": + if not body.note or not body.note.strip(): + raise HTTPException(422, "request_changes requires a note") + else: + expected_artifacts = { + key: contract.document["artifacts"][key]["digest"] + for key in scope["artifact_ids"] + } + if body.artifact_hashes != expected_artifacts: + raise HTTPException( + 422, "approval artifact hashes do not exactly match the owner scope" + ) + expected_assertions = {item["id"] for item in scope["assertions"]} + if set(body.assertion_ids) != expected_assertions or len( + body.assertion_ids + ) != len(expected_assertions): + raise HTTPException( + 422, "approval assertions do not exactly match the owner scope" + ) + expected_checks = set(scope["check_ids"]) + actual_checks = { + item.get("id") for item in body.checks if isinstance(item, dict) + } + if actual_checks != expected_checks or len(body.checks) != len(expected_checks): + raise HTTPException( + 422, "approval checks do not exactly match the owner scope" + ) + if any( + item.get("passed") is not True or item.get("read_only") is not True + for item in body.checks + ): + raise HTTPException( + 422, "every approval check must be passed and explicitly read_only" + ) + return { + "owner_id": body.owner_id, + "actor": body.actor, + "disposition": body.disposition, + "contract_digest": body.contract_digest, + "artifact_hashes": body.artifact_hashes, + "assertion_ids": body.assertion_ids, + "checks": body.checks, + "note": body.note, + "source": body.source.model_dump(), + } + + +async def active_contract(session: AsyncSession, contract_key: str) -> ReviewContract: + result = await session.execute( + select(ReviewContract).where( + ReviewContract.contract_key == contract_key, ReviewContract.active.is_(True) + ) + ) + contract = result.scalar_one_or_none() + if contract is None: + raise HTTPException(404, f"active review contract not found: {contract_key}") + return contract + + +async def aggregate( + session: AsyncSession, contract: ReviewContract +) -> ReviewAggregateRead: + rows = await session.execute( + select(ReviewReceipt) + .where(ReviewReceipt.contract_id == contract.id) + .order_by(ReviewReceipt.submitted_at.desc(), ReviewReceipt.id.desc()) + ) + latest: dict[str, ReviewReceipt] = {} + for receipt in rows.scalars(): + latest.setdefault(receipt.owner_id, receipt) + prior = await session.execute( + select(ReviewReceipt.owner_id) + .join(ReviewContract) + .where( + ReviewContract.contract_key == contract.contract_key, + ReviewContract.id != contract.id, + ) + .distinct() + ) + stale_owners = set(prior.scalars()) + owner_states, statuses = [], {} + for owner in contract.document["owners"]: + owner_id, receipt = owner["id"], latest.get(owner["id"]) + state = ( + receipt.disposition.replace("approve", "approved") + if receipt + else ("stale" if owner_id in stale_owners else "missing") + ) + statuses[owner_id] = state + owner_states.append( + OwnerReviewState( + owner_id=owner_id, + status=state, + receipt_id=receipt.id if receipt else None, + receipt_digest=receipt.receipt_digest if receipt else None, + submitted_at=receipt.submitted_at if receipt else None, + ) + ) + gates = [ + GateReviewState( + gate_id=gate["id"], + policy="all_required", + owners=gate["owners"], + satisfied=all(statuses[owner] == "approved" for owner in gate["owners"]), + ) + for gate in contract.document["gates"] + ] + return ReviewAggregateRead( + contract_key=contract.contract_key, + contract_digest=contract.contract_digest, + satisfied=all(gate.satisfied for gate in gates), + authorizes_execution=False, + owners=owner_states, + gates=gates, + ) + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/sbom_nexus.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/sbom_nexus.py new file mode 100644 index 00000000..229a43ec --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/sbom_nexus.py @@ -0,0 +1,70 @@ +"""Read-only client for the reversible SBOM Nexus compatibility facade.""" +from __future__ import annotations + +from typing import Any + +import httpx + +from api.config import settings + + +class SBOMNexusError(RuntimeError): + """An SBOM Nexus request failed without a usable compatibility response.""" + + def __init__(self, status_code: int, detail: str): + super().__init__(detail) + self.status_code = status_code + self.detail = detail + + +def reads_from_nexus() -> bool: + return settings.sbom_nexus_read_mode == "nexus" + + +def writes_to_nexus() -> bool: + return settings.sbom_nexus_write_mode == "nexus" + + +async def request_json( + method: str, + path: str, + *, + params: dict[str, Any] | None = None, + body: dict[str, Any] | None = None, +) -> Any: + """Call Nexus; never fall back silently to legacy storage.""" + if not settings.sbom_nexus_url: + raise SBOMNexusError(503, "SBOM Nexus read mode is enabled without SBOM_NEXUS_URL") + + try: + async with httpx.AsyncClient( + base_url=settings.sbom_nexus_url.rstrip("/"), + timeout=settings.sbom_nexus_timeout_seconds, + ) as client: + response = await client.request(method, path, params=params, json=body) + except httpx.RequestError as exc: + raise SBOMNexusError(502, f"SBOM Nexus is unavailable: {exc.__class__.__name__}") from exc + + if response.status_code >= 400: + detail = f"SBOM Nexus returned HTTP {response.status_code}" + try: + payload = response.json() + if isinstance(payload, dict) and isinstance(payload.get("detail"), str): + detail = payload["detail"] + except ValueError: + pass + status_code = 404 if response.status_code == 404 else 502 + raise SBOMNexusError(status_code, detail) + + try: + return response.json() + except ValueError as exc: + raise SBOMNexusError(502, "SBOM Nexus returned invalid JSON") from exc + + +async def get_json(path: str, *, params: dict[str, Any] | None = None) -> Any: + return await request_json("GET", path, params=params) + + +async def post_json(path: str, *, body: dict[str, Any]) -> Any: + return await request_json("POST", path, body=body) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/schema_state.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/schema_state.py new file mode 100644 index 00000000..24ba58a8 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/schema_state.py @@ -0,0 +1,63 @@ +"""Report whether the database schema matches the code (STATE-WP-0083-T07). + +Central ran two revisions behind the code it was serving, and `review_contracts` +did not exist there although its migration shipped inside the running image. +Nothing surfaced that: the API starts happily against a schema it was not built +for, and only fails when a request happens to touch the missing table. + +A hub that cannot say which schema it is running has the same problem as a +projection that cannot name its source commit — it is asserting correctness it +cannot demonstrate. +""" + +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path +from typing import Any + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +_MIGRATIONS = Path(__file__).resolve().parents[2] / "migrations" + + +@lru_cache(maxsize=1) +def code_head_revision() -> str | None: + """The head revision the shipped migration scripts define. + + Read from the migration files rather than the database: this is what the + code expects, and it must be knowable without a working connection. + """ + try: + from alembic.config import Config + from alembic.script import ScriptDirectory + + cfg = Config() + cfg.set_main_option("script_location", str(_MIGRATIONS)) + heads = ScriptDirectory.from_config(cfg).get_heads() + return heads[0] if len(heads) == 1 else ",".join(sorted(heads)) or None + except Exception: + return None + + +async def db_revision(session: AsyncSession) -> str | None: + try: + result = await session.execute(text("select version_num from alembic_version")) + return result.scalar() + except Exception: + return None + + +async def schema_state(session: AsyncSession) -> dict[str, Any]: + applied = await db_revision(session) + expected = code_head_revision() + # "unknown" is deliberately not "ok": an instance that cannot determine its + # own schema state must not report agreement it has not established. + if applied is None or expected is None: + status = "unknown" + elif applied == expected: + status = "ok" + else: + status = "behind" + return {"status": status, "applied": applied, "expected": expected} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/suggestion_relevance.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/suggestion_relevance.py new file mode 100644 index 00000000..4855dca1 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/suggestion_relevance.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import uuid +from datetime import datetime, timedelta, timezone + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from api.models.suggestion import ( + OPEN_SUGGESTION_STAGES, + Suggestion, + SuggestionRelevanceBump, + SuggestionStage, +) + +DEBOUNCE_SECONDS = 3600 + + +async def bump_relevance( + session: AsyncSession, + suggestion: Suggestion, + *, + source: str, + source_key: str, + reason: str | None = None, +) -> bool: + """Increment relevance when a suggestion is needed-but-unmet. + + Returns True when the counter was incremented, False when debounced. + """ + if suggestion.stage not in OPEN_SUGGESTION_STAGES: + return False + + cutoff = datetime.now(tz=timezone.utc) - timedelta(seconds=DEBOUNCE_SECONDS) + existing = ( + await session.execute( + select(SuggestionRelevanceBump.id) + .where(SuggestionRelevanceBump.suggestion_id == suggestion.id) + .where(SuggestionRelevanceBump.source == source) + .where(SuggestionRelevanceBump.source_key == source_key) + .where(SuggestionRelevanceBump.created_at >= cutoff) + .limit(1) + ) + ).scalar_one_or_none() + if existing is not None: + return False + + now = datetime.now(tz=timezone.utc) + suggestion.relevance += 1 + suggestion.relevance_events += 1 + suggestion.last_requested_at = now + session.add( + SuggestionRelevanceBump( + suggestion_id=suggestion.id, + source=source, + source_key=source_key, + reason=reason, + ) + ) + return True + + +async def bump_matching_for_capability_request( + session: AsyncSession, + *, + request_id: uuid.UUID, + title: str, + description: str, + capability_type: str, + catalog_entry_id: uuid.UUID | None = None, +) -> int: + """Bump open suggestions that match an unfulfilled capability need.""" + rows = ( + await session.execute( + select(Suggestion).where(Suggestion.stage.in_(OPEN_SUGGESTION_STAGES)) + ) + ).scalars().all() + + haystack = f"{title} {description} {capability_type}".lower() + bumped = 0 + for suggestion in rows: + matched = False + if suggestion.origin_ref: + ref = suggestion.origin_ref.lower() + if ref in haystack or any(token in haystack for token in ref.split("-") if len(token) > 4): + matched = True + if not matched and suggestion.title.lower() in haystack: + matched = True + if not matched: + continue + if await bump_relevance( + session, + suggestion, + source="capability_request", + source_key=str(request_id), + reason=f"Capability request matched: {title}", + ): + bumped += 1 + return bumped + + +async def bump_suggestions_for_next_steps( + session: AsyncSession, + suggestions: list[Suggestion], +) -> int: + """Bump only suggestions surfaced in a next-steps response.""" + bucket = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H") + bumped = 0 + for suggestion in suggestions: + if await bump_relevance( + session, + suggestion, + source="next_steps", + source_key=bucket, + reason="Surfaced during get_next_steps lookup", + ): + bumped += 1 + return bumped \ No newline at end of file diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/suggestion_wsjf.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/suggestion_wsjf.py new file mode 100644 index 00000000..39df65de --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/suggestion_wsjf.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from api.models.suggestion import Suggestion + + +def cost_of_delay(suggestion: Suggestion) -> float: + return suggestion.base_value + (suggestion.relevance_weight * suggestion.relevance) + + +def compute_wsjf(suggestion: Suggestion) -> float: + size = suggestion.job_size if suggestion.job_size > 0 else 3.0 + return round(cost_of_delay(suggestion) / size, 2) + + +def suggestion_sort_key(suggestion: Suggestion) -> tuple[float, int, str]: + """Higher WSJF first; tie-break on relevance then title.""" + return (-compute_wsjf(suggestion), -suggestion.relevance, suggestion.title.lower()) \ No newline at end of file diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/summary_cache.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/summary_cache.py new file mode 100644 index 00000000..55c6e29d --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/summary_cache.py @@ -0,0 +1,298 @@ +"""Revision-gated cache for ``GET /state/summary`` with stale-while-revalidate.""" +from __future__ import annotations + +import asyncio +import logging +from dataclasses import dataclass +from datetime import datetime, timezone +from collections.abc import Awaitable, Callable +from typing import Literal + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import noload + +from api.models.capability_request import CapabilityRequest +from api.models.contribution import Contribution +from api.models.decision import Decision +from api.models.domain import Domain +from api.models.extension_point import ExtensionPoint +from api.models.managed_repo import ManagedRepo +from api.models.progress_event import ProgressEvent +from api.models.sbom_snapshot import SBOMSnapshot +from api.models.task import Task +from api.models.technical_debt import TechnicalDebt +from api.models.topic import Topic +from api.models.workplan import Workplan +from api.models.workplan_dependency import WorkplanDependency +from api.schemas.progress_event import ProgressEventRead +from api.schemas.state import StateSummary +from api.services.sbom_nexus import get_json as get_sbom_nexus_json +from api.services.sbom_nexus import reads_from_nexus + +logger = logging.getLogger(__name__) + +_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) +_MAX_STALE_AGE_SECONDS = 300.0 +InvalidateScope = Literal["all", "core", "progress"] +CacheStatus = Literal["hit-revision", "stale", "miss", "progress-section"] +BuildSummaryFn = Callable[[AsyncSession], Awaitable[StateSummary]] + +# Tables feeding the stable (non-progress) summary core. +_CORE_TABLES: tuple[tuple[str, type], ...] = ( + ("topics", Topic), + ("workplans", Workplan), + ("tasks", Task), + ("decisions", Decision), + ("workplan_dependencies", WorkplanDependency), + ("managed_repos", ManagedRepo), + ("contributions", Contribution), + ("capability_requests", CapabilityRequest), + ("domains", Domain), + ("extension_points", ExtensionPoint), + ("technical_debt", TechnicalDebt), +) + + +@dataclass(frozen=True) +class SummaryRevision: + """Cheap fingerprints of hub data that affect ``/state/summary``.""" + + core: datetime + progress: datetime | None + sbom: datetime | None + + def core_fingerprint(self) -> str: + return _fingerprint(self.core, self.sbom) + + def progress_fingerprint(self) -> str: + return self.progress.isoformat() if self.progress else "" + + def combined_fingerprint(self) -> str: + return f"{self.core_fingerprint()}|{self.progress_fingerprint()}" + + +def _fingerprint(*parts: datetime | None) -> str: + normalized = [ + (part or _EPOCH).astimezone(timezone.utc).isoformat() + for part in parts + ] + return "|".join(normalized) + + +async def fetch_summary_revision(session: AsyncSession) -> SummaryRevision: + """Return per-section revision watermarks (indexed MAX scans).""" + core_parts: list[datetime] = [] + for _name, model in _CORE_TABLES: + value = ( + await session.execute(select(func.max(model.updated_at))) + ).scalar_one_or_none() + if value is not None: + core_parts.append(value) + + if reads_from_nexus(): + snapshots = await get_sbom_nexus_json("/sbom/snapshots/") + snapshot_times = [ + datetime.fromisoformat(item["snapshot_at"].replace("Z", "+00:00")) + for item in snapshots + ] + sbom_at = max(snapshot_times, default=None) + else: + sbom_at = ( + await session.execute(select(func.max(SBOMSnapshot.snapshot_at))) + ).scalar_one_or_none() + + progress_at = ( + await session.execute(select(func.max(ProgressEvent.created_at))) + ).scalar_one_or_none() + + core = max(core_parts, default=_EPOCH) + if sbom_at is not None and sbom_at > core: + core = sbom_at + + return SummaryRevision(core=core, progress=progress_at, sbom=sbom_at) + + +async def fetch_recent_progress(session: AsyncSession, *, limit: int = 20) -> list[ProgressEventRead]: + rows = await session.execute( + select(ProgressEvent) + .options(noload("*")) + .order_by(ProgressEvent.created_at.desc()) + .limit(limit) + ) + return [ProgressEventRead.model_validate(event) for event in rows.scalars().all()] + + +def merge_summary(core: StateSummary, recent_progress: list[ProgressEventRead]) -> StateSummary: + return core.model_copy(update={"recent_progress": recent_progress}) + + +@dataclass +class _CacheEntry: + summary: StateSummary + core_revision: str + progress_revision: str + built_at: float + + +class SummaryCache: + def __init__(self) -> None: + self._entry: _CacheEntry | None = None + self._refresh_task: asyncio.Task | None = None + self.last_error: str | None = None + self._build_fn: BuildSummaryFn | None = None + + def configure(self, build_fn: BuildSummaryFn) -> None: + self._build_fn = build_fn + + def reset(self) -> None: + self._entry = None + self.last_error = None + if self._refresh_task is not None and not self._refresh_task.done(): + self._refresh_task.cancel() + self._refresh_task = None + + def invalidate(self, scope: InvalidateScope = "all") -> None: + if scope == "all" or self._entry is None: + self.reset() + return + if scope == "core": + self.reset() + elif scope == "progress": + self._entry.progress_revision = "__invalid__" + + def store(self, summary: StateSummary, revision: SummaryRevision) -> None: + import time + + self._entry = _CacheEntry( + summary=summary, + core_revision=revision.core_fingerprint(), + progress_revision=revision.progress_fingerprint(), + built_at=time.monotonic(), + ) + self.last_error = None + + def _entry_age(self) -> float | None: + import time + + if self._entry is None: + return None + return time.monotonic() - self._entry.built_at + + def _entry_matches(self, revision: SummaryRevision) -> tuple[bool, bool]: + if self._entry is None: + return False, False + core_match = self._entry.core_revision == revision.core_fingerprint() + progress_match = self._entry.progress_revision == revision.progress_fingerprint() + return core_match, progress_match + + def resolve( + self, + revision: SummaryRevision, + *, + force_refresh: bool, + ) -> tuple[CacheStatus, StateSummary | None]: + import time + + if force_refresh: + return "miss", None + + if self._entry is None: + return "miss", None + + age = self._entry_age() + if age is not None and age > _MAX_STALE_AGE_SECONDS: + return "miss", None + + core_match, progress_match = self._entry_matches(revision) + if core_match and progress_match: + return "hit-revision", self._entry.summary + + if core_match and not progress_match: + return "progress-section", self._entry.summary + + # Core changed — serve stale full snapshot while refreshing. + return "stale", self._entry.summary + + def schedule_refresh(self, revision: SummaryRevision) -> None: + if self._build_fn is None: + return + if self._refresh_task is not None and not self._refresh_task.done(): + return + self._refresh_task = asyncio.create_task( + self._refresh_background(revision), + name="summary-cache-refresh", + ) + + async def _refresh_background(self, revision: SummaryRevision) -> None: + from api.database import async_session_factory + + if self._build_fn is None: + return + try: + async with async_session_factory() as session: + current = await fetch_summary_revision(session) + summary = await self._build_fn(session) + self.store(summary, current) + except Exception as exc: + self.last_error = str(exc) + logger.exception("summary cache background refresh failed") + + +_summary_cache = SummaryCache() + + +def get_summary_cache() -> SummaryCache: + return _summary_cache + + +def invalidate_summary_cache(scope: InvalidateScope = "all") -> None: + _summary_cache.invalidate(scope) + + +def reset_summary_cache_for_tests() -> None: + _summary_cache.reset() + + +_INVALIDATION_REGISTERED = False + + +def register_summary_cache_invalidation() -> None: + """Clear summary cache when ORM rows that affect summary are written.""" + global _INVALIDATION_REGISTERED + if _INVALIDATION_REGISTERED: + return + _INVALIDATION_REGISTERED = True + + from sqlalchemy import event + + def _invalidate_core(*_args: object, **_kwargs: object) -> None: + invalidate_summary_cache("core") + + def _invalidate_progress(*_args: object, **_kwargs: object) -> None: + invalidate_summary_cache("progress") + + for _name, model in _CORE_TABLES: + event.listen(model, "after_insert", _invalidate_core) + event.listen(model, "after_update", _invalidate_core) + event.listen(model, "after_delete", _invalidate_core) + + event.listen(SBOMSnapshot, "after_insert", _invalidate_core) + event.listen(SBOMSnapshot, "after_delete", _invalidate_core) + event.listen(ProgressEvent, "after_insert", _invalidate_progress) + + +async def apply_progress_section( + session: AsyncSession, + summary: StateSummary, + revision: SummaryRevision, +) -> StateSummary: + recent = await fetch_recent_progress(session) + merged = merge_summary(summary, recent) + cache = get_summary_cache() + if cache._entry is not None and cache._entry.core_revision == revision.core_fingerprint(): + cache._entry.summary = merged + cache._entry.progress_revision = revision.progress_fingerprint() + else: + cache.store(merged, revision) + return merged diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/task_record_id_backfill.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/task_record_id_backfill.py new file mode 100644 index 00000000..c2ff3c49 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/task_record_id_backfill.py @@ -0,0 +1,214 @@ +"""Backfill canonical record ids onto existing task rows (STATE-WP-0083-T06). + +Only the repository files hold the mapping. A file task declares both its +canonical id and the projection UUID it was registered under: + + ```task + id: CUST-WP-0067-T01 + state_hub_task_id: "f3608db4-..." + ``` + +so the pairing can be read directly rather than guessed from titles. Anything a +file does not claim is left alone: a task row whose canonical id cannot be +established keeps `record_id` null, and the reset continues to refuse to act on +it. An unknown identity must stay unknown rather than be inferred. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +_TASK_BLOCK_RE = re.compile(r"```task\s*\n(.*?)\n```", re.DOTALL) +_ID_RE = re.compile(r"^id:\s*(\S+)", re.MULTILINE) +_UUID_RE = re.compile(r'state_hub_task_id:\s*"?([0-9a-f-]{36})"?') +_FRONTMATTER_ID_RE = re.compile(r"^id:\s*(\S+)", re.MULTILINE) +# A task id written as a bare "T01" is unique only inside its own workplan. +# Storing it as a canonical identifier makes every workplan's first task share +# one identity — 51 such ids were assigned to 148 rows before this was caught. +_SHORT_TASK_ID_RE = re.compile(r"^T\d+$", re.IGNORECASE) + + +def qualify_task_id(task_id: str, workplan_id: str | None) -> str | None: + """Return a canonical task id, or None when identity cannot be established.""" + task_id = task_id.strip() + if not _SHORT_TASK_ID_RE.fullmatch(task_id): + return task_id + if not workplan_id: + # Unqualifiable: leaving it unidentified is correct, since an identity + # that is not unique is worse than none. + return None + return f"{workplan_id.strip()}-{task_id.upper()}" + + +@dataclass +class BackfillReport: + scanned_files: int = 0 + pairs_found: int = 0 + updated: int = 0 + already_set: int = 0 + conflicts: list[dict[str, str]] = field(default_factory=list) + unmatched_uuids: int = 0 + + def to_dict(self) -> dict[str, Any]: + return { + "schema": "state-hub.task-record-id-backfill.v1", + "scanned_files": self.scanned_files, + "pairs_found": self.pairs_found, + "updated": self.updated, + "already_set": self.already_set, + "unmatched_uuids": self.unmatched_uuids, + "conflicts": self.conflicts, + } + + +def collect_pairs(roots: list[Path]) -> tuple[dict[str, str], BackfillReport]: + """Map projection UUID -> canonical record id, from workplan files.""" + report = BackfillReport() + pairs: dict[str, str] = {} + for root in roots: + wp_dir = root / "workplans" + if not wp_dir.is_dir(): + continue + for path in sorted(wp_dir.rglob("*.md")): + if path.name.startswith("."): + continue + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + report.scanned_files += 1 + head = text.split("---", 2)[1] if text.startswith("---") and text.count("---") >= 2 else "" + wp_match = _FRONTMATTER_ID_RE.search(head) + workplan_id = wp_match.group(1).strip() if wp_match else None + for block in _TASK_BLOCK_RE.finditer(text): + body = block.group(1) + rid = _ID_RE.search(body) + uid = _UUID_RE.search(body) + if not rid or not uid: + continue + record_id = qualify_task_id(rid.group(1), workplan_id) + if record_id is None: + continue + task_uuid = uid.group(1) + prior = pairs.get(task_uuid) + if prior and prior != record_id: + # One UUID claimed by two canonical ids: a duplicate + # registration. Recording it and skipping is the only safe + # option — picking one would fabricate an identity. + report.conflicts.append( + {"uuid": task_uuid, "first": prior, "second": record_id} + ) + continue + pairs[task_uuid] = record_id + report.pairs_found = len(pairs) + return pairs, report + + +async def backfill_task_record_ids( + session: Any, roots: list[Path], *, dry_run: bool = True +) -> BackfillReport: + from sqlalchemy import select + + from api.models.task import Task + + pairs, report = collect_pairs(roots) + if not pairs: + return report + + rows = list((await session.execute(select(Task))).scalars()) + by_id = {str(r.id): r for r in rows} + for task_uuid, record_id in pairs.items(): + row = by_id.get(task_uuid) + if row is None: + report.unmatched_uuids += 1 + continue + if row.record_id == record_id: + report.already_set += 1 + continue + if row.record_id and row.record_id != record_id: + report.conflicts.append( + {"uuid": task_uuid, "first": row.record_id, "second": record_id} + ) + continue + report.updated += 1 + if not dry_run: + row.record_id = record_id + return report + + +async def backfill_from_forge( + session: Any, + repo_slugs: list[str], + *, + forge_base: str | None = None, + dry_run: bool = True, +) -> BackfillReport: + """Backfill from repositories cloned out of the forge. + + The local-path variant above needs a workstation checkout, which central + does not have and should not depend on: `ADR-012` decision 1 makes the forge + the projection source, and a backfill sourced from someone's laptop would + reintroduce exactly the coupling that ADR removes. + + Central can clone the forge directly, so it reads the pairing from the same + place it derives everything else. + """ + import tempfile + + from api.services.forge_projection import ( + DEFAULT_FORGE_BASE, + ForgeDeriveError, + _run_git, + ) + + base = forge_base or DEFAULT_FORGE_BASE + report = BackfillReport() + pairs: dict[str, str] = {} + + for slug in repo_slugs: + url = f"{base.rstrip('/')}/{slug}.git" + with tempfile.TemporaryDirectory(prefix=f"backfill-{slug}-") as tmp: + try: + _run_git("clone", "--depth", "1", "--quiet", url, tmp) + except (ForgeDeriveError, Exception): + # A repository that cannot be read contributes nothing. It must + # not silently reduce what the rest can identify. + continue + repo_pairs, repo_report = collect_pairs([Path(tmp)]) + report.scanned_files += repo_report.scanned_files + report.conflicts.extend(repo_report.conflicts) + for uid, rid in repo_pairs.items(): + prior = pairs.get(uid) + if prior and prior != rid: + report.conflicts.append({"uuid": uid, "first": prior, "second": rid}) + continue + pairs[uid] = rid + + report.pairs_found = len(pairs) + if not pairs: + return report + + from sqlalchemy import select + + from api.models.task import Task + + rows = list((await session.execute(select(Task))).scalars()) + by_id = {str(r.id): r for r in rows} + for uid, rid in pairs.items(): + row = by_id.get(uid) + if row is None: + report.unmatched_uuids += 1 + continue + if row.record_id == rid: + report.already_set += 1 + continue + if row.record_id and row.record_id != rid: + report.conflicts.append({"uuid": uid, "first": row.record_id, "second": rid}) + continue + report.updated += 1 + if not dry_run: + row.record_id = rid + return report diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/__init__.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/__init__.py new file mode 100644 index 00000000..f0e95f54 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/__init__.py @@ -0,0 +1,16 @@ +"""Token source adapters for measured agent usage.""" + +from api.services.token_sources.base import TokenSourceRecord, parse_iso +from api.services.token_sources.codex import collect_codex_sessions, iter_codex_session_files, parse_codex_session +from api.services.token_sources.claude import collect_claude_transcripts, iter_claude_transcript_files, parse_claude_transcript + +__all__ = [ + "TokenSourceRecord", + "parse_iso", + "collect_codex_sessions", + "iter_codex_session_files", + "parse_codex_session", + "collect_claude_transcripts", + "iter_claude_transcript_files", + "parse_claude_transcript", +] diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/attribution.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/attribution.py new file mode 100644 index 00000000..2a9eec45 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/attribution.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class RepoRef: + repo_id: str + slug: str + local_path: str | None = None + host_paths: dict[str, Any] | None = None + remote_url: str | None = None + git_fingerprint: str | None = None + + +@dataclass(frozen=True) +class RepoMatch: + repo_id: str + slug: str + method: str + confidence: float + + +def normalise_cwd(raw: str | None) -> str | None: + if not raw: + return None + value = raw.replace("\\", "/") + prefixes = ( + "//wsl.localhost/Ubuntu-24.04", + "//wsl$/Ubuntu-24.04", + ) + for prefix in prefixes: + if value.startswith(prefix): + return value[len(prefix):] or "/" + if len(value) >= 3 and value[1:3] == ":/": + drive = value[0].lower() + return f"/mnt/{drive}{value[2:]}" + return value + + +def normalise_remote_url(raw: str | None) -> str | None: + if not raw: + return None + value = raw.strip() + if value.endswith(".git"): + value = value[:-4] + if value.startswith("git@") and ":" in value: + host, path = value[4:].split(":", 1) + value = f"ssh://{host}/{path}" + return value.lower().rstrip("/") + + +def repo_refs_from_api(repos: list[dict[str, Any]]) -> list[RepoRef]: + refs = [] + for repo in repos: + repo_id = repo.get("id") + slug = repo.get("slug") + if not repo_id or not slug: + continue + refs.append( + RepoRef( + repo_id=str(repo_id), + slug=str(slug), + local_path=repo.get("local_path"), + host_paths=repo.get("host_paths") if isinstance(repo.get("host_paths"), dict) else {}, + remote_url=repo.get("remote_url"), + git_fingerprint=repo.get("git_fingerprint"), + ) + ) + return refs + + +def _git(cwd: str, *args: str) -> str | None: + try: + result = subprocess.run( + ["git", *args], + cwd=cwd, + check=False, + capture_output=True, + text=True, + timeout=5, + ) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0: + return None + value = result.stdout.strip().splitlines() + return value[0] if value else None + + +def git_fingerprint_for_path(cwd: str | None) -> str | None: + path = normalise_cwd(cwd) + if not path or not Path(path).exists(): + return None + root = _git(path, "rev-parse", "--show-toplevel") + if not root: + return None + return _git(root, "rev-list", "--max-parents=0", "HEAD") + + +def git_remote_for_path(cwd: str | None) -> str | None: + path = normalise_cwd(cwd) + if not path or not Path(path).exists(): + return None + root = _git(path, "rev-parse", "--show-toplevel") + if not root: + return None + return _git(root, "remote", "get-url", "origin") + + +def _repo_paths(repo: RepoRef) -> list[str]: + paths = [repo.local_path] + if repo.host_paths: + paths.extend(str(v) for v in repo.host_paths.values() if v) + result = [] + for raw in paths: + path = normalise_cwd(str(raw)) if raw and raw != "(unknown)" else None + if path: + result.append(path.rstrip("/")) + return result + + +def resolve_repo(cwd: str | None, repos: list[RepoRef]) -> RepoMatch | None: + path = normalise_cwd(cwd) + fingerprint = git_fingerprint_for_path(path) + remote = normalise_remote_url(git_remote_for_path(path)) + + if fingerprint: + candidates = [repo for repo in repos if repo.git_fingerprint == fingerprint] + if len(candidates) == 1: + repo = candidates[0] + return RepoMatch(repo.repo_id, repo.slug, "git_fingerprint", 0.98) + if remote: + remote_candidates = [ + repo for repo in candidates + if normalise_remote_url(repo.remote_url) == remote + ] + if len(remote_candidates) == 1: + repo = remote_candidates[0] + return RepoMatch(repo.repo_id, repo.slug, "git_fingerprint_remote", 0.99) + + if remote: + candidates = [repo for repo in repos if normalise_remote_url(repo.remote_url) == remote] + if len(candidates) == 1: + repo = candidates[0] + return RepoMatch(repo.repo_id, repo.slug, "remote_url", 0.90) + + if not path: + return None + + path_matches: list[tuple[str, RepoRef]] = [] + for repo in repos: + for repo_path in _repo_paths(repo): + if path == repo_path or path.startswith(f"{repo_path}/"): + path_matches.append((repo_path, repo)) + if not path_matches: + return None + path_matches.sort(key=lambda item: len(item[0]), reverse=True) + exact = [item for item in path_matches if path == item[0]] + if exact: + basename = Path(path).name + for _, repo in exact: + if repo.slug == basename: + return RepoMatch(repo.repo_id, repo.slug, "path_exact_slug", 0.85) + repo = exact[0][1] + return RepoMatch(repo.repo_id, repo.slug, "path_exact", 0.80) + repo = path_matches[0][1] + return RepoMatch(repo.repo_id, repo.slug, "path_prefix", 0.75) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/base.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/base.py new file mode 100644 index 00000000..77a13928 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/base.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +def parse_iso(value: str) -> datetime: + raw = value.strip() + if raw.endswith("Z"): + raw = raw[:-1] + "+00:00" + if "T" not in raw: + raw = f"{raw}T00:00:00+00:00" + parsed = datetime.fromisoformat(raw) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +@dataclass +class TokenSourceRecord: + source_provider: str + source_id: str + source_path: Path + source_created_at: datetime | None + session_id: str | None = None + cwd: str | None = None + model: str | None = None + agent: str | None = None + tokens_in: int = 0 + tokens_out: int = 0 + cached_input_tokens: int = 0 + reasoning_output_tokens: int = 0 + raw_total_tokens: int | None = None + parser_version: str | None = None + confidence: float = 1.0 + raw_metadata: dict[str, Any] = field(default_factory=dict) + + @property + def tokens_total(self) -> int: + return self.tokens_in + self.tokens_out + + def to_token_event_payload(self, repo_id: str | None = None) -> dict[str, Any]: + raw_total = self.raw_total_tokens + if raw_total is None: + raw_total = self.tokens_in + self.tokens_out + created_at = self.source_created_at.isoformat() if self.source_created_at else None + return { + "tokens_in": self.tokens_in, + "tokens_out": self.tokens_out, + "repo_id": repo_id, + "session_id": self.session_id, + "model": self.model, + "agent": self.agent, + "ref_type": "session", + "ref_id": self.source_id, + "note": f"measured:{self.source_provider}", + "created_at": created_at, + "measurement_kind": "measured", + "source_provider": self.source_provider, + "source_id": self.source_id, + "source_path": str(self.source_path), + "source_created_at": created_at, + "parser_version": self.parser_version, + "confidence": self.confidence, + "cached_input_tokens": self.cached_input_tokens, + "reasoning_output_tokens": self.reasoning_output_tokens, + "raw_total_tokens": raw_total, + "raw_metadata": self.raw_metadata, + } diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/claude.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/claude.py new file mode 100644 index 00000000..e97acc2b --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/claude.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import json +from datetime import datetime +from pathlib import Path +from typing import Any + +from api.services.token_sources.base import TokenSourceRecord, parse_iso + +PARSER_VERSION = "claude-transcript-v1" + + +def iter_claude_transcript_files(claude_home: Path) -> list[Path]: + projects = claude_home / "projects" + if not projects.is_dir(): + return [] + return sorted(projects.glob("**/*.jsonl")) + + +def _usage_from_entry(entry: dict[str, Any]) -> dict[str, Any]: + message = entry.get("message") + if isinstance(message, dict) and isinstance(message.get("usage"), dict): + return message["usage"] + usage = entry.get("usage") + return usage if isinstance(usage, dict) else {} + + +def parse_claude_transcript(path: Path, since: datetime) -> TokenSourceRecord | None: + session_id = path.stem + cwd: str | None = None + model: str | None = None + first_at: datetime | None = None + last_at: datetime | None = None + tokens_in = tokens_out = 0 + cached_input_tokens = 0 + raw_total_tokens = 0 + usage_records = 0 + malformed_lines = 0 + + try: + handle = path.open("r", encoding="utf-8", errors="ignore") + except OSError: + return None + + with handle: + for line in handle: + try: + entry: dict[str, Any] = json.loads(line) + except json.JSONDecodeError: + malformed_lines += 1 + continue + + ts = entry.get("timestamp") or entry.get("created_at") + parsed_ts = parse_iso(ts) if isinstance(ts, str) else None + if parsed_ts: + first_at = first_at or parsed_ts + last_at = parsed_ts + + session_id = str(entry.get("session_id") or entry.get("conversation_id") or session_id) + cwd = entry.get("cwd") or entry.get("project_cwd") or cwd + model = entry.get("model") or model + message = entry.get("message") + if isinstance(message, dict): + model = message.get("model") or model + + usage = _usage_from_entry(entry) + if not usage: + continue + if parsed_ts is not None and parsed_ts < since: + continue + + input_tokens = int(usage.get("input_tokens") or 0) + cache_creation = int(usage.get("cache_creation_input_tokens") or 0) + cache_read = int(usage.get("cache_read_input_tokens") or 0) + output_tokens = int(usage.get("output_tokens") or 0) + if input_tokens == 0 and output_tokens == 0 and cache_creation == 0 and cache_read == 0: + continue + tokens_in += input_tokens + tokens_out += output_tokens + cached_input_tokens += cache_creation + cache_read + raw_total_tokens += input_tokens + cache_creation + cache_read + output_tokens + usage_records += 1 + + if usage_records == 0 or tokens_in + tokens_out + cached_input_tokens == 0: + return None + + return TokenSourceRecord( + source_provider="claude_transcript", + source_id=f"claude:{session_id}", + source_path=path, + source_created_at=last_at, + session_id=session_id, + cwd=cwd, + model=model, + agent="claude", + tokens_in=tokens_in, + tokens_out=tokens_out, + cached_input_tokens=cached_input_tokens, + raw_total_tokens=raw_total_tokens or None, + parser_version=PARSER_VERSION, + confidence=1.0, + raw_metadata={ + "started_at": first_at.isoformat() if first_at else None, + "usage_records": usage_records, + "malformed_lines": malformed_lines, + "source_file_name": path.name, + }, + ) + + +def collect_claude_transcripts(claude_home: Path, since: datetime) -> list[TokenSourceRecord]: + by_id: dict[str, TokenSourceRecord] = {} + for path in iter_claude_transcript_files(claude_home): + parsed = parse_claude_transcript(path, since) + if parsed is None: + continue + current = by_id.get(parsed.source_id) + if current is None or parsed.tokens_total > current.tokens_total: + by_id[parsed.source_id] = parsed + return sorted(by_id.values(), key=lambda item: item.source_created_at or datetime.min.replace(tzinfo=since.tzinfo)) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/codex.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/codex.py new file mode 100644 index 00000000..991f7a23 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/token_sources/codex.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import json +from datetime import datetime +from pathlib import Path +from typing import Any + +from api.services.token_sources.base import TokenSourceRecord, parse_iso + +PARSER_VERSION = "codex-desktop-v1" + + +def iter_codex_session_files(codex_home: Path) -> list[Path]: + files: list[Path] = [] + sessions = codex_home / "sessions" + archived = codex_home / "archived_sessions" + if sessions.is_dir(): + files.extend(sorted(sessions.glob("*/*/*/*.jsonl"))) + if archived.is_dir(): + files.extend(sorted(archived.glob("*.jsonl"))) + return files + + +def parse_codex_session(path: Path, since: datetime) -> TokenSourceRecord | None: + fallback_id = path.stem.removeprefix("rollout-") + session_id = fallback_id + started_at: datetime | None = None + last_at: datetime | None = None + cwd: str | None = None + model: str | None = None + tokens_in = tokens_out = 0 + cached_input_tokens = reasoning_output_tokens = 0 + raw_total_tokens = 0 + usage_records = 0 + malformed_lines = 0 + + try: + handle = path.open("r", encoding="utf-8", errors="ignore") + except OSError: + return None + + with handle: + for line in handle: + try: + entry: dict[str, Any] = json.loads(line) + except json.JSONDecodeError: + malformed_lines += 1 + continue + + ts = entry.get("timestamp") + parsed_ts = parse_iso(ts) if isinstance(ts, str) else None + if parsed_ts: + last_at = parsed_ts + started_at = started_at or parsed_ts + + payload = entry.get("payload") or {} + if entry.get("type") == "session_meta": + meta_id = payload.get("id") + if meta_id: + session_id = str(meta_id) + cwd = payload.get("cwd") or cwd + meta_ts = payload.get("timestamp") + if isinstance(meta_ts, str): + started_at = parse_iso(meta_ts) + elif entry.get("type") == "turn_context": + cwd = payload.get("cwd") or cwd + model = payload.get("model") or model + elif entry.get("type") == "event_msg" and payload.get("type") == "token_count": + if parsed_ts is None or parsed_ts < since: + continue + info = payload.get("info") or {} + last = info.get("last_token_usage") or {} + if not isinstance(last, dict): + continue + input_tokens = int(last.get("input_tokens") or 0) + output_tokens = int(last.get("output_tokens") or 0) + if input_tokens == 0 and output_tokens == 0: + continue + tokens_in += input_tokens + tokens_out += output_tokens + cached_input_tokens += int(last.get("cached_input_tokens") or 0) + reasoning_output_tokens += int(last.get("reasoning_output_tokens") or 0) + raw_total_tokens += int(last.get("total_tokens") or input_tokens + output_tokens) + usage_records += 1 + last_at = parsed_ts + + if usage_records == 0 or tokens_in + tokens_out == 0: + return None + + return TokenSourceRecord( + source_provider="codex_session", + source_id=f"codex:{session_id}", + source_path=path, + source_created_at=last_at, + session_id=session_id, + cwd=cwd, + model=model, + agent="codex", + tokens_in=tokens_in, + tokens_out=tokens_out, + cached_input_tokens=cached_input_tokens, + reasoning_output_tokens=reasoning_output_tokens, + raw_total_tokens=raw_total_tokens or None, + parser_version=PARSER_VERSION, + confidence=1.0, + raw_metadata={ + "started_at": started_at.isoformat() if started_at else None, + "usage_records": usage_records, + "malformed_lines": malformed_lines, + "source_file_name": path.name, + }, + ) + + +def collect_codex_sessions(codex_home: Path, since: datetime) -> list[TokenSourceRecord]: + by_id: dict[str, TokenSourceRecord] = {} + for path in iter_codex_session_files(codex_home): + parsed = parse_codex_session(path, since) + if parsed is None: + continue + current = by_id.get(parsed.source_id) + if current is None or parsed.tokens_total > current.tokens_total: + by_id[parsed.source_id] = parsed + return sorted(by_id.values(), key=lambda item: item.source_created_at or datetime.min.replace(tzinfo=since.tzinfo)) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/wait_kind.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/wait_kind.py new file mode 100644 index 00000000..28ab11a3 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/wait_kind.py @@ -0,0 +1,138 @@ +"""Derived wait qualifiers (CUST-WP-0074 rule 5) — read model only. + +A ``wait`` task is *external* when it carries at least one dependency edge +(``workplan_dependencies.from_task_id``), an owner-return marker +(``blocked_on: message-from:``, rule 6) or a ``wait_until`` date (a +time window clears by itself); *human* when ``needs_human`` is set, *both* +when both hold and *unqualified* otherwise. A ``blocked`` workplan is +*human* when any of its wait tasks is human or both, else *external* when any +is external, else *none*. Nothing here is stored; the routers call the +``annotate_*`` helpers and Pydantic picks the attributes up on serialisation. +""" +from __future__ import annotations + +import re +import uuid +from collections.abc import Iterable, Sequence +from datetime import date +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from api.models.task import Task, TaskStatus +from api.models.workplan import Workplan +from api.models.workplan_dependency import WorkplanDependency +from api.task_status import normalize_task_status +from api.workplan_status import normalize_workplan_status + +WAIT_KIND_EXTERNAL = "external" +WAIT_KIND_HUMAN = "human" +WAIT_KIND_BOTH = "both" +WAIT_KIND_UNQUALIFIED = "unqualified" + +BLOCKED_KIND_EXTERNAL = "external" +BLOCKED_KIND_HUMAN = "human" +BLOCKED_KIND_NONE = "none" + +# Rule 6 owner-return marker; mirrors _BLOCKED_ON_RE in scripts/consistency_check.py. +MESSAGE_FROM_RE = re.compile(r"^message-from:.+$") + + +def is_external_wait( + *, + has_dependency: bool, + blocked_on: str | None = None, + wait_until: date | None = None, +) -> bool: + """Case A holds for a dependency edge, an owner-return marker or a wait window.""" + if has_dependency or wait_until is not None: + return True + return bool(blocked_on and MESSAGE_FROM_RE.match(blocked_on.strip())) + + +def derive_wait_kind( + status: Any, + *, + needs_human: bool, + has_dependency: bool, + blocked_on: str | None = None, + wait_until: date | None = None, +) -> str | None: + """Rule 1 / rule 5 for one task. ``None`` unless the task is waiting.""" + if normalize_task_status(status, default="todo") != "wait": + return None + external = is_external_wait( + has_dependency=has_dependency, blocked_on=blocked_on, wait_until=wait_until + ) + if external and needs_human: + return WAIT_KIND_BOTH + if needs_human: + return WAIT_KIND_HUMAN + if external: + return WAIT_KIND_EXTERNAL + return WAIT_KIND_UNQUALIFIED + + +def derive_blocked_kind(status: Any, wait_kinds: Iterable[str | None]) -> str | None: + """Rule 5 for one workplan. ``None`` unless the workplan is blocked.""" + if normalize_workplan_status(status) != "blocked": + return None + kinds = {kind for kind in wait_kinds if kind} + if WAIT_KIND_HUMAN in kinds or WAIT_KIND_BOTH in kinds: + return BLOCKED_KIND_HUMAN + if WAIT_KIND_EXTERNAL in kinds: + return BLOCKED_KIND_EXTERNAL + return BLOCKED_KIND_NONE + + +async def task_ids_with_dependencies( + session: AsyncSession, task_ids: Sequence[uuid.UUID] +) -> set[uuid.UUID]: + if not task_ids: + return set() + rows = await session.execute( + select(WorkplanDependency.from_task_id).where( + WorkplanDependency.from_task_id.in_(list(task_ids)) + ) + ) + return {row[0] for row in rows if row[0] is not None} + + +async def annotate_tasks(session: AsyncSession, tasks: Sequence[Task]) -> Sequence[Task]: + """Set ``task.wait_kind`` on each ORM task (serialised by TaskRead).""" + waiting = [t for t in tasks if normalize_task_status(t.status, default="todo") == "wait"] + with_deps = await task_ids_with_dependencies(session, [t.id for t in waiting]) + for task in tasks: + task.wait_kind = derive_wait_kind( + task.status, + needs_human=bool(task.needs_human), + has_dependency=task.id in with_deps, + blocked_on=task.blocked_on, + wait_until=task.wait_until, + ) + return tasks + + +async def annotate_workplans( + session: AsyncSession, workplans: Sequence[Workplan] +) -> Sequence[Workplan]: + """Set ``workplan.blocked_kind`` on each ORM workplan (serialised by WorkplanRead).""" + blocked = [ + wp for wp in workplans if normalize_workplan_status(wp.status) == "blocked" + ] + kinds_by_workplan: dict[uuid.UUID, list[str | None]] = {wp.id: [] for wp in blocked} + if blocked: + rows = await session.execute( + select(Task).where( + Task.workplan_id.in_(list(kinds_by_workplan)), + Task.status == TaskStatus.wait, + ) + ) + wait_tasks = list(rows.scalars().all()) + await annotate_tasks(session, wait_tasks) + for task in wait_tasks: + kinds_by_workplan[task.workplan_id].append(task.wait_kind) + for wp in workplans: + wp.blocked_kind = derive_blocked_kind(wp.status, kinds_by_workplan.get(wp.id, [])) + return workplans diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/work_record_identifier_migration.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/work_record_identifier_migration.py new file mode 100644 index 00000000..aaafcad6 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/work_record_identifier_migration.py @@ -0,0 +1,800 @@ +"""Transactional State Hub projection migration for deterministic work-record UUIDs.""" + +from __future__ import annotations + +import hashlib +import json +import re +import uuid +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any + +from sqlalchemy import func, select, text +from sqlalchemy.ext.asyncio import AsyncSession + +from api.models.managed_repo import ManagedRepo +from api.models.task import Task, TaskPriority, TaskStatus +from api.models.work_record_identifier_alias import WorkRecordIdentifierAlias +from api.services.repository_aliases import resolve_repository_slug +from api.models.workplan import Workplan + +PLAN_SCHEMA = "repo-manager.identifier-migration-plan.v1" +FLEET_NAMESPACE = "helixforge" +DERIVATION_NAMESPACE_UUID = uuid.UUID("a4058507-5c4a-5a00-ab06-fffa4fb46009") +_RECORD_ID_RE = re.compile(r"^[A-Z][A-Z0-9-]*-WP-[0-9]{4}(?:-T[0-9]{2,})?$") + + +class IdentifierMigrationError(ValueError): + """The signed plan or current projection is unsafe to migrate.""" + + +@dataclass(frozen=True) +class IdentifierMigrationResult: + repo_slug: str + plan_sha256: str + direction: str + replacements: int + migrated: int + already_derived: int + assignments_deferred: int + + +@dataclass(frozen=True) +class SealedProjectionRepairReceipt: + """Non-secret evidence for one bounded pre-derivation projection repair.""" + + schema: str + outcome: str + repo_slug: str + repository_id: str + workplan_record_id: str + workplan_old_id: str + task_records: tuple[tuple[str, str], ...] + plan_sha256: str + source_revision: str + source_fingerprint: str + projection_identity: str + observed_at: str + + +def verify_plan(plan: dict[str, Any]) -> str: + """Validate the immutable plan envelope and deterministic UUID mappings.""" + if plan.get("schema") != PLAN_SCHEMA: + raise IdentifierMigrationError("unsupported identifier migration plan schema") + if plan.get("namespace") != FLEET_NAMESPACE: + raise IdentifierMigrationError( + f"plan namespace must be {FLEET_NAMESPACE!r}" + ) + if plan.get("apply_policy") != "all-or-nothing per repository": + raise IdentifierMigrationError("plan does not declare repository-atomic apply") + if plan.get("ready_to_apply") is not True: + raise IdentifierMigrationError("plan is not ready_to_apply") + + expected = plan.get("plan_sha256") + if not isinstance(expected, str) or not re.fullmatch(r"[0-9a-f]{64}", expected): + raise IdentifierMigrationError("plan has no valid SHA-256 seal") + unsealed = {key: value for key, value in plan.items() if key != "plan_sha256"} + canonical = json.dumps(unsealed, sort_keys=True, separators=(",", ":")).encode() + actual = hashlib.sha256(canonical).hexdigest() + if actual != expected: + raise IdentifierMigrationError("plan SHA-256 mismatch") + return expected + + +def _repository_mappings( + plan: dict[str, Any], repo_slug: str +) -> tuple[list[dict[str, Any]], int]: + matches = [item for item in plan.get("repositories", []) if item.get("repo") == repo_slug] + if len(matches) != 1: + raise IdentifierMigrationError( + f"repository {repo_slug!r} must occur exactly once in plan" + ) + repository = matches[0] + if repository.get("eligible") is not True or repository.get("atomic_unit") is not True: + raise IdentifierMigrationError(f"repository {repo_slug!r} is not eligible and atomic") + + replacements: list[dict[str, Any]] = [] + assignments = 0 + seen_old: set[uuid.UUID] = set() + seen_new: set[uuid.UUID] = set() + for raw in repository.get("mappings", []): + action = raw.get("action") + if action == "assign": + assignments += 1 + continue + if action == "unchanged": + continue + if action != "replace": + raise IdentifierMigrationError(f"unsupported mapping action {action!r}") + if raw.get("repo") != repo_slug: + raise IdentifierMigrationError("mapping repository does not match atomic unit") + kind = raw.get("kind") + record_id = raw.get("record_id") + if kind not in {"workplan", "task"}: + raise IdentifierMigrationError(f"unsupported record kind {kind!r}") + if not isinstance(record_id, str) or not _RECORD_ID_RE.fullmatch(record_id): + raise IdentifierMigrationError(f"noncanonical record id {record_id!r}") + try: + old_id = uuid.UUID(str(raw.get("current_uuid"))) + new_id = uuid.UUID(str(raw.get("derived_uuid"))) + except (TypeError, ValueError) as exc: + raise IdentifierMigrationError(f"invalid UUID mapping for {record_id}") from exc + expected_new = uuid.uuid5( + DERIVATION_NAMESPACE_UUID, + f"{FLEET_NAMESPACE}\n{record_id}", + ) + if new_id != expected_new: + raise IdentifierMigrationError(f"derived UUID mismatch for {record_id}") + if old_id == new_id or old_id in seen_old or new_id in seen_new: + raise IdentifierMigrationError(f"non-unique replacement for {record_id}") + seen_old.add(old_id) + seen_new.add(new_id) + replacements.append( + { + "kind": kind, + "record_id": record_id, + "old_id": old_id, + "new_id": new_id, + } + ) + return replacements, assignments + + +def _repository_entry(plan: dict[str, Any], repo_slug: str) -> dict[str, Any]: + matches = [item for item in plan.get("repositories", []) if item.get("repo") == repo_slug] + if len(matches) != 1: + raise IdentifierMigrationError( + f"repository {repo_slug!r} must occur exactly once in plan" + ) + return matches[0] + + +def _repair_unit_mappings( + plan: dict[str, Any], repo_slug: str, unit: dict[str, Any] +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + replacements, _assignments = _repository_mappings(plan, repo_slug) + workplan = unit.get("workplan") + tasks = unit.get("tasks") + if not isinstance(workplan, dict) or not isinstance(tasks, list): + raise IdentifierMigrationError("repair unit requires one workplan and a task list") + + workplan_record_id = workplan.get("record_id") + workplan_matches = [ + mapping + for mapping in replacements + if mapping["kind"] == "workplan" and mapping["record_id"] == workplan_record_id + ] + if len(workplan_matches) != 1: + raise IdentifierMigrationError( + "repair workplan must occur exactly once as a replacement in the sealed plan" + ) + workplan_mapping = workplan_matches[0] + raw_workplan_mapping = next( + raw + for raw in _repository_entry(plan, repo_slug).get("mappings", []) + if raw.get("kind") == "workplan" and raw.get("record_id") == workplan_record_id + ) + authoritative_path = raw_workplan_mapping.get("path") + if not isinstance(authoritative_path, str) or not authoritative_path: + raise IdentifierMigrationError("sealed workplan mapping has no authoritative path") + + task_record_ids = [task.get("record_id") for task in tasks if isinstance(task, dict)] + if len(task_record_ids) != len(tasks) or len(set(task_record_ids)) != len(tasks): + raise IdentifierMigrationError("repair task record ids must be explicit and unique") + task_mappings = [ + mapping + for mapping in replacements + if mapping["kind"] == "task" and mapping["record_id"] in task_record_ids + ] + mapped_task_ids = {mapping["record_id"] for mapping in task_mappings} + if mapped_task_ids != set(task_record_ids): + raise IdentifierMigrationError( + "repair tasks do not exactly match replacement mappings in the sealed plan" + ) + + raw_unit_task_ids = { + raw.get("record_id") + for raw in _repository_entry(plan, repo_slug).get("mappings", []) + if raw.get("kind") == "task" + and raw.get("action") == "replace" + and raw.get("path") == authoritative_path + } + if raw_unit_task_ids != set(task_record_ids): + raise IdentifierMigrationError( + "repair must include every replaced task in the sealed workplan unit" + ) + return workplan_mapping, task_mappings + + +def _uuid_field(value: Any, label: str) -> uuid.UUID: + try: + return uuid.UUID(str(value)) + except (TypeError, ValueError) as exc: + raise IdentifierMigrationError(f"invalid UUID for {label}") from exc + + +def _repair_payloads( + unit: dict[str, Any], + workplan_mapping: dict[str, Any], + task_mappings: list[dict[str, Any]], +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + raw_workplan = unit["workplan"] + if _uuid_field(raw_workplan.get("id"), "workplan") != workplan_mapping["old_id"]: + raise IdentifierMigrationError("repair workplan UUID does not match sealed old UUID") + slug = raw_workplan.get("slug") + title = raw_workplan.get("title") + status = raw_workplan.get("status") + if not isinstance(slug, str) or not slug or not isinstance(title, str) or not title: + raise IdentifierMigrationError("repair workplan requires explicit slug and title") + if status not in { + "proposed", "ready", "active", "blocked", "backlog", "finished", "archived" + }: + raise IdentifierMigrationError(f"unsupported repair workplan status {status!r}") + workplan_payload = { + "id": workplan_mapping["old_id"], + "topic_id": _uuid_field(raw_workplan.get("topic_id"), "workplan topic"), + "slug": slug, + "title": title, + "description": raw_workplan.get("description"), + "status": status, + "owner": raw_workplan.get("owner"), + "planning_priority": raw_workplan.get("planning_priority"), + "planning_order": raw_workplan.get("planning_order"), + } + + mappings_by_record = {mapping["record_id"]: mapping for mapping in task_mappings} + task_payloads: list[dict[str, Any]] = [] + for raw_task in unit["tasks"]: + mapping = mappings_by_record[raw_task["record_id"]] + if _uuid_field(raw_task.get("id"), raw_task["record_id"]) != mapping["old_id"]: + raise IdentifierMigrationError( + f"repair task UUID does not match sealed old UUID for {raw_task['record_id']}" + ) + title = raw_task.get("title") + if not isinstance(title, str) or not title: + raise IdentifierMigrationError( + f"repair task {raw_task['record_id']} requires an explicit title" + ) + try: + status_value = TaskStatus(raw_task.get("status", "todo")) + priority_value = TaskPriority(raw_task.get("priority", "medium")) + except ValueError as exc: + raise IdentifierMigrationError( + f"invalid task state for {raw_task['record_id']}" + ) from exc + parent_record_id = raw_task.get("parent_record_id") + if parent_record_id is not None and parent_record_id not in mappings_by_record: + raise IdentifierMigrationError( + f"parent task for {raw_task['record_id']} is outside the sealed repair unit" + ) + task_payloads.append( + { + "record_id": raw_task["record_id"], + "id": mapping["old_id"], + "title": title, + "description": raw_task.get("description"), + "status": status_value, + "priority": priority_value, + "assignee": raw_task.get("assignee"), + "blocking_reason": raw_task.get("blocking_reason"), + "needs_human": bool(raw_task.get("needs_human", False)), + "intervention_note": raw_task.get("intervention_note"), + "parent_task_id": ( + mappings_by_record[parent_record_id]["old_id"] + if parent_record_id is not None + else None + ), + } + ) + if task_payloads[-1]["needs_human"] and not task_payloads[-1]["intervention_note"]: + raise IdentifierMigrationError( + f"repair task {raw_task['record_id']} needs an intervention note" + ) + if ( + task_payloads[-1]["status"] == TaskStatus.wait + and task_payloads[-1]["needs_human"] + and not task_payloads[-1]["blocking_reason"] + ): + raise IdentifierMigrationError( + f"repair task {raw_task['record_id']} needs a blocking reason" + ) + + pending = list(task_payloads) + ordered: list[dict[str, Any]] = [] + emitted: set[uuid.UUID] = set() + while pending: + ready = [ + task + for task in pending + if task["parent_task_id"] is None or task["parent_task_id"] in emitted + ] + if not ready: + raise IdentifierMigrationError("repair task parentage contains a cycle") + for task in ready: + pending.remove(task) + ordered.append(task) + emitted.add(task["id"]) + return workplan_payload, ordered + + +def _same_workplan(row: Workplan, expected: dict[str, Any], repo_id: uuid.UUID) -> bool: + fields = ( + "id", "topic_id", "slug", "title", "description", "status", "owner", + "planning_priority", "planning_order", + ) + return row.repo_id == repo_id and all( + getattr(row, field) == expected[field] for field in fields + ) + + +def _same_task(row: Task, expected: dict[str, Any], workplan_id: uuid.UUID) -> bool: + fields = ( + "id", "title", "description", "status", "priority", "assignee", + "blocking_reason", "needs_human", "intervention_note", "parent_task_id", + ) + return row.workplan_id == workplan_id and all( + getattr(row, field) == expected[field] for field in fields + ) + + +_TASK_DUPLICATE_FIELDS = ( + "workplan_id", + "title", + "description", + "status", + "priority", + "assignee", + "due_date", + "blocking_reason", + "needs_human", + "intervention_note", + "parent_task_id", +) + + +def _same_duplicate_task( + legacy: Task, + derived: Task, + record_id: str, +) -> bool: + """Accept only a deterministic duplicate of an otherwise identical legacy row.""" + return ( + legacy.record_id in {None, record_id} + and derived.record_id == record_id + and all( + getattr(legacy, field) == getattr(derived, field) + for field in _TASK_DUPLICATE_FIELDS + ) + ) + + +async def _task_reference_count(session: AsyncSession, task_id: uuid.UUID) -> int: + """Count every inbound task reference before duplicate coalescence. + + Coalescence deliberately refuses referenced legacy rows. Updating references + could collide with relationships already attached to the deterministic row; + a zero-reference proof keeps the repair lossless and reversible. + """ + return int( + await session.scalar( + text( + "SELECT " + "(SELECT count(*) FROM token_events WHERE task_id = :task_id) + " + "(SELECT count(*) FROM tasks WHERE parent_task_id = :task_id) + " + "(SELECT count(*) FROM review_contracts WHERE task_id = :task_id) + " + "(SELECT count(*) FROM progress_events WHERE task_id = :task_id) + " + "(SELECT count(*) FROM capability_requests WHERE blocking_task_id = :task_id) + " + "(SELECT count(*) FROM workplan_dependencies WHERE to_task_id = :task_id) + " + "(SELECT count(*) FROM workplan_dependencies WHERE from_task_id = :task_id) + " + "(SELECT count(*) FROM suggestions WHERE promoted_task_id = :task_id)" + ), + {"task_id": task_id}, + ) + or 0 + ) + + +async def repair_absent_prederivation_projection( + session: AsyncSession, + plan: dict[str, Any], + repo_slug: str, + unit: dict[str, Any], + *, + expected_plan_sha256: str, + source_revision: str, + source_fingerprint: str, + source_clean: bool, + source_synchronized: bool, + primary_confirmed: bool, + projection_identity: str, + observed_at: datetime | None = None, +) -> SealedProjectionRepairReceipt: + """Restore one exact random-ID workplan unit into an absent projection. + + This is intentionally separate from ordinary consistency repair. The caller + must independently prove Git cleanliness/synchronization and provide the + authoritative unit parsed from the pinned source revision. + """ + plan_sha256 = verify_plan(plan) + if plan_sha256 != expected_plan_sha256: + raise IdentifierMigrationError("explicit plan SHA-256 does not match sealed plan") + if not primary_confirmed: + raise IdentifierMigrationError("projection repair requires explicit primary confirmation") + if not source_clean or not source_synchronized: + raise IdentifierMigrationError("projection repair requires a clean synchronized source") + if not re.fullmatch(r"[0-9a-f]{40}", source_revision): + raise IdentifierMigrationError("source revision must be a full Git SHA") + if not re.fullmatch(r"[0-9a-f]{64}", source_fingerprint): + raise IdentifierMigrationError("source fingerprint must be SHA-256") + if not projection_identity.strip(): + raise IdentifierMigrationError("projection identity is required") + + repository_entry = _repository_entry(plan, repo_slug) + if source_revision != repository_entry.get("planned_head_sha"): + raise IdentifierMigrationError("source revision drifted from sealed plan") + if source_fingerprint != repository_entry.get("source_fingerprint"): + raise IdentifierMigrationError("source fingerprint drifted from sealed plan") + workplan_mapping, task_mappings = _repair_unit_mappings(plan, repo_slug, unit) + workplan_payload, task_payloads = _repair_payloads( + unit, workplan_mapping, task_mappings + ) + if session.in_transaction(): + raise IdentifierMigrationError("projection repair requires a fresh database session") + + outcome = "repaired" + async with session.begin(): + resolution = await resolve_repository_slug(session, repo_slug, required=False) + if resolution is None: + raise IdentifierMigrationError(f"repository projection is absent: {repo_slug}") + repo = await session.scalar( + select(ManagedRepo) + .where(ManagedRepo.id == resolution.repo.id) + .with_for_update() + ) + if repo.topic_id != workplan_payload["topic_id"]: + raise IdentifierMigrationError("repair topic does not match repository projection") + + old_ids = [workplan_mapping["old_id"], *(m["old_id"] for m in task_mappings)] + new_workplan = await session.get(Workplan, workplan_mapping["new_id"]) + new_tasks = [await session.get(Task, mapping["new_id"]) for mapping in task_mappings] + if new_workplan is not None or any(task is not None for task in new_tasks): + raise IdentifierMigrationError("derived target presence blocks projection repair") + alias_count = await session.scalar( + select(func.count()).select_from(WorkRecordIdentifierAlias).where( + WorkRecordIdentifierAlias.old_id.in_(old_ids) + ) + ) + if alias_count: + raise IdentifierMigrationError("durable alias presence blocks projection repair") + + old_workplan = await session.get(Workplan, workplan_mapping["old_id"]) + old_tasks = [await session.get(Task, mapping["old_id"]) for mapping in task_mappings] + present_count = int(old_workplan is not None) + sum(task is not None for task in old_tasks) + expected_count = 1 + len(task_mappings) + if present_count not in {0, expected_count}: + raise IdentifierMigrationError("partial old projection presence blocks repair") + + if present_count == expected_count: + if not _same_workplan(old_workplan, workplan_payload, repo.id): + raise IdentifierMigrationError( + "existing old workplan differs from sealed repair unit" + ) + expected_tasks = {task["id"]: task for task in task_payloads} + if any( + not _same_task(task, expected_tasks[task.id], workplan_mapping["old_id"]) + for task in old_tasks + ): + raise IdentifierMigrationError("existing old tasks differ from sealed repair unit") + outcome = "verified_noop" + else: + slug_conflict = await session.scalar( + select(Workplan.id).where(Workplan.slug == workplan_payload["slug"]) + ) + if slug_conflict is not None: + raise IdentifierMigrationError("workplan slug is already projected") + session.add( + Workplan(repo_id=repo.id, **workplan_payload) + ) + await session.flush() + for task_payload in task_payloads: + values = {key: value for key, value in task_payload.items() if key != "record_id"} + session.add(Task(workplan_id=workplan_mapping["old_id"], **values)) + await session.flush() + + timestamp = observed_at or datetime.now(UTC) + if timestamp.tzinfo is None: + raise IdentifierMigrationError("receipt observation time must be timezone-aware") + return SealedProjectionRepairReceipt( + schema="state-hub.sealed-prederivation-projection-repair-receipt.v1", + outcome=outcome, + repo_slug=repo_slug, + repository_id=str(repo.id), + workplan_record_id=workplan_mapping["record_id"], + workplan_old_id=str(workplan_mapping["old_id"]), + task_records=tuple( + (mapping["record_id"], str(mapping["old_id"])) for mapping in task_mappings + ), + plan_sha256=plan_sha256, + source_revision=source_revision, + source_fingerprint=source_fingerprint, + projection_identity=projection_identity, + observed_at=timestamp.astimezone(UTC).isoformat().replace("+00:00", "Z"), + ) + + +async def _assert_projection_preconditions( + session: AsyncSession, + repository_id: uuid.UUID, + repo_slug: str, + replacements: list[dict[str, Any]], + *, + reverse: bool, +) -> dict[str, str]: + states: dict[str, str] = {} + for mapping in replacements: + source_id = mapping["new_id"] if reverse else mapping["old_id"] + target_id = mapping["old_id"] if reverse else mapping["new_id"] + if mapping["kind"] == "workplan": + query = text( + "SELECT workplans.id, workplans.repo_id, workplans.slug " + "FROM workplans WHERE workplans.id = :record_id FOR UPDATE" + ) + else: + query = text( + "SELECT tasks.id, workplans.repo_id, tasks.record_id " + "FROM tasks " + "JOIN workplans ON workplans.id = tasks.workplan_id " + "WHERE tasks.id = :record_id FOR UPDATE" + ) + source = (await session.execute(query, {"record_id": source_id})).one_or_none() + target = (await session.execute(query, {"record_id": target_id})).one_or_none() + for label, row in (("source", source), ("target", target)): + if row is not None and row.repo_id != repository_id: + raise IdentifierMigrationError( + f"{label} {mapping['kind']} {row.id} belongs to another repository" + ) + + if reverse: + if source is None: + raise IdentifierMigrationError( + f"source {mapping['kind']} {source_id} is absent from repository {repo_slug}" + ) + if target is not None: + raise IdentifierMigrationError( + f"target {mapping['kind']} {target_id} already exists" + ) + states[mapping["record_id"]] = "derived_source" + continue + + if source is not None and target is not None: + if mapping["kind"] != "task": + raise IdentifierMigrationError( + f"both legacy and derived {mapping['kind']} rows exist for {mapping['record_id']}" + ) + legacy_task = await session.scalar( + select(Task).where(Task.id == source_id).with_for_update() + ) + derived_task = await session.scalar( + select(Task).where(Task.id == target_id).with_for_update() + ) + if ( + legacy_task is None + or derived_task is None + or not _same_duplicate_task( + legacy_task, derived_task, mapping["record_id"] + ) + ): + raise IdentifierMigrationError( + f"legacy and derived task rows differ for {mapping['record_id']}" + ) + references = await _task_reference_count(session, source_id) + if references: + raise IdentifierMigrationError( + f"legacy duplicate task has {references} inbound references for " + f"{mapping['record_id']}" + ) + states[mapping["record_id"]] = "duplicate_target" + continue + if source is None and target is None: + raise IdentifierMigrationError( + f"neither legacy nor derived {mapping['kind']} row exists for {mapping['record_id']}" + ) + if source is not None: + states[mapping["record_id"]] = "legacy_source" + continue + + # A pre-existing derived target is safe only when it already represents + # the canonical record from this repository. This is the state produced + # when forge reconciliation reached a partial projection before the + # sealed file migration did. + identity = target.slug if mapping["kind"] == "workplan" else target.record_id + if not isinstance(identity, str) or identity.strip().lower() != mapping["record_id"].lower(): + raise IdentifierMigrationError( + f"derived target identity mismatch for {mapping['record_id']}" + ) + states[mapping["record_id"]] = "derived_target" + return states + + +async def apply_repository_identifier_migration( + session: AsyncSession, + plan: dict[str, Any], + repo_slug: str, +) -> IdentifierMigrationResult: + """Replace one repository's projected UUIDs in one database transaction.""" + plan_sha256 = verify_plan(plan) + replacements, assignments = _repository_mappings(plan, repo_slug) + if session.in_transaction(): + raise IdentifierMigrationError("migration requires a fresh database session") + + async with session.begin(): + resolution = await resolve_repository_slug(session, repo_slug, required=False) + if resolution is None: + raise IdentifierMigrationError(f"repository projection is absent: {repo_slug}") + projection_states = await _assert_projection_preconditions( + session, resolution.repo.id, repo_slug, replacements, reverse=False + ) + aliases = { + alias.old_id: alias + for alias in ( + await session.execute( + select(WorkRecordIdentifierAlias).where( + WorkRecordIdentifierAlias.old_id.in_( + [mapping["old_id"] for mapping in replacements] + ) + ) + ) + ).scalars() + } + for mapping in replacements: + alias = aliases.get(mapping["old_id"]) + if alias is None: + alias = WorkRecordIdentifierAlias( + old_id=mapping["old_id"], + new_id=mapping["new_id"], + record_kind=mapping["kind"], + record_id=mapping["record_id"], + repo_slug=repo_slug, + namespace=FLEET_NAMESPACE, + plan_sha256=plan_sha256, + ) + session.add(alias) + aliases[mapping["old_id"]] = alias + elif ( + alias.new_id != mapping["new_id"] + or alias.record_kind != mapping["kind"] + or alias.record_id != mapping["record_id"] + or alias.repo_slug != repo_slug + or alias.namespace != FLEET_NAMESPACE + or alias.plan_sha256 != plan_sha256 + or alias.migration_status not in {"reversed", "applied"} + ): + raise IdentifierMigrationError( + f"conflicting durable alias for {mapping['old_id']}" + ) + await session.flush() + + for mapping in ( + item + for item in replacements + if projection_states[item["record_id"]] == "duplicate_target" + ): + result = await session.execute( + text("DELETE FROM tasks WHERE id = :old_id"), + {"old_id": mapping["old_id"]}, + ) + if result.rowcount != 1: + raise IdentifierMigrationError( + f"failed to coalesce duplicate {mapping['record_id']}" + ) + + for kind in ("workplan", "task"): + table = "workplans" if kind == "workplan" else "tasks" + for mapping in ( + item + for item in replacements + if item["kind"] == kind + and projection_states[item["record_id"]] == "legacy_source" + ): + result = await session.execute( + text(f"UPDATE {table} SET id = :new_id WHERE id = :old_id"), + {"old_id": mapping["old_id"], "new_id": mapping["new_id"]}, + ) + if result.rowcount != 1: + raise IdentifierMigrationError( + f"failed to replace {mapping['record_id']}" + ) + + now = datetime.now(UTC) + for alias in aliases.values(): + alias.migration_status = "applied" + alias.applied_at = now + alias.reversed_at = None + + return IdentifierMigrationResult( + repo_slug=repo_slug, + plan_sha256=plan_sha256, + direction="forward", + replacements=len(replacements), + migrated=sum( + state in {"legacy_source", "duplicate_target"} + for state in projection_states.values() + ), + already_derived=sum( + state == "derived_target" for state in projection_states.values() + ), + assignments_deferred=assignments, + ) + + +async def reverse_repository_identifier_migration( + session: AsyncSession, + plan: dict[str, Any], + repo_slug: str, +) -> IdentifierMigrationResult: + """Restore one repository's prior projected UUIDs and retain aliases.""" + plan_sha256 = verify_plan(plan) + replacements, assignments = _repository_mappings(plan, repo_slug) + if session.in_transaction(): + raise IdentifierMigrationError("migration requires a fresh database session") + + async with session.begin(): + resolution = await resolve_repository_slug(session, repo_slug, required=False) + if resolution is None: + raise IdentifierMigrationError(f"repository projection is absent: {repo_slug}") + aliases = list( + ( + await session.execute( + select(WorkRecordIdentifierAlias).where( + WorkRecordIdentifierAlias.plan_sha256 == plan_sha256, + WorkRecordIdentifierAlias.repo_slug == repo_slug, + ) + ) + ).scalars() + ) + aliases_by_old = {alias.old_id: alias for alias in aliases} + if len(aliases_by_old) != len(replacements): + raise IdentifierMigrationError("durable alias set is incomplete") + for mapping in replacements: + alias = aliases_by_old.get(mapping["old_id"]) + if ( + alias is None + or alias.new_id != mapping["new_id"] + or alias.record_kind != mapping["kind"] + or alias.record_id != mapping["record_id"] + or alias.migration_status != "applied" + ): + raise IdentifierMigrationError( + f"durable alias is not applied for {mapping['record_id']}" + ) + await _assert_projection_preconditions( + session, resolution.repo.id, repo_slug, replacements, reverse=True + ) + + for kind in ("task", "workplan"): + table = "tasks" if kind == "task" else "workplans" + for mapping in (item for item in replacements if item["kind"] == kind): + result = await session.execute( + text(f"UPDATE {table} SET id = :old_id WHERE id = :new_id"), + {"old_id": mapping["old_id"], "new_id": mapping["new_id"]}, + ) + if result.rowcount != 1: + raise IdentifierMigrationError( + f"failed to reverse {mapping['record_id']}" + ) + + now = datetime.now(UTC) + for alias in aliases: + alias.migration_status = "reversed" + alias.reversed_at = now + + return IdentifierMigrationResult( + repo_slug=repo_slug, + plan_sha256=plan_sha256, + direction="reverse", + replacements=len(replacements), + migrated=len(replacements), + already_derived=0, + assignments_deferred=assignments, + ) diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/workplan_files.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/workplan_files.py new file mode 100644 index 00000000..fef57fe9 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/workplan_files.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import re +import socket +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from api.models.managed_repo import ManagedRepo + + +_TASK_BLOCK_RE = re.compile(r"```task\s*\n(.*?)\n```", re.DOTALL) + + +@dataclass(frozen=True) +class WorkplanFileRef: + repo_path: Path + path: Path + archived: bool + + @property + def relative_path(self) -> str: + return str(self.path.relative_to(self.repo_path)) + + +def resolve_repo_path(repo: ManagedRepo | None) -> Path | None: + if repo is None: + return None + hostname = socket.gethostname() + host_paths = repo.host_paths or {} + candidates = [host_paths.get(hostname), repo.local_path] + for raw in candidates: + if not raw: + continue + path = Path(raw).expanduser() + if path.is_dir(): + return path + return None + + +def find_workplan_for_workplan( + repo: ManagedRepo | None, + workplan_id: uuid.UUID, +) -> WorkplanFileRef | None: + repo_path = resolve_repo_path(repo) + if repo_path is None: + return None + workplans_dir = repo_path / "workplans" + for directory, archived in ( + (workplans_dir, False), + (workplans_dir / "archived", True), + ): + if not directory.is_dir(): + continue + for path in sorted(directory.glob("*.md")): + meta = _frontmatter(path) + file_id = meta.get("state_hub_workplan_id") or meta.get("state_hub_workstream_id") + if str(file_id or "").strip().strip('"') == str(workplan_id): + return WorkplanFileRef(repo_path=repo_path, path=path, archived=archived) + return None + + +find_workplan_for_workstream = find_workplan_for_workplan + + +def task_block_linked(path: Path, task_id: uuid.UUID) -> bool: + return _task_block_for_task(path, task_id) is not None + + +def workplan_status(path: Path) -> str | None: + status = _frontmatter(path).get("status") + return str(status).strip() if status is not None else None + + +def task_block_status(path: Path, task_id: uuid.UUID) -> str | None: + meta = _task_block_for_task(path, task_id) + if meta is None: + return None + status = meta.get("status") + return str(status).strip() if status is not None else None + + +def patch_workplan_status(path: Path, status: str) -> bool: + return _patch_frontmatter_field(path, "status", status) + + +def patch_task_status(path: Path, task_id: uuid.UUID, status: str) -> bool: + text = path.read_text(encoding="utf-8") + + def _replace(match: re.Match) -> str: + block = match.group(0) + meta = _parse_task_block(match.group(1)) + if str(meta.get("state_hub_task_id", "")).strip().strip('"') != str(task_id): + return block + replaced = re.sub( + r"^(status:\s*)\S+", + rf"\g<1>{status}", + block, + count=1, + flags=re.MULTILINE, + ) + if replaced != block: + return replaced + return block.replace("\n```", f"\nstatus: {status}\n```", 1) + + new_text = _TASK_BLOCK_RE.sub(_replace, text) + if new_text == text: + return False + path.write_text(new_text, encoding="utf-8") + return True + + +def _frontmatter(path: Path) -> dict[str, Any]: + try: + text = path.read_text(encoding="utf-8") + except OSError: + return {} + if not text.startswith("---"): + return {} + parts = text.split("---", 2) + if len(parts) < 3: + return {} + try: + return yaml.safe_load(parts[1].strip()) or {} + except yaml.YAMLError: + return {} + + +def _patch_frontmatter_field(path: Path, key: str, value: str) -> bool: + text = path.read_text(encoding="utf-8") + if not text.startswith("---"): + return False + lines = text.split("\n") + close_idx = None + for i, line in enumerate(lines[1:], 1): + if line.strip() == "---": + close_idx = i + break + if close_idx is None: + return False + + new_line = f"{key}: {value}" + for i in range(1, close_idx): + if re.match(rf"^\s*{re.escape(key)}\s*:", lines[i]): + if lines[i] == new_line: + return False + lines[i] = new_line + path.write_text("\n".join(lines), encoding="utf-8") + return True + + lines.insert(close_idx, new_line) + path.write_text("\n".join(lines), encoding="utf-8") + return True + + +def _task_block_for_task(path: Path, task_id: uuid.UUID) -> dict[str, Any] | None: + try: + text = path.read_text(encoding="utf-8") + except OSError: + return None + for match in _TASK_BLOCK_RE.finditer(text): + meta = _parse_task_block(match.group(1)) + if str(meta.get("state_hub_task_id", "")).strip().strip('"') == str(task_id): + return meta + return None + + +def _parse_task_block(raw: str) -> dict[str, Any]: + try: + return yaml.safe_load(raw.strip()) or {} + except yaml.YAMLError: + return {} diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/write_idempotency.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/write_idempotency.py new file mode 100644 index 00000000..7d8c5439 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/services/write_idempotency.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Any + +from sqlalchemy import select +from starlette.responses import JSONResponse +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +from api.database import async_session_factory +from api.models.write_idempotency_key import WriteIdempotencyKey + +IDEMPOTENCY_HEADER = b"idempotency-key" +REPLAY_HEADER = "X-StateHub-Idempotency-Replay" +CONFLICT_STATUS = 409 +DEFAULT_IDEMPOTENCY_TTL_DAYS = 14 + + +@dataclass(frozen=True) +class WriteRouteRule: + method: str + pattern: str + route_class: str + description: str + + def matches(self, method: str, path: str) -> bool: + normalized = path.rstrip("/") or "/" + return self.method == method.upper() and re.fullmatch(self.pattern, normalized) is not None + + +WRITE_ROUTE_RULES: tuple[WriteRouteRule, ...] = ( + WriteRouteRule("POST", r"/progress", "append", "append progress event"), + WriteRouteRule("POST", r"/messages", "append", "send agent message"), + WriteRouteRule("PATCH", r"/messages/[^/]+/read", "append", "mark known message read"), + WriteRouteRule("POST", r"/token-events", "append", "record token event"), + WriteRouteRule("POST", r"/token-events/upsert", "append", "upsert token event"), + WriteRouteRule("POST", r"/decisions", "append", "record decision"), + WriteRouteRule("PATCH", r"/tasks/[^/]+", "replace", "update task"), + WriteRouteRule("POST", r"/tasks/bulk-status-sync", "replace", "bulk task status sync"), + WriteRouteRule( + "POST", + r"/repos/[^/]+/work-record-projection/reconcile", + "replace", + "reconcile one forge-derived repository projection", + ), + WriteRouteRule("PATCH", r"/decisions/[^/]+", "replace", "update decision"), + WriteRouteRule("POST", r"/decisions/[^/]+/resolve", "replace", "resolve decision"), + WriteRouteRule("PATCH", r"/workplans/[^/]+", "replace", "update workplan"), + WriteRouteRule("PATCH", r"/workstreams/[^/]+", "replace", "update legacy workstream alias"), +) + + +def route_rule_for(method: str, path: str) -> WriteRouteRule | None: + for rule in WRITE_ROUTE_RULES: + if rule.matches(method, path): + return rule + return None + + +def route_class_for(method: str, path: str) -> str | None: + rule = route_rule_for(method, path) + return rule.route_class if rule else None + + +def canonical_request_hash(method: str, path: str, query_string: bytes, body: bytes) -> str: + try: + parsed: Any = json.loads(body.decode("utf-8")) if body else None + body_repr = json.dumps(parsed, sort_keys=True, separators=(",", ":")) + except (UnicodeDecodeError, json.JSONDecodeError): + body_repr = body.hex() + query = query_string.decode("utf-8", errors="replace") + seed = f"{method.upper()}\n{path}\n{query}\n{body_repr}".encode("utf-8") + return hashlib.sha256(seed).hexdigest() + + +def _header_value(headers: list[tuple[bytes, bytes]], name: bytes) -> str | None: + lname = name.lower() + for key, value in headers: + if key.lower() == lname: + return value.decode("utf-8", errors="replace") + return None + + +async def _send_json_response(response: JSONResponse, scope: Scope, receive: Receive, send: Send) -> None: + await response(scope, receive, send) + + +class WriteIdempotencyMiddleware: + """Replay exact duplicate write requests carrying Idempotency-Key. + + The middleware is intentionally narrow: it only participates on the offline + relay allowlist. Non-allowlisted routes keep their normal behavior even if a + caller sends an Idempotency-Key header. + """ + + def __init__(self, app: ASGIApp, *, ttl_days: int = DEFAULT_IDEMPOTENCY_TTL_DAYS) -> None: + self.app = app + self.ttl_days = ttl_days + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + method = str(scope.get("method", "")).upper() + path = str(scope.get("path", "")) + rule = route_rule_for(method, path) + headers = list(scope.get("headers") or []) + key = _header_value(headers, IDEMPOTENCY_HEADER) + if rule is None or not key: + await self.app(scope, receive, send) + return + + body = await self._read_body(receive) + request_hash = canonical_request_hash(method, path, scope.get("query_string", b""), body) + source_host = _header_value(headers, b"x-statehub-source-host") + source_agent = _header_value(headers, b"x-statehub-source-agent") + + async with async_session_factory() as session: + existing = (await session.execute( + select(WriteIdempotencyKey).where(WriteIdempotencyKey.key == key) + )).scalar_one_or_none() + if existing is not None: + existing.last_seen_at = datetime.now(tz=timezone.utc) + await session.commit() + if existing.request_hash != request_hash: + await _send_json_response( + JSONResponse( + status_code=CONFLICT_STATUS, + content={ + "error": "Idempotency-Key was reused with a different request", + "idempotency_key": key, + }, + ), + scope, + self._receive_from_body(body), + send, + ) + return + await _send_json_response( + JSONResponse( + status_code=existing.response_status, + content=existing.response_body, + headers={REPLAY_HEADER: "true"}, + ), + scope, + self._receive_from_body(body), + send, + ) + return + + start_message: Message | None = None + body_parts: list[bytes] = [] + + async def capture_send(message: Message) -> None: + nonlocal start_message + if message["type"] == "http.response.start": + start_message = message + elif message["type"] == "http.response.body": + body_parts.append(message.get("body", b"")) + await send(message) + + await self.app(scope, self._receive_from_body(body), capture_send) + + if start_message is None: + return + status = int(start_message.get("status", 500)) + if status < 200 or status >= 300: + return + + response_body_bytes = b"".join(body_parts) + try: + response_body = json.loads(response_body_bytes.decode("utf-8")) if response_body_bytes else None + except (UnicodeDecodeError, json.JSONDecodeError): + return + + async with async_session_factory() as session: + existing = (await session.execute( + select(WriteIdempotencyKey).where(WriteIdempotencyKey.key == key) + )).scalar_one_or_none() + if existing is not None: + return + now = datetime.now(tz=timezone.utc) + session.add(WriteIdempotencyKey( + key=key, + method=method, + path=path, + route_class=rule.route_class, + request_hash=request_hash, + response_status=status, + response_body=response_body, + source_host=source_host, + source_agent=source_agent, + first_seen_at=now, + last_seen_at=now, + expires_at=now + timedelta(days=self.ttl_days), + )) + await session.commit() + + @staticmethod + async def _read_body(receive: Receive) -> bytes: + chunks: list[bytes] = [] + while True: + message = await receive() + if message["type"] != "http.request": + continue + chunks.append(message.get("body", b"")) + if not message.get("more_body", False): + break + return b"".join(chunks) + + @staticmethod + def _receive_from_body(body: bytes) -> Receive: + sent = False + + async def receive() -> Message: + nonlocal sent + if sent: + return {"type": "http.request", "body": b"", "more_body": False} + sent = True + return {"type": "http.request", "body": body, "more_body": False} + + return receive diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/task_status.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/task_status.py new file mode 100644 index 00000000..0bec35f5 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/task_status.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from typing import Any + +CANONICAL_TASK_STATUSES: tuple[str, ...] = ( + "wait", + "todo", + "progress", + "done", + "cancel", +) + +TASK_STATUS_CODES: dict[str, str] = { + "WAIT": "wait", + "TODO": "todo", + "PROG": "progress", + "DONE": "done", + "CNCL": "cancel", +} + +LEGACY_TASK_STATUS_ALIASES: dict[str, str] = { + "blocked": "wait", + "waiting": "wait", + "in_progress": "progress", + "in-progress": "progress", + "prog": "progress", + "cancelled": "cancel", + "canceled": "cancel", + "cncl": "cancel", +} + +OPEN_TASK_STATUSES: frozenset[str] = frozenset({"wait", "todo", "progress"}) +ACTIVE_TASK_STATUSES: frozenset[str] = frozenset({"wait", "progress"}) +TERMINAL_TASK_STATUSES: frozenset[str] = frozenset({"done", "cancel"}) + +TASK_STATUS_ORDER: dict[str, int] = { + "todo": 0, + "wait": 1, + "progress": 1, + "done": 2, + "cancel": 2, +} + +TASK_STATUS_LABELS: dict[str, str] = { + "wait": "wait", + "todo": "todo", + "progress": "progress", + "done": "done", + "cancel": "cancel", +} + + +def raw_status_value(status: Any) -> str: + if hasattr(status, "value"): + status = status.value + return str(status or "").strip() + + +def normalize_task_status(status: Any, *, default: str | None = None) -> str: + """Normalize canon task statuses plus legacy aliases to stored values.""" + value = raw_status_value(status).lower() + if not value: + if default is not None: + return default + raise ValueError("task status is required") + value = LEGACY_TASK_STATUS_ALIASES.get(value, value) + if value not in CANONICAL_TASK_STATUSES: + allowed = ", ".join(CANONICAL_TASK_STATUSES) + aliases = ", ".join(sorted(LEGACY_TASK_STATUS_ALIASES)) + raise ValueError(f"task status must be one of {allowed}; legacy aliases: {aliases}") + return value + + +def status_value(status: Any, *, default: str | None = None) -> str: + """Compatibility wrapper used by lifecycle and reconciliation code.""" + return normalize_task_status(status, default=default) + + +def is_open_task_status(status: Any) -> bool: + return normalize_task_status(status, default="todo") in OPEN_TASK_STATUSES + + +def is_terminal_task_status(status: Any) -> bool: + return normalize_task_status(status, default="todo") in TERMINAL_TASK_STATUSES diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/work_record_flavor.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/work_record_flavor.py new file mode 100644 index 00000000..5fc9fe3b --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/work_record_flavor.py @@ -0,0 +1,48 @@ +"""Work-record flavor (STATE-WP-0092). + +Flavor is a closed bucket on workplans and tasks, orthogonal to kind and +status. Unset flavor is not residual. +""" + +from __future__ import annotations + +from typing import Any + +WORK_RECORD_FLAVORS: tuple[str, ...] = ( + "planning", + "implementation", + "refactoring", + "extension", + "residual", +) +RESIDUAL_FLAVOR = "residual" +FLAVOR_PROMOTION_REASONS: tuple[str, ...] = ("demand", "risk") + +_EMPTY = {"", "~", "null", "none", "nil"} + + +def normalize_flavor(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip().lower() + if text in _EMPTY: + return None + return text + + +def is_known_flavor(value: Any) -> bool: + flavor = normalize_flavor(value) + return flavor is None or flavor in WORK_RECORD_FLAVORS + + +def is_residual_flavor(value: Any) -> bool: + return normalize_flavor(value) == RESIDUAL_FLAVOR + + +def normalize_promotion_reason(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip().lower() + if text in _EMPTY: + return None + return text diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/workplan_status.py b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/workplan_status.py new file mode 100644 index 00000000..a9e9d695 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/workplan_status.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +import fnmatch +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +CANONICAL_WORKPLAN_STATUSES: tuple[str, ...] = ( + "proposed", + "ready", + "active", + "blocked", + "backlog", + "finished", + "archived", +) + +LEGACY_WORKPLAN_STATUS_ALIASES: dict[str, str] = { + "todo": "ready", + "done": "finished", + "completed": "finished", + "accepted": "finished", +} + +SUPPORTED_WORKPLAN_STATUSES: tuple[str, ...] = ( + *CANONICAL_WORKPLAN_STATUSES, + *LEGACY_WORKPLAN_STATUS_ALIASES.keys(), +) + +OPEN_WORKPLAN_STATUSES: tuple[str, ...] = ("ready", "active", "blocked") +CURRENT_WORKPLAN_STATUSES: tuple[str, ...] = ("active", "blocked") +CLOSED_WORKPLAN_STATUSES: tuple[str, ...] = ("finished", "archived") +PLANNING_WORKPLAN_STATUSES: tuple[str, ...] = ("proposed", "ready", "backlog") + +# Legacy aliases (workstream terminology) +CANONICAL_WORKSTREAM_STATUSES = CANONICAL_WORKPLAN_STATUSES +LEGACY_WORKSTREAM_STATUS_ALIASES = LEGACY_WORKPLAN_STATUS_ALIASES +SUPPORTED_WORKSTREAM_STATUSES = SUPPORTED_WORKPLAN_STATUSES +OPEN_WORKSTREAM_STATUSES = OPEN_WORKPLAN_STATUSES +CURRENT_WORKSTREAM_STATUSES = CURRENT_WORKPLAN_STATUSES +CLOSED_WORKSTREAM_STATUSES = CLOSED_WORKPLAN_STATUSES +PLANNING_WORKSTREAM_STATUSES = PLANNING_WORKPLAN_STATUSES + + +@dataclass(frozen=True) +class ReadyReviewStatus: + needs_review: bool + reason: str = "" + changed_paths: tuple[str, ...] = () + + +def normalize_workplan_status(status: Any, *, has_started: bool | None = None) -> str: + """Return the canonical lifecycle status for a stored or legacy value.""" + value = _status_value(status) + if value == "todo" and has_started: + return "active" + return LEGACY_WORKPLAN_STATUS_ALIASES.get(value, value) + + +normalize_workstream_status = normalize_workplan_status + + +def is_canonical_workplan_status(status: Any) -> bool: + return _status_value(status) in CANONICAL_WORKPLAN_STATUSES + + +is_canonical_workstream_status = is_canonical_workplan_status + + +def is_supported_workplan_status(status: Any) -> bool: + return _status_value(status) in SUPPORTED_WORKPLAN_STATUSES + + +is_supported_workstream_status = is_supported_workplan_status + + +def workplan_has_started(task_statuses: list[Any] | tuple[Any, ...]) -> bool: + return any(_status_value(status) not in {"", "todo"} for status in task_statuses) + + +workstream_has_started = workplan_has_started + + +def ready_review_status( + repo_dir: str | Path, + reviewed_against_commit: Any, + context_paths: Any = None, +) -> ReadyReviewStatus: + """Return whether a ready workplan needs review against current repo state. + + When context paths are supplied, only changes under those paths matter. + Missing or invalid git metadata is treated conservatively as needs review. + """ + reviewed = str(reviewed_against_commit or "").strip().strip("\"'") + if not reviewed: + return ReadyReviewStatus(False) + + repo = Path(repo_dir) + head = _git_output(repo, ["rev-parse", "HEAD"]) + if not head: + return ReadyReviewStatus(True, "could not determine repository HEAD") + if reviewed == head: + return ReadyReviewStatus(False) + + if not _git_commit_exists(repo, reviewed): + return ReadyReviewStatus(True, f"review commit {reviewed[:12]} is not available") + + patterns = _as_list(context_paths) + if not patterns: + return ReadyReviewStatus( + True, + f"reviewed against {reviewed[:12]}, current HEAD is {head[:12]}", + ) + + changed = _changed_paths_since(repo, reviewed) + if changed is None: + return ReadyReviewStatus(True, "could not compare reviewed commit with HEAD") + + matching = tuple(path for path in changed if _matches_any_context(path, patterns)) + if not matching: + return ReadyReviewStatus(False) + + return ReadyReviewStatus( + True, + f"{len(matching)} context path(s) changed since {reviewed[:12]}", + matching, + ) + + +def _status_value(status: Any) -> str: + if hasattr(status, "value") and not isinstance(status, (str, bytes, bytearray)): + status = status.value + return str(status or "").strip().lower() + + +def _as_list(value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, (list, tuple, set)): + return [str(item).strip().replace("\\", "/") for item in value if str(item).strip()] + if isinstance(value, str): + return [item.strip().replace("\\", "/") for item in value.split(",") if item.strip()] + return [str(value).strip().replace("\\", "/")] + + +def _git_output(repo: Path, args: list[str]) -> str | None: + try: + return subprocess.check_output( + ["git", "-C", str(repo), *args], + stderr=subprocess.DEVNULL, + text=True, + ).strip() + except (subprocess.CalledProcessError, FileNotFoundError, OSError): + return None + + +def _git_commit_exists(repo: Path, commit: str) -> bool: + try: + subprocess.run( + ["git", "-C", str(repo), "cat-file", "-e", f"{commit}^{{commit}}"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return True + except (subprocess.CalledProcessError, FileNotFoundError, OSError): + return False + + +def _changed_paths_since(repo: Path, commit: str) -> tuple[str, ...] | None: + output = _git_output(repo, ["diff", "--name-only", f"{commit}..HEAD", "--"]) + if output is None: + return None + return tuple(path.strip().replace("\\", "/") for path in output.splitlines() if path.strip()) + + +def _matches_any_context(path: str, patterns: list[str]) -> bool: + norm_path = path.strip().replace("\\", "/").lstrip("./") + for raw_pattern in patterns: + pattern = raw_pattern.strip().replace("\\", "/").lstrip("./") + if not pattern: + continue + if any(char in pattern for char in "*?[]"): + if fnmatch.fnmatch(norm_path, pattern): + return True + elif norm_path == pattern or norm_path.startswith(f"{pattern.rstrip('/')}/"): + return True + return False diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md new file mode 100644 index 00000000..595cd5c1 --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md @@ -0,0 +1,177 @@ +--- +id: RAIL-BS-WP-0002 +type: workplan +title: "k3s and Kubernetes Platform Baseline" +domain: financials +repo: railiance-cluster +status: finished +owner: railiance +topic_slug: railiance +repo_goal_id: "70ab2379-fb9d-4fec-a09d-b2a717e4ace8" +state_hub_workstream_id: "4c63dfc6-9eac-4e79-9f77-8f644ad7147d" +created: "2026-03-09" +updated: "2026-03-10" +completed: "2026-03-10" +--- + +> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`). + +# k3s and Kubernetes Platform Baseline + +## Goal + +Install k3s, Helm, and the baseline Kubernetes services on the converged +HostEurope node. This workplan picks up exactly where `railiance-hosts` +leaves off: a hardened, verified OS node that is ready for Kubernetes. + +## Pre-condition + +`railiance-infra` converge + Goss verify must pass before any task here +is executed: + +```bash +# In railiance-infra/ +make converge +make verify # must exit 0 +``` + +## Boundary + +This repo owns everything from k3s upward. It must not re-configure items +defined in `railiance-infra/spec/server-baseline.yaml`. See ADR-003: +`railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`. + +**Out of scope here:** platform services (PostgreSQL, storage, identity) +→ `railiance-platform`. Application deployments (Gitea, coulomb services) +→ `railiance-apps`. + +--- + +## Tasks + +### T01 — Ansible playbook: install k3s (server mode) + +```task +id: T01 +status: done +priority: high +state_hub_task_id: "3f042630-eab0-4c6a-9167-e2b28ff20e40" +completed: "2026-03-10" +``` + +Harden `ansible/bootstrap.yml` to a production-ready k3s install: + +- Use the official k3s install script pinned to a specific version + (`INSTALL_K3S_VERSION=v1.35.1+k3s1`) +- `INSTALL_K3S_EXEC="server --cluster-init --write-kubeconfig-mode=644"` + (cluster-init enables embedded etcd for future HA expansion) +- Wait for node `Ready` before proceeding: + ```bash + k3s kubectl wait node --all --for=condition=Ready --timeout=120s + ``` +- Fetch kubeconfig to the control node as `~/.kube/config-hosteurope` + +**Done when:** `k3s kubectl get nodes` returns `Ready` from both the server +and the control node (via kubeconfig). + +--- + +### T02 — Helm installation + +```task +id: T02 +status: done +priority: high +state_hub_task_id: "e8510646-46ed-4697-a345-f3d3009eea78" +completed: "2026-03-10" +``` + +Add a task (or a role `roles/helm/`) that: + +1. Downloads the Helm binary (pinned version) to `/usr/local/bin/helm` +2. Verifies the checksum +3. Confirms `helm version` succeeds + +**Done when:** `helm version` succeeds on the HostEurope node. + +--- + +### T03 — Smoke test: k3s + Helm + +```task +id: T03 +status: done +priority: high +state_hub_task_id: "dab2c07f-8aa0-4635-8df6-857e87e93fc5" +completed: "2026-03-10" +``` + +Extend `tests/smoke_kube.sh` to assert: + +- `k3s kubectl get nodes` → node in Ready state +- `helm version` exits 0 +- CoreDNS pod running in `kube-system` +- Traefik ingress controller pod running (default in k3s) + +Run via: +```bash +ansible-playbook -i ansible/hosts.ini ansible/smoke.yml +``` +or directly over SSH if the kubeconfig is available locally. + +**Done when:** all assertions pass and the script exits 0. + +--- + +### T04 — Commit kubeconfig management notes + +```task +id: T04 +status: done +priority: medium +state_hub_task_id: "5c3d40e4-239b-488e-9519-6f7a38d2325f" +completed: "2026-03-10" +``` + +Document in `docs/kubeconfig.md`: + +- Where the kubeconfig is fetched to (`~/.kube/config-hosteurope`) +- How to merge it into `~/.kube/config` +- How to switch context: `kubectl config use-context default` +- Security note: kubeconfig is gitignored (contains cluster CA + client cert) + +**Done when:** doc written and committed. + +--- + +### T05 — Add `make k3s-install` and `make smoke` targets + +```task +id: T05 +status: done +priority: medium +state_hub_task_id: "7f9e0e58-a130-467a-a2d0-b3f2564e496f" +completed: "2026-03-10" +``` + +Add to Makefile (create one if none exists): + +```makefile +k3s-install: ## Install k3s and Helm on all inventory hosts + ansible-playbook -i ansible/hosts.ini ansible/bootstrap.yml + +smoke: ## Run Kubernetes smoke tests + bash tests/smoke_kube.sh +``` + +**Done when:** both targets work and are listed in `make help`. + +--- + +## References + +- Repo goal: `70ab2379-fb9d-4fec-a09d-b2a717e4ace8` (Install k3s and Kubernetes Baseline) +- Domain goal: `6f96c712-60e6-4ea9-ab06-168878eafbce` (Three-Phoenix Secure Kubernetes Infrastructure) +- Pre-condition: railiance-infra WP-0001 (Secure Single-Server Bootstrap) — completed 2026-03-09 +- Boundary ADR: `railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md` +- k3s releases: https://github.com/k3s-io/k3s/releases diff --git a/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md new file mode 100644 index 00000000..fe7b21da --- /dev/null +++ b/docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md @@ -0,0 +1,196 @@ +--- +id: RAIL-BS-WP-0003 +type: workplan +title: "pgpool CrashLoopBackOff on PostgreSQL HA failover — missing secret key" +domain: financials +repo: railiance-cluster +status: finished +owner: tegwick +created: "2026-03-10" +updated: "2026-03-10" +state_hub_workstream_id: "7ee9ee22-1fae-4567-9194-8d70a9e0f45b" +--- + +> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`). + +# Bug Report: pgpool CrashLoopBackOff on PostgreSQL HA failover + +## Summary + +On 2026-03-10 a PostgreSQL HA failover caused all three postgresql pods to +restart. pgpool — the connection pooler between Gitea and PostgreSQL — then +entered CrashLoopBackOff and produced no logs. As a result Gitea's login +and all write operations hung indefinitely. The root page was still served +(from Valkey cache) which masked the failure. + +The fix was to patch a missing key in a Kubernetes secret. The root cause is +that the `gitea-12.2.0` Helm chart (postgresql-ha subchart v16.2.2) does not +populate the `pgpool-password` key in the `gitea-postgresql-ha-postgresql` +secret, even though the pgpool pod requires it at startup. + +--- + +## Timeline + +| Time (UTC) | Event | +|---|---| +| ~09:45 | postgresql-0, postgresql-2 pods restarted (repmgr failover) | +| ~09:45 | pgpool pod restarted and entered CrashLoopBackOff | +| ~11:00 | User noticed Gitea login hanging; home page still loading | +| ~13:00 | Root cause identified: missing `pgpool-password` secret key | +| ~13:10 | Secret patched; pgpool pod deleted and restarted cleanly | +| ~13:15 | Gitea fully operational | + +--- + +## Root Cause + +The Bitnami `pgpool` container startup script reads the file +`/opt/bitnami/pgpool/secrets/pgpool-password`, which is mounted from the +`gitea-postgresql-ha-postgresql` Kubernetes Secret via a `subPath` volume +mount. That secret key was never created by the Helm chart, so the file did +not exist. The container exited immediately with no logs. + +The pod had been running for 20 days without a restart, so this gap was +never discovered during initial deployment. + +--- + +## Evidence + +```bash +# Secret was missing the pgpool-password key +sudo k3s kubectl get secret -n default gitea-postgresql-ha-postgresql -o yaml +# data: keys were password, postgres-password, repmgr-password only +# pgpool-password was absent + +# pgpool pod describe showed 824 back-off restarts over 173 minutes +# No logs in either current or --previous output +sudo k3s kubectl logs -n default --previous +# (empty) + +# Gitea process had zero TCP connections to PostgreSQL port 5432 +# but many connections to Valkey port 6379 +cat /proc//net/tcp | grep 1538 # 1538 = 5432 hex — no results +``` + +--- + +## Immediate Fix Applied + +```bash +# Add the missing key (value = sr-check-password = changeme4 = base64: Y2hhbmdlbWU0) +sudo k3s kubectl patch secret -n default gitea-postgresql-ha-postgresql \ + --type='json' \ + -p='[{"op":"add","path":"/data/pgpool-password","value":"Y2hhbmdlbWU0"}]' + +# Restart pgpool +sudo k3s kubectl delete pod -n default +``` + +--- + +## Risk: Fix Will Be Lost on helm upgrade + +The patched secret is managed by Helm (annotation: +`meta.helm.sh/release-name: gitea`). A `helm upgrade` will regenerate the +secret from the chart template, which does not include `pgpool-password`, +and the bug will recur. + +--- + +## Tasks + +### T01 — Add pgpool-password to Helm values + +```task +id: T01 +status: done +priority: high +state_hub_task_id: "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc" +``` + +Create or update `helm/gitea-values.yaml` (or equivalent) to permanently +include the pgpool-password so it survives `helm upgrade`: + +```yaml +postgresql-ha: + postgresql: + pgpoolPassword: +``` + +**Done when:** `helm upgrade gitea` completes and pgpool starts cleanly +without manual secret patching. + +--- + +### T02 — Add pgpool health check to smoke test + +```task +id: T02 +status: done +priority: high +state_hub_task_id: "ab166073-30a7-4702-a037-4091e8706e20" +``` + +Extend `tests/smoke_kube.sh` to assert: + +```bash +# All postgresql-ha pods Running +kubectl get pods -n default | grep gitea-postgresql-ha | grep -v Running && exit 1 + +# pgpool specifically not in CrashLoopBackOff +kubectl get pod -n default -l app.kubernetes.io/component=pgpool \ + -o jsonpath='{.items[0].status.containerStatuses[0].state}' | grep -v crash +``` + +**Done when:** the smoke test catches a pgpool failure within 5 minutes. + +--- + +### T03 — Add HA failover test + +```task +id: T03 +status: done +priority: high +state_hub_task_id: "140da396-8e30-4f4d-b88c-c42c0cd46c01" +``` + +Create `tests/test_ha_failover.sh` that: + +1. Records Gitea login response time (baseline) +2. Kills the primary PostgreSQL pod: `kubectl delete pod gitea-postgresql-ha-postgresql-0 -n default` +3. Waits for repmgr to promote a replica (max 60s) +4. Asserts Gitea login POST still succeeds within 10s +5. Asserts pgpool pod is Running (not CrashLoopBackOff) +6. Asserts all postgresql pods return to Running + +This test must pass before any PostgreSQL HA deployment is considered done. + +**Done when:** script exits 0 against a live cluster. + +--- + +### T04 — Document the incident in docs/ + +```task +id: T04 +status: done +priority: medium +state_hub_task_id: "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d" +``` + +Add `docs/incidents/2026-03-10-pgpool-missing-secret.md` with the full +timeline, root cause, and fix, so future operators understand what happened +and how to recover. + +**Done when:** doc committed and linked from `docs/README.md`. + +--- + +## References + +- Bitnami postgresql-ha chart v16.2.2 +- Gitea Helm chart v12.2.0 +- Related decision: D3 (HA testing policy) in `DECISIONS.md`