Implement ACTIVITY-WP-0021 production automation reliability

Root-cause IssueSink 503 (dead Forgejo PAT on issue-core), add state-hub
task sink path B, log runs before emit, harden sync_schedules, deterministic
SBOM/triage reports, DB probe thrash fix, and prod automation-status helper.

src/activity_core/sync_schedules.py

@@ -37,12 +37,15 @@ class ScheduleSyncResult:
     upserted: int = 0
     paused: int = 0
     deleted_orphans: int = 0
+    errors: int = 0
+    error_details: list[str] | None = None
 
     def to_dict(self) -> dict[str, int]:
         return {
             "upserted": self.upserted,
             "paused": self.paused,
             "deleted_orphans": self.deleted_orphans,
+            "errors": self.errors,
         }
 
 
@@ -85,9 +88,13 @@ async def sync_schedule_rows(
     client: Client,
     rows: Sequence[ActivityDefinitionRow],
 ) -> ScheduleSyncResult:
-    """Reconcile Temporal Schedules against already-loaded definition rows."""
+    """Reconcile Temporal Schedules against already-loaded definition rows.
+
+    ACTIVITY-WP-0021-T06: one failing upsert (e.g. ScheduleAlreadyRunningError
+    on pause/unpause) must not abort the remaining rows.
+    """
     valid_schedule_activity_ids: set[str] = set()
-    result = ScheduleSyncResult()
+    result = ScheduleSyncResult(error_details=[])
 
     for row in rows:
         defn = _row_to_domain(row)
@@ -99,7 +106,15 @@ async def sync_schedule_rows(
 
         valid_schedule_activity_ids.add(_valid_schedule_activity_id(defn))
 
-        await upsert_schedule(client, defn)
+        try:
+            await upsert_schedule(client, defn)
+        except Exception as exc:  # noqa: BLE001 — continue reconcile for other rows
+            result.errors += 1
+            detail = f"{defn.id} ({defn.name}): {type(exc).__name__}: {exc}"
+            result.error_details.append(detail)
+            logger.error("upsert_schedule failed for activity %s — continuing: %s", defn.id, exc)
+            continue
+
         if defn.enabled:
             result.upserted += 1
             logger.info("upserted schedule for activity %s (%s)", defn.id, defn.name)
@@ -108,18 +123,33 @@ async def sync_schedule_rows(
             logger.info("upserted paused schedule for disabled activity %s", defn.id)
 
     # Tombstone cleanup: remove Temporal Schedules with no matching DB row.
-    existing_schedules = await list_schedules(client)
+    try:
+        existing_schedules = await list_schedules(client)
+    except Exception as exc:  # noqa: BLE001
+        result.errors += 1
+        detail = f"list_schedules: {type(exc).__name__}: {exc}"
+        result.error_details.append(detail)
+        logger.error("list_schedules failed — skipping orphan cleanup: %s", exc)
+        existing_schedules = []
+
     for entry in existing_schedules:
         if entry["activity_id"] not in valid_schedule_activity_ids:
-            await delete_schedule(client, entry["activity_id"])
-            result.deleted_orphans += 1
-            logger.info("deleted orphaned schedule %s", entry["schedule_id"])
+            try:
+                await delete_schedule(client, entry["activity_id"])
+                result.deleted_orphans += 1
+                logger.info("deleted orphaned schedule %s", entry["schedule_id"])
+            except Exception as exc:  # noqa: BLE001
+                result.errors += 1
+                detail = f"delete {entry['schedule_id']}: {type(exc).__name__}: {exc}"
+                result.error_details.append(detail)
+                logger.error("delete_schedule failed for %s — continuing: %s", entry["schedule_id"], exc)
 
     logger.info(
-        "sync_schedules complete — upserted=%d paused=%d deleted_orphans=%d",
+        "sync_schedules complete — upserted=%d paused=%d deleted_orphans=%d errors=%d",
         result.upserted,
         result.paused,
         result.deleted_orphans,
+        result.errors,
     )
     return result
 
@@ -162,8 +192,12 @@ async def main() -> None:
         "Synced schedules: "
         f"upserted={result.upserted} "
         f"paused={result.paused} "
-        f"deleted_orphans={result.deleted_orphans}"
+        f"deleted_orphans={result.deleted_orphans} "
+        f"errors={result.errors}"
     )
+    if result.error_details:
+        for detail in result.error_details:
+            print(f"  error: {detail}")
 
 
 if __name__ == "__main__":