feat(maintenance): nightly stale SSH forward cleanup at 03:00

Add bridge maintenance cleanup to detect reverse tunnels whose remote
port is bound but no longer forwards (zombie sshd sessions), kill the
stale listeners on the remote host, and optionally restart the tunnel.

Includes install-cron/uninstall-cron/show-cron helpers and README notes
for the actcore-state-hub-bridge failure mode we hit on railiance01.
This commit is contained in:
tegwick 2026-06-19 15:59:27 +02:00
parent a6857fb8f7
commit 4e9882909f
5 changed files with 565 additions and 1 deletions

View file

@ -12,6 +12,13 @@ from typing import Optional
import typer
from bridge.audit import AuditLogger
from bridge.cleanup import (
build_cron_line,
cleanup_all_tunnels,
install_cleanup_cron,
read_installed_cron,
uninstall_cleanup_cron,
)
from bridge.config import ConfigError, load_config
from bridge.diagnostics import check_all_tunnels, check_tunnel
from bridge.manager import TunnelManager
@ -25,9 +32,11 @@ app = typer.Typer(
targets_app = typer.Typer(help="Inspect infrastructure targets from the OpsCatalog.")
catalog_app = typer.Typer(help="Inspect and validate the OpsCatalog.")
maintenance_app = typer.Typer(help="Scheduled maintenance for tunnel hygiene.")
app.add_typer(targets_app, name="targets")
app.add_typer(catalog_app, name="catalog")
app.add_typer(maintenance_app, name="maintenance")
def _state_dir() -> Path:
@ -661,6 +670,90 @@ Full specification:
"""
@maintenance_app.command("cleanup")
def maintenance_cleanup(
tunnel: Optional[str] = typer.Argument(
None,
help="Tunnel name (omit for all reverse tunnels)",
),
restart: bool = typer.Option(
False,
"--restart",
help="Restart tunnels after clearing stale remote bindings",
),
as_json: bool = typer.Option(False, "--json", help="Output as JSON"),
):
"""Clear stale SSH remote port forwards that block tunnel reconnects."""
cfg = _load_or_exit()
sd = _state_dir()
state_mgr = StateManager(state_dir=sd)
try:
report = cleanup_all_tunnels(
cfg,
state_mgr,
restart=restart,
tunnel_name=tunnel,
)
except KeyError:
typer.echo(f"Error: tunnel '{tunnel}' not found in config", err=True)
raise typer.Exit(1)
if as_json:
payload = {
"cleaned_count": report.cleaned_count,
"actions": [
{"tunnel": a.tunnel, "action": a.action, "detail": a.detail}
for a in report.actions
],
}
typer.echo(json.dumps(payload, indent=2))
return
if not report.actions:
typer.echo("No reverse tunnels configured.")
return
for action in report.actions:
typer.echo(f"{action.tunnel}: {action.action}{action.detail}")
typer.echo(f"done ({report.cleaned_count} cleaned)")
@maintenance_app.command("install-cron")
def maintenance_install_cron():
"""Install a 03:00 daily cron job for `bridge maintenance cleanup --restart`."""
installed, message = install_cleanup_cron()
if installed:
typer.echo("Installed nightly cleanup cron:")
typer.echo(f" {message}")
else:
typer.echo(message)
raise typer.Exit(2)
@maintenance_app.command("uninstall-cron")
def maintenance_uninstall_cron():
"""Remove the nightly cleanup cron job."""
removed, message = uninstall_cleanup_cron()
if removed:
typer.echo(message)
else:
typer.echo(message)
raise typer.Exit(2)
@maintenance_app.command("show-cron")
def maintenance_show_cron():
"""Show the configured nightly cleanup cron line."""
existing = read_installed_cron()
if existing:
typer.echo(existing)
else:
typer.echo("Nightly cleanup cron is not installed.")
typer.echo("Would install:")
typer.echo(f" {build_cron_line()}")
@app.command()
def conventions():
"""Show the actor naming conventions enforced by tunnels.yaml."""