feat(maintenance): nightly stale SSH forward cleanup at 03:00
Add bridge maintenance cleanup to detect reverse tunnels whose remote port is bound but no longer forwards (zombie sshd sessions), kill the stale listeners on the remote host, and optionally restart the tunnel. Includes install-cron/uninstall-cron/show-cron helpers and README notes for the actcore-state-hub-bridge failure mode we hit on railiance01.
This commit is contained in:
parent
a6857fb8f7
commit
4e9882909f
5 changed files with 565 additions and 1 deletions
|
|
@ -12,6 +12,13 @@ from typing import Optional
|
|||
import typer
|
||||
|
||||
from bridge.audit import AuditLogger
|
||||
from bridge.cleanup import (
|
||||
build_cron_line,
|
||||
cleanup_all_tunnels,
|
||||
install_cleanup_cron,
|
||||
read_installed_cron,
|
||||
uninstall_cleanup_cron,
|
||||
)
|
||||
from bridge.config import ConfigError, load_config
|
||||
from bridge.diagnostics import check_all_tunnels, check_tunnel
|
||||
from bridge.manager import TunnelManager
|
||||
|
|
@ -25,9 +32,11 @@ app = typer.Typer(
|
|||
|
||||
targets_app = typer.Typer(help="Inspect infrastructure targets from the OpsCatalog.")
|
||||
catalog_app = typer.Typer(help="Inspect and validate the OpsCatalog.")
|
||||
maintenance_app = typer.Typer(help="Scheduled maintenance for tunnel hygiene.")
|
||||
|
||||
app.add_typer(targets_app, name="targets")
|
||||
app.add_typer(catalog_app, name="catalog")
|
||||
app.add_typer(maintenance_app, name="maintenance")
|
||||
|
||||
|
||||
def _state_dir() -> Path:
|
||||
|
|
@ -661,6 +670,90 @@ Full specification:
|
|||
"""
|
||||
|
||||
|
||||
@maintenance_app.command("cleanup")
|
||||
def maintenance_cleanup(
|
||||
tunnel: Optional[str] = typer.Argument(
|
||||
None,
|
||||
help="Tunnel name (omit for all reverse tunnels)",
|
||||
),
|
||||
restart: bool = typer.Option(
|
||||
False,
|
||||
"--restart",
|
||||
help="Restart tunnels after clearing stale remote bindings",
|
||||
),
|
||||
as_json: bool = typer.Option(False, "--json", help="Output as JSON"),
|
||||
):
|
||||
"""Clear stale SSH remote port forwards that block tunnel reconnects."""
|
||||
cfg = _load_or_exit()
|
||||
sd = _state_dir()
|
||||
state_mgr = StateManager(state_dir=sd)
|
||||
|
||||
try:
|
||||
report = cleanup_all_tunnels(
|
||||
cfg,
|
||||
state_mgr,
|
||||
restart=restart,
|
||||
tunnel_name=tunnel,
|
||||
)
|
||||
except KeyError:
|
||||
typer.echo(f"Error: tunnel '{tunnel}' not found in config", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
if as_json:
|
||||
payload = {
|
||||
"cleaned_count": report.cleaned_count,
|
||||
"actions": [
|
||||
{"tunnel": a.tunnel, "action": a.action, "detail": a.detail}
|
||||
for a in report.actions
|
||||
],
|
||||
}
|
||||
typer.echo(json.dumps(payload, indent=2))
|
||||
return
|
||||
|
||||
if not report.actions:
|
||||
typer.echo("No reverse tunnels configured.")
|
||||
return
|
||||
|
||||
for action in report.actions:
|
||||
typer.echo(f"{action.tunnel}: {action.action} — {action.detail}")
|
||||
typer.echo(f"done ({report.cleaned_count} cleaned)")
|
||||
|
||||
|
||||
@maintenance_app.command("install-cron")
|
||||
def maintenance_install_cron():
|
||||
"""Install a 03:00 daily cron job for `bridge maintenance cleanup --restart`."""
|
||||
installed, message = install_cleanup_cron()
|
||||
if installed:
|
||||
typer.echo("Installed nightly cleanup cron:")
|
||||
typer.echo(f" {message}")
|
||||
else:
|
||||
typer.echo(message)
|
||||
raise typer.Exit(2)
|
||||
|
||||
|
||||
@maintenance_app.command("uninstall-cron")
|
||||
def maintenance_uninstall_cron():
|
||||
"""Remove the nightly cleanup cron job."""
|
||||
removed, message = uninstall_cleanup_cron()
|
||||
if removed:
|
||||
typer.echo(message)
|
||||
else:
|
||||
typer.echo(message)
|
||||
raise typer.Exit(2)
|
||||
|
||||
|
||||
@maintenance_app.command("show-cron")
|
||||
def maintenance_show_cron():
|
||||
"""Show the configured nightly cleanup cron line."""
|
||||
existing = read_installed_cron()
|
||||
if existing:
|
||||
typer.echo(existing)
|
||||
else:
|
||||
typer.echo("Nightly cleanup cron is not installed.")
|
||||
typer.echo("Would install:")
|
||||
typer.echo(f" {build_cron_line()}")
|
||||
|
||||
|
||||
@app.command()
|
||||
def conventions():
|
||||
"""Show the actor naming conventions enforced by tunnels.yaml."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue