feat: TTL enforcement and operational hardening (SAND-WP-0009)
Add TTL parser, expires_at on create, extend_ttl and expire/reap APIs, activity-core integration doc, repo classification, registry refresh, HTTP parity, and 69 tests.
This commit is contained in:
parent
b58191b23e
commit
df658e7ef9
20 changed files with 913 additions and 39 deletions
|
|
@ -6,6 +6,8 @@ from fastapi import FastAPI, HTTPException
|
|||
|
||||
from sandboxer.core.manager import SandboxManager
|
||||
from sandboxer.models import (
|
||||
ExpireActionResult,
|
||||
ExtendTtlRequest,
|
||||
SandboxCreateRequest,
|
||||
SandboxStatus,
|
||||
SnapshotRecord,
|
||||
|
|
@ -82,4 +84,29 @@ def get_snapshot(snapshot_id: str) -> SnapshotRecord:
|
|||
record = _manager.get_snapshot(snapshot_id)
|
||||
if not record:
|
||||
raise HTTPException(status_code=404, detail="snapshot not found")
|
||||
return record
|
||||
return record
|
||||
|
||||
|
||||
@app.post("/v1/sandboxes/{sandbox_id}/recreate", response_model=SandboxStatus)
|
||||
def recreate_sandbox(sandbox_id: str) -> SandboxStatus:
|
||||
try:
|
||||
return _manager.recreate(sandbox_id)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.patch("/v1/sandboxes/{sandbox_id}/ttl", response_model=SandboxStatus)
|
||||
def extend_sandbox_ttl(sandbox_id: str, request: ExtendTtlRequest) -> SandboxStatus:
|
||||
try:
|
||||
return _manager.extend_ttl(sandbox_id, request.duration)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except (RuntimeError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.post("/v1/sandboxes/expire", response_model=list[ExpireActionResult])
|
||||
def expire_sandboxes(apply: bool = False) -> list[ExpireActionResult]:
|
||||
return _manager.expire(apply=apply)
|
||||
|
|
@ -90,6 +90,7 @@ def sandbox_create(
|
|||
actor: Annotated[str, typer.Option(help="Consumer actor type")] = "adm",
|
||||
project: Annotated[str, typer.Option(help="Calling project id")] = "sand-boxer",
|
||||
host: Annotated[str | None, typer.Option(help="Override placement host")] = None,
|
||||
ttl: Annotated[str | None, typer.Option(help="TTL override (e.g. 4h)")] = None,
|
||||
) -> None:
|
||||
"""Provision a sandbox. No args → canary self-deploy of sand-boxer."""
|
||||
parsed = _parse_inputs(input or [])
|
||||
|
|
@ -98,6 +99,7 @@ def sandbox_create(
|
|||
profile=resolved_profile,
|
||||
inputs=resolved_inputs,
|
||||
consumer=Consumer(actor=ActorType(actor), project=project),
|
||||
ttl=ttl,
|
||||
)
|
||||
manager = SandboxManager()
|
||||
try:
|
||||
|
|
@ -196,6 +198,33 @@ def snapshots_get(snapshot_id: str) -> None:
|
|||
_print_json(record.model_dump(mode="json"))
|
||||
|
||||
|
||||
@app.command("extend-ttl")
|
||||
def sandbox_extend_ttl(
|
||||
sandbox_id: str,
|
||||
duration: Annotated[str, typer.Option("--duration", help="Extension duration (e.g. 2h)")],
|
||||
) -> None:
|
||||
"""Extend sandbox time-to-live (capped at profile max)."""
|
||||
manager = SandboxManager()
|
||||
try:
|
||||
status = manager.extend_ttl(sandbox_id, duration)
|
||||
except (KeyError, RuntimeError, ValueError) as exc:
|
||||
typer.echo(f"Error: {exc}", err=True)
|
||||
raise typer.Exit(code=1) from exc
|
||||
_print_json(status.model_dump(mode="json"))
|
||||
|
||||
|
||||
@app.command("expire")
|
||||
def sandbox_expire(
|
||||
apply: Annotated[bool, typer.Option("--apply", help="Destroy expired sandboxes")] = False,
|
||||
) -> None:
|
||||
"""Report or destroy sandboxes past TTL or idle-reap threshold."""
|
||||
manager = SandboxManager()
|
||||
results = manager.expire(apply=apply)
|
||||
mode = "apply" if apply else "dry-run"
|
||||
typer.echo(f"expire ({mode}): {len(results)} candidate(s)", err=True)
|
||||
_print_json([r.model_dump(mode="json") for r in results])
|
||||
|
||||
|
||||
@app.command("recreate")
|
||||
def sandbox_recreate(sandbox_id: str) -> None:
|
||||
"""Destroy and reprovision from stored inputs."""
|
||||
|
|
|
|||
|
|
@ -3,10 +3,17 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sandboxer.extensions.registry import load_extension, resolve_backend
|
||||
from sandboxer.lifecycle.expire import (
|
||||
ExpireCandidate,
|
||||
apply_expired_state,
|
||||
find_expire_candidates,
|
||||
)
|
||||
from sandboxer.lifecycle.state_hub import emit_lifecycle_event, event_type_for_state
|
||||
from sandboxer.lifecycle.store import SandboxStore, utcnow
|
||||
from sandboxer.lifecycle.ttl import expires_at_from, extend_expires_at, resolve_initial_ttl
|
||||
from sandboxer.models import (
|
||||
Consumer,
|
||||
ExpireActionResult,
|
||||
MeterRecord,
|
||||
Reachability,
|
||||
SandboxCreateRequest,
|
||||
|
|
@ -60,6 +67,18 @@ class SandboxManager:
|
|||
return extension.config.get("provider", "saas")
|
||||
return resolve_host(profile, override=host_override)
|
||||
|
||||
@staticmethod
|
||||
def _assign_ttl(
|
||||
status: SandboxStatus,
|
||||
profile,
|
||||
*,
|
||||
request_ttl: str | None,
|
||||
) -> None:
|
||||
ttl_str = resolve_initial_ttl(profile, request_ttl)
|
||||
anchor = status.ready_at or utcnow()
|
||||
status.ttl = ttl_str
|
||||
status.expires_at = expires_at_from(anchor, ttl_str)
|
||||
|
||||
def create(self, request: SandboxCreateRequest, *, host: str | None = None) -> SandboxStatus:
|
||||
profile = load_profile(request.profile)
|
||||
extension = resolve_extension(profile, request.inputs, host_override=host)
|
||||
|
|
@ -119,6 +138,7 @@ class SandboxManager:
|
|||
status.state = SandboxState.READY
|
||||
status.ready_at = utcnow()
|
||||
status.updated_at = status.ready_at
|
||||
self._assign_ttl(status, profile, request_ttl=request.ttl)
|
||||
|
||||
if wants_telemetry and provision_before:
|
||||
provision_after = collect_host_snapshot(resolved_host)
|
||||
|
|
@ -224,11 +244,98 @@ class SandboxManager:
|
|||
profile=existing.profile_id,
|
||||
inputs=dict(existing.inputs),
|
||||
consumer=existing.consumer,
|
||||
ttl=existing.ttl,
|
||||
)
|
||||
if existing.state != SandboxState.DESTROYED:
|
||||
self.destroy(sandbox_id)
|
||||
return self.create(request, host=existing.host)
|
||||
|
||||
def extend_ttl(self, sandbox_id: str, duration: str) -> SandboxStatus:
|
||||
status = self.store.get(sandbox_id)
|
||||
if not status:
|
||||
raise KeyError(f"Sandbox not found: {sandbox_id}")
|
||||
if status.state not in (SandboxState.READY, SandboxState.ACTIVE):
|
||||
raise RuntimeError(
|
||||
f"Cannot extend TTL for sandbox in state {status.state.value}"
|
||||
)
|
||||
if not status.expires_at or not status.ready_at:
|
||||
raise RuntimeError("Sandbox has no expiry metadata")
|
||||
|
||||
profile = load_profile(status.profile_id)
|
||||
new_expires, applied = extend_expires_at(
|
||||
status.expires_at,
|
||||
anchor=status.ready_at,
|
||||
extension=duration,
|
||||
max_duration=profile.ttl.max,
|
||||
)
|
||||
status.expires_at = new_expires
|
||||
status.ttl = applied
|
||||
status.updated_at = utcnow()
|
||||
self.store.save(status)
|
||||
emit_lifecycle_event(
|
||||
status,
|
||||
summary=f"TTL extended by {applied} (expires {new_expires.isoformat()})",
|
||||
event_type="note",
|
||||
)
|
||||
return status
|
||||
|
||||
def expire(
|
||||
self,
|
||||
*,
|
||||
apply: bool = False,
|
||||
now=None,
|
||||
) -> list[ExpireActionResult]:
|
||||
candidates = find_expire_candidates(self.store, now=now)
|
||||
results: list[ExpireActionResult] = []
|
||||
|
||||
for candidate in candidates:
|
||||
if not apply:
|
||||
results.append(
|
||||
ExpireActionResult(
|
||||
sandbox_id=candidate.sandbox_id,
|
||||
reason=candidate.reason,
|
||||
action="dry-run",
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
status = self.store.get(candidate.sandbox_id)
|
||||
if not status or status.state not in (
|
||||
SandboxState.READY,
|
||||
SandboxState.ACTIVE,
|
||||
):
|
||||
continue
|
||||
status = apply_expired_state(status, now=now)
|
||||
self.store.save(status)
|
||||
emit_lifecycle_event(
|
||||
status,
|
||||
summary=f"Sandbox expired ({candidate.reason})",
|
||||
event_type=event_type_for_state(status.state),
|
||||
)
|
||||
self.destroy(candidate.sandbox_id)
|
||||
results.append(
|
||||
ExpireActionResult(
|
||||
sandbox_id=candidate.sandbox_id,
|
||||
reason=candidate.reason,
|
||||
action="destroyed",
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
results.append(
|
||||
ExpireActionResult(
|
||||
sandbox_id=candidate.sandbox_id,
|
||||
reason=candidate.reason,
|
||||
action="failed",
|
||||
error=str(exc),
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def list_expire_candidates(self, *, now=None) -> list[ExpireCandidate]:
|
||||
return find_expire_candidates(self.store, now=now)
|
||||
|
||||
def snapshot(self, sandbox_id: str, *, name: str | None = None) -> SnapshotRecord:
|
||||
status = self.store.get(sandbox_id)
|
||||
if not status:
|
||||
|
|
@ -345,6 +452,7 @@ class SandboxManager:
|
|||
status.state = SandboxState.READY
|
||||
status.ready_at = utcnow()
|
||||
status.updated_at = status.ready_at
|
||||
self._assign_ttl(status, profile, request_ttl=None)
|
||||
self.store.save(status)
|
||||
emit_lifecycle_event(
|
||||
status,
|
||||
|
|
|
|||
77
src/sandboxer/lifecycle/expire.py
Normal file
77
src/sandboxer/lifecycle/expire.py
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
"""TTL and idle-reap expiry candidate selection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from typing import Literal
|
||||
|
||||
from sandboxer.lifecycle.store import SandboxStore
|
||||
from sandboxer.lifecycle.ttl import is_idle_expired, is_past_expiry
|
||||
from sandboxer.models import SandboxState, SandboxStatus
|
||||
from sandboxer.profiles.loader import load_profile
|
||||
|
||||
ExpireReason = Literal["ttl", "idle"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExpireCandidate:
|
||||
sandbox_id: str
|
||||
profile_id: str
|
||||
reason: ExpireReason
|
||||
expires_at: datetime | None = None
|
||||
updated_at: datetime | None = None
|
||||
|
||||
|
||||
_LIVE_STATES = frozenset({SandboxState.READY, SandboxState.ACTIVE})
|
||||
|
||||
|
||||
def find_expire_candidates(
|
||||
store: SandboxStore,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
) -> list[ExpireCandidate]:
|
||||
ref = now or datetime.now(UTC)
|
||||
candidates: list[ExpireCandidate] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for status in store.list_all():
|
||||
if status.state not in _LIVE_STATES:
|
||||
continue
|
||||
|
||||
if is_past_expiry(status.expires_at, now=ref):
|
||||
candidates.append(
|
||||
ExpireCandidate(
|
||||
sandbox_id=status.sandbox_id,
|
||||
profile_id=status.profile_id,
|
||||
reason="ttl",
|
||||
expires_at=status.expires_at,
|
||||
)
|
||||
)
|
||||
seen.add(status.sandbox_id)
|
||||
continue
|
||||
|
||||
try:
|
||||
profile = load_profile(status.profile_id)
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
|
||||
if is_idle_expired(status.updated_at, profile.ttl.idle_reap, now=ref):
|
||||
candidates.append(
|
||||
ExpireCandidate(
|
||||
sandbox_id=status.sandbox_id,
|
||||
profile_id=status.profile_id,
|
||||
reason="idle",
|
||||
updated_at=status.updated_at,
|
||||
)
|
||||
)
|
||||
seen.add(status.sandbox_id)
|
||||
|
||||
return sorted(candidates, key=lambda c: c.sandbox_id)
|
||||
|
||||
|
||||
def apply_expired_state(status: SandboxStatus, *, now: datetime | None = None) -> SandboxStatus:
|
||||
ref = now or datetime.now(UTC)
|
||||
status.state = SandboxState.EXPIRED
|
||||
status.updated_at = ref
|
||||
return status
|
||||
|
|
@ -38,6 +38,8 @@ def emit_lifecycle_event(
|
|||
"consumer": status.consumer.model_dump(),
|
||||
"actor_type": status.consumer.actor.value,
|
||||
"state": status.state.value,
|
||||
"ttl": status.ttl,
|
||||
"expires_at": status.expires_at.isoformat() if status.expires_at else None,
|
||||
"reachability": status.reachability.model_dump() if status.reachability else None,
|
||||
"telemetry": status.telemetry,
|
||||
"timestamps": {
|
||||
|
|
@ -58,6 +60,6 @@ def emit_lifecycle_event(
|
|||
|
||||
|
||||
def event_type_for_state(state: SandboxState) -> str:
|
||||
if state in (SandboxState.READY, SandboxState.DESTROYED):
|
||||
if state in (SandboxState.READY, SandboxState.DESTROYED, SandboxState.EXPIRED):
|
||||
return "milestone"
|
||||
return "note"
|
||||
121
src/sandboxer/lifecycle/ttl.py
Normal file
121
src/sandboxer/lifecycle/ttl.py
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
"""TTL duration parsing and expiry calculation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sandboxer.models import Profile
|
||||
|
||||
_DURATION_RE = re.compile(r"^(\d+)([smhd])$", re.IGNORECASE)
|
||||
_UNIT_SECONDS = {"s": 1, "m": 60, "h": 3600, "d": 86400}
|
||||
|
||||
|
||||
def parse_duration(value: str) -> timedelta:
|
||||
"""Parse a duration string like ``4h``, ``30m``, ``1d``."""
|
||||
raw = value.strip()
|
||||
match = _DURATION_RE.match(raw)
|
||||
if not match:
|
||||
raise ValueError(f"Invalid duration: {value!r} (expected e.g. 4h, 30m, 1d)")
|
||||
amount = int(match.group(1))
|
||||
if amount <= 0:
|
||||
raise ValueError(f"Duration must be positive: {value!r}")
|
||||
unit = match.group(2).lower()
|
||||
return timedelta(seconds=amount * _UNIT_SECONDS[unit])
|
||||
|
||||
|
||||
def duration_seconds(value: str) -> int:
|
||||
return int(parse_duration(value).total_seconds())
|
||||
|
||||
|
||||
def resolve_initial_ttl(profile: Profile, request_ttl: str | None) -> str:
|
||||
"""Pick create TTL from request override or profile default, capped at profile max."""
|
||||
requested = request_ttl or profile.ttl.default
|
||||
return cap_duration(requested, profile.ttl.max)
|
||||
|
||||
|
||||
def cap_duration(requested: str, maximum: str) -> str:
|
||||
"""Return ``requested`` if within ``maximum``; otherwise return ``maximum``."""
|
||||
req_s = duration_seconds(requested)
|
||||
max_s = duration_seconds(maximum)
|
||||
if req_s > max_s:
|
||||
return maximum
|
||||
return requested
|
||||
|
||||
|
||||
def expires_at_from(base: datetime, duration: str) -> datetime:
|
||||
if base.tzinfo is None:
|
||||
base = base.replace(tzinfo=UTC)
|
||||
return base + parse_duration(duration)
|
||||
|
||||
|
||||
def cap_expires_at(
|
||||
candidate: datetime,
|
||||
*,
|
||||
anchor: datetime,
|
||||
max_duration: str,
|
||||
) -> datetime:
|
||||
"""Cap ``candidate`` so it does not exceed ``anchor + max_duration``."""
|
||||
ceiling = expires_at_from(anchor, max_duration)
|
||||
if candidate.tzinfo is None:
|
||||
candidate = candidate.replace(tzinfo=UTC)
|
||||
return min(candidate, ceiling)
|
||||
|
||||
|
||||
def extend_expires_at(
|
||||
current: datetime,
|
||||
*,
|
||||
anchor: datetime,
|
||||
extension: str,
|
||||
max_duration: str,
|
||||
) -> tuple[datetime, str]:
|
||||
"""Add ``extension`` to ``current`` and cap at ``anchor + max_duration``."""
|
||||
now = datetime.now(UTC)
|
||||
base = max(current, now)
|
||||
proposed = expires_at_from(base, extension)
|
||||
capped = cap_expires_at(proposed, anchor=anchor, max_duration=max_duration)
|
||||
applied = extension
|
||||
if capped < proposed:
|
||||
remaining = capped - base
|
||||
if remaining.total_seconds() <= 0:
|
||||
raise ValueError(f"Cannot extend: already at profile max ({max_duration})")
|
||||
applied = format_timedelta(remaining)
|
||||
return capped, applied
|
||||
|
||||
|
||||
def format_timedelta(delta: timedelta) -> str:
|
||||
seconds = int(delta.total_seconds())
|
||||
if seconds <= 0:
|
||||
raise ValueError("Duration must be positive")
|
||||
if seconds >= 86400 and seconds % 86400 == 0:
|
||||
return f"{seconds // 86400}d"
|
||||
if seconds >= 3600 and seconds % 3600 == 0:
|
||||
return f"{seconds // 3600}h"
|
||||
if seconds >= 60 and seconds % 60 == 0:
|
||||
return f"{seconds // 60}m"
|
||||
return f"{seconds}s"
|
||||
|
||||
|
||||
def is_past_expiry(expires_at: datetime | None, *, now: datetime | None = None) -> bool:
|
||||
if expires_at is None:
|
||||
return False
|
||||
ref = now or datetime.now(UTC)
|
||||
if expires_at.tzinfo is None:
|
||||
expires_at = expires_at.replace(tzinfo=UTC)
|
||||
return expires_at <= ref
|
||||
|
||||
|
||||
def is_idle_expired(
|
||||
updated_at: datetime,
|
||||
idle_reap: str | None,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
) -> bool:
|
||||
if not idle_reap:
|
||||
return False
|
||||
ref = now or datetime.now(UTC)
|
||||
if updated_at.tzinfo is None:
|
||||
updated_at = updated_at.replace(tzinfo=UTC)
|
||||
return updated_at + parse_duration(idle_reap) <= ref
|
||||
|
||||
|
||||
|
|
@ -164,6 +164,8 @@ class SandboxStatus(BaseModel):
|
|||
host: str | None = None
|
||||
reachability: Reachability | None = None
|
||||
inputs: dict[str, str] = Field(default_factory=dict)
|
||||
ttl: str | None = None
|
||||
expires_at: datetime | None = None
|
||||
error: str | None = None
|
||||
meter: MeterRecord | None = None
|
||||
telemetry: dict | None = None # IntrospectionReport JSON when canary
|
||||
|
|
@ -173,6 +175,17 @@ class SandboxStatus(BaseModel):
|
|||
destroyed_at: datetime | None = None
|
||||
|
||||
|
||||
class ExtendTtlRequest(BaseModel):
|
||||
duration: str
|
||||
|
||||
|
||||
class ExpireActionResult(BaseModel):
|
||||
sandbox_id: str
|
||||
reason: Literal["ttl", "idle"]
|
||||
action: Literal["dry-run", "expired", "destroyed", "failed"]
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class SnapshotRestoreRequest(BaseModel):
|
||||
host: str | None = None
|
||||
consumer: Consumer | None = None
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue