`d8fc3ba`

Respawn crashed ers from BorderManager watchdog

When ers exits unexpectedly the user is left with no borders and no
indication anything went wrong, since BorderManager just stores the
child handle and never reaps it. Add a health_check called from the
50ms poll-timer tick (throttled to ~1Hz) that try_waits on the child,
respawns on death, and rate-limits to 5 restarts per minute followed
by a 5-minute backoff so a permanently-broken ers doesn't fork-bomb.

Authored by mfwolffe <wolffemf@dukes.jmu.edu> 2 weeks ago

SHA: d8fc3ba4a7614f7da69f3bd52629c55e54db6a3f
Parents: 102c4cf
Tree: 7102f77

2 changed files

Status	File	+	-
M	`tarmac/src/main.rs`	1	0
M	`tarmac/src/platform/border.rs`	92	1

tarmac/src/main.rsmodified

      WM_STATE.with(|s| {
          if let Some(state) = s.borrow_mut().as_mut() {
              state.process_events();
 +            state.borders.health_check();
+         }
      });

tarmac/src/platform/border.rsmodified

  use std::path::PathBuf;
  use std::process::{Child, Command};
 +use std::time::{Duration, Instant};
  /// RGBA color for border configuration.
  #[derive(Debug, Clone, Copy)]
      pub unfocused_color: BorderColor,
      pub radius: f64,
      child: Option<Child>,
 +    /// Timestamps of recent automatic respawns (oldest first). Used to
 +    /// rate-limit the watchdog so a permanently-broken ers backs off
 +    /// instead of fork-bombing the system.
 +    recent_restarts: Vec<Instant>,
 +    /// When set, suppress further respawn attempts until this time.
 +    backoff_until: Option<Instant>,
 +    /// Last time `health_check` ran try_wait on the child. Throttles the
 +    /// poll to ~1Hz from the 50ms run-loop tick.
 +    last_health_check: Option<Instant>,
+ }
 +const RESTART_WINDOW: Duration = Duration::from_secs(60);
 +const RESTART_BUDGET: usize = 5;
 +const BACKOFF_DURATION: Duration = Duration::from_secs(300);
 +const HEALTH_CHECK_INTERVAL: Duration = Duration::from_secs(1);
++
  impl BorderManager {
      #[allow(clippy::new_without_default)]
      pub fn new() -> Self {
              unfocused_color: BorderColor::from_hex("#2d2d2d"),
              radius: 10.0,
              child: None,
 +            recent_restarts: Vec::new(),
 +            backoff_until: None,
 +            last_health_check: None,
+         }
+     }
          self.child = None;
+     }
 -    /// Restart ers with current settings (used on config reload).
 +    /// Restart ers with current settings (used on config reload). Resets
 +    /// the watchdog's backoff so a deliberate user reload always tries
 +    /// to spawn fresh.
      pub fn restart(&mut self) {
 +        self.recent_restarts.clear();
 +        self.backoff_until = None;
 +        self.spawn();
 +    }
++
 +    /// Periodically reap the ers child and respawn it if it died, so a
 +    /// crash inside the renderer doesn't strand tarmac with no borders.
 +    /// Throttled internally so callers can invoke from the run-loop tick.
 +    /// Restarts are budgeted: more than `RESTART_BUDGET` restarts inside
 +    /// `RESTART_WINDOW` triggers a `BACKOFF_DURATION` cooldown, after
 +    /// which the watchdog tries once more.
 +    pub fn health_check(&mut self) {
 +        if !self.is_enabled() {
 +            return;
 +        }
 +        let now = Instant::now();
 +        if let Some(prev) = self.last_health_check
 +            && now.duration_since(prev) < HEALTH_CHECK_INTERVAL
 +        {
 +            return;
 +        }
 +        self.last_health_check = Some(now);
++
 +        let died = match self.child.as_mut() {
 +            Some(child) => match child.try_wait() {
 +                Ok(Some(status)) => Some(status),
 +                Ok(None) => None,
 +                Err(e) => {
 +                    tracing::warn!(err = %e, "ers try_wait failed");
 +                    None
 +                }
 +            },
 +            None => None,
 +        };
++
 +        let Some(status) = died else {
 +            return;
 +        };
 +        // Drop the dead handle so kill()/spawn() don't try to wait on it
 +        // again.
 +        self.child = None;
++
 +        if let Some(until) = self.backoff_until
 +            && now < until
 +        {
 +            tracing::warn!(
 +                ?status,
 +                remaining_secs = (until - now).as_secs(),
 +                "ers exited; respawn suppressed by backoff"
 +            );
 +            return;
 +        }
 +        self.backoff_until = None;
++
 +        // Sliding-window rate limit.
 +        let cutoff = now - RESTART_WINDOW;
 +        self.recent_restarts.retain(|t| *t >= cutoff);
 +        if self.recent_restarts.len() >= RESTART_BUDGET {
 +            tracing::error!(
 +                ?status,
 +                budget = RESTART_BUDGET,
 +                window_secs = RESTART_WINDOW.as_secs(),
 +                "ers crashed too many times; backing off"
 +            );
 +            self.backoff_until = Some(now + BACKOFF_DURATION);
 +            self.recent_restarts.clear();
 +            return;
 +        }
++
 +        tracing::warn!(?status, "ers exited unexpectedly; respawning");
 +        self.recent_restarts.push(now);
          self.spawn();
+     }