Implement heartbeat-based host connectivity detection

- Add agent_heartbeat metric to agent transmission for reliable host detection
- Update dashboard to track heartbeat timestamps per host instead of general metrics
- Add configurable heartbeat_timeout_seconds to dashboard ZMQ config (default 10s)
- Remove unused timeout_ms from agent config and revert to non-blocking command reception
- Remove unused heartbeat_interval_ms from agent configuration
- Host disconnect detection now uses dedicated heartbeat metrics for improved reliability
- Bump version to 0.1.57

Cargo.lock

@@ -270,7 +270,7 @@ checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d"
 
 [[package]]
 name = "cm-dashboard"
-version = "0.1.51"
+version = "0.1.56"
 dependencies = [
  "anyhow",
  "chrono",
@@ -286,12 +286,13 @@ dependencies = [
  "toml",
  "tracing",
  "tracing-subscriber",
+ "wake-on-lan",
  "zmq",
 ]
 
 [[package]]
 name = "cm-dashboard-agent"
-version = "0.1.51"
+version = "0.1.56"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -314,7 +315,7 @@ dependencies = [
 
 [[package]]
 name = "cm-dashboard-shared"
-version = "0.1.51"
+version = "0.1.56"
 dependencies = [
  "chrono",
  "serde",
@@ -2064,6 +2065,12 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
+[[package]]
+name = "wake-on-lan"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ccf60b60ad7e5b1b37372c5134cbcab4db0706c231d212e0c643a077462bc8f"
+
 [[package]]
 name = "walkdir"
 version = "2.5.0"

agent/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cm-dashboard-agent"
-version = "0.1.52"
+version = "0.1.57"
 edition = "2021"
 
 [dependencies]

agent/src/agent.rs

@@ -180,6 +180,10 @@ impl Agent {
         let version_metric = self.get_agent_version_metric();
         metrics.push(version_metric);
 
+        // Add heartbeat metric for host connectivity detection
+        let heartbeat_metric = self.get_heartbeat_metric();
+        metrics.push(heartbeat_metric);
+
         // Check for user-stopped services that are now active and clear their flags
         self.clear_user_stopped_flags_for_active_services(&metrics);
 
@@ -201,6 +205,12 @@ impl Agent {
     async fn process_metrics(&mut self, metrics: &[Metric]) -> bool {
         let mut status_changed = false;
         for metric in metrics {
+            // Filter excluded metrics from email notification processing only
+            if self.config.exclude_email_metrics.contains(&metric.name) {
+                debug!("Excluding metric '{}' from email notification processing", metric.name);
+                continue;
+            }
+            
             if self.host_status_manager.process_metric(metric, &mut self.notification_manager).await {
                 status_changed = true;
             }
@@ -226,6 +236,22 @@ impl Agent {
         format!("v{}", env!("CARGO_PKG_VERSION"))
     }
 
+    /// Create heartbeat metric for host connectivity detection
+    fn get_heartbeat_metric(&self) -> Metric {
+        use std::time::{SystemTime, UNIX_EPOCH};
+        
+        let timestamp = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap()
+            .as_secs();
+        
+        Metric::new(
+            "agent_heartbeat".to_string(),
+            MetricValue::Integer(timestamp as i64),
+            Status::Ok,
+        )
+    }
+
     async fn handle_commands(&mut self) -> Result<()> {
         // Try to receive commands (non-blocking)
         match self.zmq_handler.try_receive_command() {

agent/src/collectors/backup.rs

@@ -140,6 +140,7 @@ impl Collector for BackupCollector {
                 Status::Warning => "warning".to_string(),
                 Status::Critical => "critical".to_string(),
                 Status::Unknown => "unknown".to_string(),
+                Status::Offline => "offline".to_string(),
             }),
             status: overall_status,
             timestamp,
@@ -202,6 +203,7 @@ impl Collector for BackupCollector {
                     Status::Warning => "warning".to_string(),
                     Status::Critical => "critical".to_string(),
                     Status::Unknown => "unknown".to_string(),
+                    Status::Offline => "offline".to_string(),
                 }),
                 status: service_status,
                 timestamp,

agent/src/communication/mod.rs

@@ -66,8 +66,6 @@ impl ZmqHandler {
     }
 
 
-    /// Send heartbeat (placeholder for future use)
-
     /// Try to receive a command (non-blocking)
     pub fn try_receive_command(&self) -> Result<Option<AgentCommand>> {
         match self.command_receiver.recv_bytes(zmq::DONTWAIT) {

agent/src/config/mod.rs

@@ -17,6 +17,9 @@ pub struct AgentConfig {
     pub notifications: NotificationConfig,
     pub status_aggregation: HostStatusConfig,
     pub collection_interval_seconds: u64,
+    /// List of metric names to exclude from email notifications
+    #[serde(default)]
+    pub exclude_email_metrics: Vec<String>,
 }
 
 /// ZMQ communication configuration
@@ -25,8 +28,6 @@ pub struct ZmqConfig {
     pub publisher_port: u16,
     pub command_port: u16,
     pub bind_address: String,
-    pub timeout_ms: u64,
-    pub heartbeat_interval_ms: u64,
     pub transmission_interval_seconds: u64,
 }
 

agent/src/config/validation.rs

@@ -19,10 +19,6 @@ pub fn validate_config(config: &AgentConfig) -> Result<()> {
         bail!("ZMQ bind address cannot be empty");
     }
 
-    if config.zmq.timeout_ms == 0 {
-        bail!("ZMQ timeout cannot be 0");
-    }
-
     // Validate collection interval
     if config.collection_interval_seconds == 0 {
         bail!("Collection interval cannot be 0");

dashboard/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cm-dashboard"
-version = "0.1.52"
+version = "0.1.57"
 edition = "2021"
 
 [dependencies]
@@ -18,4 +18,5 @@ tracing-subscriber = { workspace = true }
 ratatui = { workspace = true }
 crossterm = { workspace = true }
 toml = { workspace = true }
-gethostname = { workspace = true }
+gethostname = { workspace = true }
+wake-on-lan = "0.2"

dashboard/src/app.rs

@@ -22,7 +22,7 @@ pub struct Dashboard {
     terminal: Option<Terminal<CrosstermBackend<io::Stdout>>>,
     headless: bool,
     initial_commands_sent: std::collections::HashSet<String>,
-    _config: DashboardConfig,
+    config: DashboardConfig,
 }
 
 impl Dashboard {
@@ -67,8 +67,8 @@ impl Dashboard {
             }
         };
 
-        // Connect to predefined hosts from configuration
-        let hosts = config.hosts.predefined_hosts.clone();
+        // Connect to configured hosts from configuration
+        let hosts: Vec<String> = config.hosts.keys().cloned().collect();
 
         // Try to connect to hosts but don't fail if none are available
         match zmq_consumer.connect_to_predefined_hosts(&hosts).await {
@@ -133,7 +133,7 @@ impl Dashboard {
             terminal,
             headless,
             initial_commands_sent: std::collections::HashSet::new(),
-            _config: config,
+            config,
         })
     }
 
@@ -247,7 +247,7 @@ impl Dashboard {
                     if let Some(ref mut tui_app) = self.tui_app {
                         let connected_hosts = self
                             .metric_store
-                            .get_connected_hosts(Duration::from_secs(30));
+                            .get_connected_hosts(Duration::from_secs(self.config.zmq.heartbeat_timeout_seconds));
                         
                         
                         tui_app.update_hosts(connected_hosts);

dashboard/src/communication/mod.rs

@@ -141,9 +141,9 @@ impl ZmqConsumer {
         }
     }
 
-    /// Receive metrics from any connected agent (non-blocking)
+    /// Receive metrics from any connected agent (with timeout)
     pub async fn receive_metrics(&mut self) -> Result<Option<MetricMessage>> {
-        match self.subscriber.recv_bytes(zmq::DONTWAIT) {
+        match self.subscriber.recv_bytes(0) {
             Ok(data) => {
                 debug!("Received {} bytes from ZMQ", data.len());
 

dashboard/src/config/mod.rs

@@ -6,7 +6,7 @@ use std::path::Path;
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct DashboardConfig {
     pub zmq: ZmqConfig,
-    pub hosts: HostsConfig,
+    pub hosts: std::collections::HashMap<String, HostDetails>,
     pub system: SystemConfig,
     pub ssh: SshConfig,
     pub service_logs: std::collections::HashMap<String, Vec<ServiceLogConfig>>,
@@ -16,12 +16,19 @@ pub struct DashboardConfig {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ZmqConfig {
     pub subscriber_ports: Vec<u16>,
+    /// Heartbeat timeout in seconds - hosts considered offline if no heartbeat received within this time
+    #[serde(default = "default_heartbeat_timeout_seconds")]
+    pub heartbeat_timeout_seconds: u64,
 }
 
-/// Hosts configuration
+fn default_heartbeat_timeout_seconds() -> u64 {
+    10 // Default to 10 seconds - allows for multiple missed heartbeats
+}
+
+/// Individual host configuration details
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct HostsConfig {
-    pub predefined_hosts: Vec<String>,
+pub struct HostDetails {
+    pub mac_address: Option<String>,
 }
 
 /// System configuration
@@ -68,8 +75,3 @@ impl Default for ZmqConfig {
     }
 }
 
-impl Default for HostsConfig {
-    fn default() -> Self {
-        panic!("Dashboard configuration must be loaded from file - no hardcoded defaults allowed")
-    }
-}

dashboard/src/metrics/store.rs

@@ -11,8 +11,8 @@ pub struct MetricStore {
     current_metrics: HashMap<String, HashMap<String, Metric>>,
     /// Historical metrics for trending
     historical_metrics: HashMap<String, Vec<MetricDataPoint>>,
-    /// Last update timestamp per host
-    last_update: HashMap<String, Instant>,
+    /// Last heartbeat timestamp per host
+    last_heartbeat: HashMap<String, Instant>,
     /// Configuration
     max_metrics_per_host: usize,
     history_retention: Duration,
@@ -23,7 +23,7 @@ impl MetricStore {
         Self {
             current_metrics: HashMap::new(),
             historical_metrics: HashMap::new(),
-            last_update: HashMap::new(),
+            last_heartbeat: HashMap::new(),
             max_metrics_per_host,
             history_retention: Duration::from_secs(history_retention_hours * 3600),
         }
@@ -56,10 +56,13 @@ impl MetricStore {
 
             // Add to history
             host_history.push(MetricDataPoint { received_at: now });
-        }
 
-        // Update last update timestamp
-        self.last_update.insert(hostname.to_string(), now);
+            // Track heartbeat metrics for connectivity detection
+            if metric_name == "agent_heartbeat" {
+                self.last_heartbeat.insert(hostname.to_string(), now);
+                debug!("Updated heartbeat for host {}", hostname);
+            }
+        }
 
         // Get metrics count before cleanup
         let metrics_count = host_metrics.len();
@@ -88,16 +91,18 @@ impl MetricStore {
         }
     }
 
-    /// Get connected hosts (hosts with recent updates)
+    /// Get connected hosts (hosts with recent heartbeats)
     pub fn get_connected_hosts(&self, timeout: Duration) -> Vec<String> {
         let now = Instant::now();
 
-        self.last_update
+        self.last_heartbeat
             .iter()
-            .filter_map(|(hostname, &last_update)| {
-                if now.duration_since(last_update) <= timeout {
+            .filter_map(|(hostname, &last_heartbeat)| {
+                if now.duration_since(last_heartbeat) <= timeout {
                     Some(hostname.clone())
                 } else {
+                    debug!("Host {} considered offline - last heartbeat was {:?} ago", 
+                           hostname, now.duration_since(last_heartbeat));
                     None
                 }
             })

dashboard/src/ui/mod.rs

@@ -9,6 +9,7 @@ use ratatui::{
 use std::collections::HashMap;
 use std::time::Instant;
 use tracing::info;
+use wake_on_lan::MagicPacket;
 
 pub mod theme;
 pub mod widgets;
@@ -93,15 +94,25 @@ pub struct TuiApp {
 
 impl TuiApp {
     pub fn new(config: DashboardConfig) -> Self {
-        Self {
+        let mut app = Self {
             host_widgets: HashMap::new(),
             current_host: None,
-            available_hosts: Vec::new(),
+            available_hosts: config.hosts.keys().cloned().collect(),
             host_index: 0,
             should_quit: false,
             user_navigated_away: false,
             config,
+        };
+        
+        // Sort predefined hosts
+        app.available_hosts.sort();
+        
+        // Initialize with first host if available
+        if !app.available_hosts.is_empty() {
+            app.current_host = Some(app.available_hosts[0].clone());
         }
+        
+        app
     }
 
     /// Get or create host widgets for the given hostname
@@ -186,21 +197,28 @@ impl TuiApp {
     }
 
     /// Update available hosts with localhost prioritization
-    pub fn update_hosts(&mut self, hosts: Vec<String>) {
-        // Sort hosts alphabetically
-        let mut sorted_hosts = hosts.clone();
+    pub fn update_hosts(&mut self, discovered_hosts: Vec<String>) {
+        // Start with configured hosts (always visible)
+        let mut all_hosts: Vec<String> = self.config.hosts.keys().cloned().collect();
+        
+        // Add any discovered hosts that aren't already configured
+        for host in discovered_hosts {
+            if !all_hosts.contains(&host) {
+                all_hosts.push(host);
+            }
+        }
         
         // Keep hosts that have pending transitions even if they're offline
         for (hostname, host_widgets) in &self.host_widgets {
             if !host_widgets.pending_service_transitions.is_empty() {
-                if !sorted_hosts.contains(hostname) {
-                    sorted_hosts.push(hostname.clone());
+                if !all_hosts.contains(hostname) {
+                    all_hosts.push(hostname.clone());
                 }
             }
         }
         
-        sorted_hosts.sort();
-        self.available_hosts = sorted_hosts;
+        all_hosts.sort();
+        self.available_hosts = all_hosts;
         
         // Get the current hostname (localhost) for auto-selection
         let localhost = gethostname::gethostname().to_string_lossy().to_string();
@@ -246,18 +264,7 @@ impl TuiApp {
                     if let Some(hostname) = self.current_host.clone() {
                         // Create command that shows logo, rebuilds, and waits for user input
                         let logo_and_rebuild = format!(
-                            r"bash -c 'cat << \"EOF\"
-NixOS System Rebuild
-Target: {}
-
-EOF
-ssh -tt {}@{} \"bash -ic {}\"
-echo
-echo \"========================================\"
-echo \"Rebuild completed. Press any key to close...\"
-echo \"========================================\"
-read -n 1 -s
-exit'",
+                            "bash -c 'cat << \"EOF\"\nNixOS System Rebuild\nTarget: {}\n\nEOF\nssh -tt {}@{} \"bash -ic {}\"\necho\necho \"========================================\"\necho \"Rebuild completed. Press any key to close...\"\necho \"========================================\"\nread -n 1 -s\nexit'",
                             hostname,
                             self.config.ssh.rebuild_user,
                             hostname,
@@ -342,6 +349,33 @@ exit'",
                         return Ok(Some(UiCommand::TriggerBackup { hostname }));
                     }
                 }
+                KeyCode::Char('w') => {
+                    // Wake on LAN for offline hosts
+                    if let Some(hostname) = self.current_host.clone() {
+                        // Check if host has MAC address configured
+                        if let Some(host_details) = self.config.hosts.get(&hostname) {
+                            if let Some(mac_address) = &host_details.mac_address {
+                                // Parse MAC address and send WoL packet
+                                let mac_bytes = Self::parse_mac_address(mac_address);
+                                match mac_bytes {
+                                    Ok(mac) => {
+                                        match MagicPacket::new(&mac).send() {
+                                            Ok(_) => {
+                                                info!("WakeOnLAN packet sent successfully to {} ({})", hostname, mac_address);
+                                            }
+                                            Err(e) => {
+                                                tracing::error!("Failed to send WakeOnLAN packet to {}: {}", hostname, e);
+                                            }
+                                        }
+                                    }
+                                    Err(_) => {
+                                        tracing::error!("Invalid MAC address format for {}: {}", hostname, mac_address);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
                 KeyCode::Tab => {
                     // Tab cycles to next host
                     self.navigate_host(1);
@@ -603,11 +637,14 @@ exit'",
             return;
         }
 
-        // Calculate worst-case status across all hosts
+        // Calculate worst-case status across all hosts (excluding offline)
         let mut worst_status = Status::Ok;
         for host in &self.available_hosts {
             let host_status = self.calculate_host_status(host, metric_store);
-            worst_status = Status::aggregate(&[worst_status, host_status]);
+            // Don't include offline hosts in status aggregation
+            if host_status != Status::Offline {
+                worst_status = Status::aggregate(&[worst_status, host_status]);
+            }
         }
 
         // Use the worst status color as background
@@ -685,7 +722,7 @@ exit'",
         let metrics = metric_store.get_metrics_for_host(hostname);
 
         if metrics.is_empty() {
-            return Status::Unknown;
+            return Status::Offline;
         }
 
         // First check if we have the aggregated host status summary from the agent
@@ -705,7 +742,8 @@ exit'",
                 Status::Warning => has_warning = true,
                 Status::Pending => has_pending = true,
                 Status::Ok => ok_count += 1,
-                Status::Unknown => {} // Ignore unknown for aggregation
+                Status::Unknown => {}, // Ignore unknown for aggregation
+                Status::Offline => {}, // Ignore offline for aggregation
             }
         }
 
@@ -746,6 +784,7 @@ exit'",
         shortcuts.push("s/S: Start/Stop".to_string());
         shortcuts.push("J: Logs".to_string());
         shortcuts.push("L: Custom".to_string());
+        shortcuts.push("w: Wake".to_string());
         
         // Always show quit
         shortcuts.push("q: Quit".to_string());
@@ -784,5 +823,20 @@ exit'",
         }
     }
 
+    /// Parse MAC address string (e.g., "AA:BB:CC:DD:EE:FF") to [u8; 6]
+    fn parse_mac_address(mac_str: &str) -> Result<[u8; 6], &'static str> {
+        let parts: Vec<&str> = mac_str.split(':').collect();
+        if parts.len() != 6 {
+            return Err("MAC address must have 6 parts separated by colons");
+        }
 
+        let mut mac = [0u8; 6];
+        for (i, part) in parts.iter().enumerate() {
+            match u8::from_str_radix(part, 16) {
+                Ok(byte) => mac[i] = byte,
+                Err(_) => return Err("Invalid hexadecimal byte in MAC address"),
+            }
+        }
+        Ok(mac)
+    }
 }

dashboard/src/ui/theme.rs

@@ -147,6 +147,7 @@ impl Theme {
             Status::Warning => Self::warning(),
             Status::Critical => Self::error(),
             Status::Unknown => Self::muted_text(),
+            Status::Offline => Self::muted_text(), // Dark gray for offline
         }
     }
 
@@ -244,8 +245,9 @@ impl StatusIcons {
             Status::Ok => "●",
             Status::Pending => "◉", // Hollow circle for pending
             Status::Warning => "◐",
-            Status::Critical => "◯",
+            Status::Critical => "!",
             Status::Unknown => "?",
+            Status::Offline => "○", // Empty circle for offline
         }
     }
 
@@ -258,6 +260,7 @@ impl StatusIcons {
             Status::Warning => Theme::warning(),    // Yellow
             Status::Critical => Theme::error(),     // Red
             Status::Unknown => Theme::muted_text(), // Gray
+            Status::Offline => Theme::muted_text(), // Dark gray for offline
         };
 
         vec![

dashboard/src/ui/widgets/services.rs

@@ -146,6 +146,7 @@ impl ServicesWidget {
             Status::Warning => Theme::warning(),
             Status::Critical => Theme::error(),
             Status::Unknown => Theme::muted_text(),
+            Status::Offline => Theme::muted_text(),
         };
         
         (icon.to_string(), info.status.clone(), status_color)

shared/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cm-dashboard-shared"
-version = "0.1.52"
+version = "0.1.57"
 edition = "2021"
 
 [dependencies]

shared/src/metrics.rs

@@ -87,6 +87,7 @@ pub enum Status {
     Warning,
     Critical,
     Unknown,
+    Offline,
 }
 
 impl Status {
@@ -190,6 +191,16 @@ impl HysteresisThresholds {
                     Status::Ok
                 }
             }
+            Status::Offline => {
+                // Host coming back online, use normal thresholds like first measurement
+                if value >= self.critical_high {
+                    Status::Critical
+                } else if value >= self.warning_high {
+                    Status::Warning
+                } else {
+                    Status::Ok
+                }
+            }
         }
     }
 }