Improve host disconnection detection and fix notification exclusions

- Add dedicated heartbeat transmission every 5 seconds independent of metric collection
- Fix host offline detection by clearing metrics for disconnected hosts
- Move exclude_email_metrics to NotificationConfig for better organization
- Add cleanup_offline_hosts method to remove stale metrics after heartbeat timeout
- Ensure offline hosts show proper status icons and visual indicators

Version 0.1.63

Cargo.lock

@@ -270,7 +270,7 @@ checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d"
 
 [[package]]
 name = "cm-dashboard"
-version = "0.1.57"
+version = "0.1.62"
 dependencies = [
  "anyhow",
  "chrono",
@@ -292,7 +292,7 @@ dependencies = [
 
 [[package]]
 name = "cm-dashboard-agent"
-version = "0.1.57"
+version = "0.1.62"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -315,7 +315,7 @@ dependencies = [
 
 [[package]]
 name = "cm-dashboard-shared"
-version = "0.1.57"
+version = "0.1.62"
 dependencies = [
  "chrono",
  "serde",

agent/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cm-dashboard-agent"
-version = "0.1.58"
+version = "0.1.63"
 edition = "2021"
 
 [dependencies]

agent/src/agent.rs

@@ -78,10 +78,11 @@ impl Agent {
             info!("Initial metric collection completed - all data cached and ready");
         }
 
-        // Separate intervals for collection, transmission, and email notifications
+        // Separate intervals for collection, transmission, heartbeat, and email notifications
         let mut collection_interval =
             interval(Duration::from_secs(self.config.collection_interval_seconds));
         let mut transmission_interval = interval(Duration::from_secs(self.config.zmq.transmission_interval_seconds));
+        let mut heartbeat_interval = interval(Duration::from_secs(self.config.zmq.heartbeat_interval_seconds));
         let mut notification_interval = interval(Duration::from_secs(self.config.notifications.aggregation_interval_seconds));
 
         loop {
@@ -98,6 +99,12 @@ impl Agent {
                         error!("Failed to broadcast metrics: {}", e);
                     }
                 }
+                _ = heartbeat_interval.tick() => {
+                    // Send standalone heartbeat for host connectivity detection
+                    if let Err(e) = self.send_heartbeat().await {
+                        error!("Failed to send heartbeat: {}", e);
+                    }
+                }
                 _ = notification_interval.tick() => {
                     // Process batched email notifications (separate from dashboard updates)
                     if let Err(e) = self.host_status_manager.process_pending_notifications(&mut self.notification_manager).await {
@@ -206,7 +213,7 @@ impl Agent {
         let mut status_changed = false;
         for metric in metrics {
             // Filter excluded metrics from email notification processing only
-            if self.config.exclude_email_metrics.contains(&metric.name) {
+            if self.config.notifications.exclude_email_metrics.contains(&metric.name) {
                 debug!("Excluding metric '{}' from email notification processing", metric.name);
                 continue;
             }
@@ -252,6 +259,19 @@ impl Agent {
         )
     }
 
+    /// Send standalone heartbeat for connectivity detection
+    async fn send_heartbeat(&mut self) -> Result<()> {
+        let heartbeat_metric = self.get_heartbeat_metric();
+        let message = MetricMessage::new(
+            self.hostname.clone(),
+            vec![heartbeat_metric],
+        );
+
+        self.zmq_handler.publish_metrics(&message).await?;
+        debug!("Sent standalone heartbeat for connectivity detection");
+        Ok(())
+    }
+
     async fn handle_commands(&mut self) -> Result<()> {
         // Try to receive commands (non-blocking)
         match self.zmq_handler.try_receive_command() {

agent/src/config/mod.rs

@@ -17,9 +17,6 @@ pub struct AgentConfig {
     pub notifications: NotificationConfig,
     pub status_aggregation: HostStatusConfig,
     pub collection_interval_seconds: u64,
-    /// List of metric names to exclude from email notifications
-    #[serde(default)]
-    pub exclude_email_metrics: Vec<String>,
 }
 
 /// ZMQ communication configuration
@@ -29,6 +26,9 @@ pub struct ZmqConfig {
     pub command_port: u16,
     pub bind_address: String,
     pub transmission_interval_seconds: u64,
+    /// Heartbeat transmission interval in seconds for host connectivity detection
+    #[serde(default = "default_heartbeat_interval_seconds")]
+    pub heartbeat_interval_seconds: u64,
 }
 
 /// Collector configuration
@@ -147,9 +147,16 @@ pub struct NotificationConfig {
     pub rate_limit_minutes: u64,
     /// Email notification batching interval in seconds (default: 60)
     pub aggregation_interval_seconds: u64,
+    /// List of metric names to exclude from email notifications
+    #[serde(default)]
+    pub exclude_email_metrics: Vec<String>,
 }
 
 
+fn default_heartbeat_interval_seconds() -> u64 {
+    5
+}
+
 impl AgentConfig {
     pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
         loader::load_config(path)

dashboard/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cm-dashboard"
-version = "0.1.58"
+version = "0.1.63"
 edition = "2021"
 
 [dependencies]

dashboard/src/app.rs

@@ -149,6 +149,8 @@ impl Dashboard {
 
         let mut last_metrics_check = Instant::now();
         let metrics_check_interval = Duration::from_millis(100); // Check for metrics every 100ms
+        let mut last_heartbeat_check = Instant::now();
+        let heartbeat_check_interval = Duration::from_secs(1); // Check for host connectivity every 1 second
 
         loop {
             // Handle terminal events (keyboard input) only if not headless
@@ -191,6 +193,17 @@ impl Dashboard {
                         break;
                     }
                 }
+
+                // Render UI immediately after handling input for responsive feedback
+                if let Some(ref mut terminal) = self.terminal {
+                    if let Some(ref mut tui_app) = self.tui_app {
+                        if let Err(e) = terminal.draw(|frame| {
+                            tui_app.render(frame, &self.metric_store);
+                        }) {
+                            error!("Error rendering TUI after input: {}", e);
+                        }
+                    }
+                }
             }
 
             // Check for new metrics
@@ -243,14 +256,8 @@ impl Dashboard {
                         }
                     }
 
-                    // Update TUI with new hosts and metrics (only if not headless)
+                    // Update TUI with new metrics (only if not headless)
                     if let Some(ref mut tui_app) = self.tui_app {
-                        let connected_hosts = self
-                            .metric_store
-                            .get_connected_hosts(Duration::from_secs(self.config.zmq.heartbeat_timeout_seconds));
-                        
-                        
-                        tui_app.update_hosts(connected_hosts);
                         tui_app.update_metrics(&self.metric_store);
                     }
                 }
@@ -269,6 +276,20 @@ impl Dashboard {
                 last_metrics_check = Instant::now();
             }
 
+            // Check for host connectivity changes (heartbeat timeouts) periodically
+            if last_heartbeat_check.elapsed() >= heartbeat_check_interval {
+                let timeout = Duration::from_secs(self.config.zmq.heartbeat_timeout_seconds);
+                
+                // Clean up metrics for offline hosts
+                self.metric_store.cleanup_offline_hosts(timeout);
+                
+                if let Some(ref mut tui_app) = self.tui_app {
+                    let connected_hosts = self.metric_store.get_connected_hosts(timeout);
+                    tui_app.update_hosts(connected_hosts);
+                }
+                last_heartbeat_check = Instant::now();
+            }
+
             // Render TUI (only if not headless)
             if !self.headless {
                 if let Some(ref mut terminal) = self.terminal {

dashboard/src/communication/mod.rs

@@ -141,9 +141,9 @@ impl ZmqConsumer {
         }
     }
 
-    /// Receive metrics from any connected agent (with timeout)
+    /// Receive metrics from any connected agent (non-blocking)
     pub async fn receive_metrics(&mut self) -> Result<Option<MetricMessage>> {
-        match self.subscriber.recv_bytes(0) {
+        match self.subscriber.recv_bytes(zmq::DONTWAIT) {
             Ok(data) => {
                 debug!("Received {} bytes from ZMQ", data.len());
 

dashboard/src/metrics/store.rs

@@ -109,6 +109,28 @@ impl MetricStore {
             .collect()
     }
 
+    /// Clean up data for offline hosts
+    pub fn cleanup_offline_hosts(&mut self, timeout: Duration) {
+        let now = Instant::now();
+        let mut hosts_to_cleanup = Vec::new();
+
+        // Find hosts that are offline (no recent heartbeat)
+        for (hostname, &last_heartbeat) in &self.last_heartbeat {
+            if now.duration_since(last_heartbeat) > timeout {
+                hosts_to_cleanup.push(hostname.clone());
+            }
+        }
+
+        // Clear metrics for offline hosts
+        for hostname in hosts_to_cleanup {
+            if let Some(metrics) = self.current_metrics.remove(&hostname) {
+                info!("Cleared {} metrics for offline host: {}", metrics.len(), hostname);
+            }
+            // Keep heartbeat timestamp for reconnection detection
+            // Don't remove from last_heartbeat to track when host was last seen
+        }
+    }
+
     /// Cleanup old data and enforce limits
     fn cleanup_host_data(&mut self, hostname: &str) {
         let now = Instant::now();

dashboard/src/ui/mod.rs

@@ -90,10 +90,13 @@ pub struct TuiApp {
     user_navigated_away: bool,
     /// Dashboard configuration
     config: DashboardConfig,
+    /// Cached localhost hostname to avoid repeated system calls
+    localhost: String,
 }
 
 impl TuiApp {
     pub fn new(config: DashboardConfig) -> Self {
+        let localhost = gethostname::gethostname().to_string_lossy().to_string();
         let mut app = Self {
             host_widgets: HashMap::new(),
             current_host: None,
@@ -102,6 +105,7 @@ impl TuiApp {
             should_quit: false,
             user_navigated_away: false,
             config,
+            localhost,
         };
         
         // Sort predefined hosts
@@ -207,13 +211,12 @@ impl TuiApp {
         self.available_hosts = all_hosts;
         
         // Get the current hostname (localhost) for auto-selection
-        let localhost = gethostname::gethostname().to_string_lossy().to_string();
         if !self.available_hosts.is_empty() {
-            if self.available_hosts.contains(&localhost) && !self.user_navigated_away {
+            if self.available_hosts.contains(&self.localhost) && !self.user_navigated_away {
                 // Localhost is available and user hasn't navigated away - switch to it
-                self.current_host = Some(localhost.clone());
+                self.current_host = Some(self.localhost.clone());
                 // Find the actual index of localhost in the sorted list
-                self.host_index = self.available_hosts.iter().position(|h| h == &localhost).unwrap_or(0);
+                self.host_index = self.available_hosts.iter().position(|h| h == &self.localhost).unwrap_or(0);
             } else if self.current_host.is_none() {
                 // No current host - select first available (which is localhost if available)
                 self.current_host = Some(self.available_hosts[0].clone());
@@ -410,9 +413,8 @@ impl TuiApp {
         self.current_host = Some(self.available_hosts[self.host_index].clone());
         
         // Check if user navigated away from localhost
-        let localhost = gethostname::gethostname().to_string_lossy().to_string();
         if let Some(ref current) = self.current_host {
-            if current != &localhost {
+            if current != &self.localhost {
                 self.user_navigated_away = true;
             } else {
                 self.user_navigated_away = false; // User navigated back to localhost

shared/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cm-dashboard-shared"
-version = "0.1.58"
+version = "0.1.63"
 edition = "2021"
 
 [dependencies]