Replace complex SystemRebuild with simple SSH + tmux popup approach

- Remove all SystemRebuild command infrastructure from agent and dashboard
- Replace with direct tmux popup execution: ssh {user}@{host} {alias}
- Add configurable SSH user and rebuild alias in dashboard config
- Eliminate agent process crashes during rebuilds
- Simplify architecture by removing ZMQ command streaming complexity
- Clean up all related dead code and fix compilation warnings

Benefits:
- Process isolation: rebuild runs independently via SSH
- Crash resilience: agent/dashboard can restart without affecting rebuilds
- Configuration flexibility: SSH user and alias configurable per deployment
- Operational simplicity: standard tmux popup interface

dashboard/src/app.rs

@@ -22,7 +22,7 @@ pub struct Dashboard {
     terminal: Option<Terminal<CrosstermBackend<io::Stdout>>>,
     headless: bool,
     initial_commands_sent: std::collections::HashSet<String>,
-    config: DashboardConfig,
+    _config: DashboardConfig,
 }
 
 impl Dashboard {
@@ -91,7 +91,7 @@ impl Dashboard {
             (None, None)
         } else {
             // Initialize TUI app
-            let tui_app = TuiApp::new();
+            let tui_app = TuiApp::new(config.clone());
 
             // Setup terminal
             if let Err(e) = enable_raw_mode() {
@@ -133,7 +133,7 @@ impl Dashboard {
             terminal,
             headless,
             initial_commands_sent: std::collections::HashSet::new(),
-            config,
+            _config: config,
         })
     }
 
@@ -245,24 +245,10 @@ impl Dashboard {
 
                     // Update TUI with new hosts and metrics (only if not headless)
                     if let Some(ref mut tui_app) = self.tui_app {
-                        let mut connected_hosts = self
+                        let connected_hosts = self
                             .metric_store
                             .get_connected_hosts(Duration::from_secs(30));
                         
-                        // Add hosts that are rebuilding but may be temporarily disconnected
-                        // Use extended timeout (5 minutes) for rebuilding hosts
-                        let rebuilding_hosts = self
-                            .metric_store
-                            .get_connected_hosts(Duration::from_secs(300));
-                        
-                        for host in rebuilding_hosts {
-                            if !connected_hosts.contains(&host) {
-                                // Check if this host is rebuilding in the UI
-                                if tui_app.is_host_rebuilding(&host) {
-                                    connected_hosts.push(host);
-                                }
-                            }
-                        }
                         
                         tui_app.update_hosts(connected_hosts);
                         tui_app.update_metrics(&self.metric_store);
@@ -290,14 +276,14 @@ impl Dashboard {
 
             // Render TUI (only if not headless)
             if !self.headless {
-                if let (Some(ref mut terminal), Some(ref mut tui_app)) =
-                    (&mut self.terminal, &mut self.tui_app)
-                {
-                    if let Err(e) = terminal.draw(|frame| {
-                        tui_app.render(frame, &self.metric_store);
-                    }) {
-                        error!("Error rendering TUI: {}", e);
-                        break;
+                if let Some(ref mut terminal) = self.terminal {
+                    if let Some(ref mut tui_app) = self.tui_app {
+                        if let Err(e) = terminal.draw(|frame| {
+                            tui_app.render(frame, &self.metric_store);
+                        }) {
+                            error!("Error rendering TUI: {}", e);
+                            break;
+                        }
                     }
                 }
             }
@@ -337,16 +323,6 @@ impl Dashboard {
                 };
                 self.zmq_command_sender.send_command(&hostname, agent_command).await?;
             }
-            UiCommand::SystemRebuild { hostname } => {
-                info!("Sending system rebuild command to {}", hostname);
-                let agent_command = AgentCommand::SystemRebuild { 
-                    git_url: self.config.system.nixos_config_git_url.clone(),
-                    git_branch: self.config.system.nixos_config_branch.clone(),
-                    working_dir: self.config.system.nixos_config_working_dir.clone(),
-                    api_key_file: self.config.system.nixos_config_api_key_file.clone(),
-                };
-                self.zmq_command_sender.send_command(&hostname, agent_command).await?;
-            }
             UiCommand::TriggerBackup { hostname } => {
                 info!("Trigger backup requested for {}", hostname);
                 // TODO: Implement backup trigger command

dashboard/src/config/mod.rs

@@ -8,6 +8,7 @@ pub struct DashboardConfig {
     pub zmq: ZmqConfig,
     pub hosts: HostsConfig,
     pub system: SystemConfig,
+    pub ssh: SshConfig,
 }
 
 /// ZMQ consumer configuration
@@ -31,6 +32,13 @@ pub struct SystemConfig {
     pub nixos_config_api_key_file: Option<String>,
 }
 
+/// SSH configuration for rebuild operations
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SshConfig {
+    pub rebuild_user: String,
+    pub rebuild_alias: String,
+}
+
 impl DashboardConfig {
     pub fn load_from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
         let path = path.as_ref();

dashboard/src/ui/mod.rs

@@ -13,6 +13,7 @@ use tracing::info;
 pub mod theme;
 pub mod widgets;
 
+use crate::config::DashboardConfig;
 use crate::metrics::MetricStore;
 use cm_dashboard_shared::{Metric, Status};
 use theme::{Components, Layout as ThemeLayout, Theme, Typography};
@@ -24,7 +25,6 @@ pub enum UiCommand {
     ServiceRestart { hostname: String, service_name: String },
     ServiceStart { hostname: String, service_name: String },
     ServiceStop { hostname: String, service_name: String },
-    SystemRebuild { hostname: String },
     TriggerBackup { hostname: String },
 }
 
@@ -33,8 +33,6 @@ pub enum UiCommand {
 pub enum CommandStatus {
     /// Command is executing
     InProgress { command_type: CommandType, target: String, start_time: std::time::Instant },
-    /// Command completed successfully
-    Success { command_type: CommandType, completed_at: std::time::Instant },
 }
 
 /// Types of commands for status tracking
@@ -43,7 +41,6 @@ pub enum CommandType {
     ServiceRestart,
     ServiceStart,
     ServiceStop,
-    SystemRebuild,
     BackupTrigger,
 }
 
@@ -98,7 +95,7 @@ pub struct TerminalPopup {
     /// Is the popup currently visible
     pub visible: bool,
     /// Command being executed
-    pub command_type: CommandType,
+    pub _command_type: CommandType,
     /// Target hostname
     pub hostname: String,
     /// Target service/operation name
@@ -112,10 +109,10 @@ pub struct TerminalPopup {
 }
 
 impl TerminalPopup {
-    pub fn new(command_type: CommandType, hostname: String, target: String) -> Self {
+    pub fn _new(command_type: CommandType, hostname: String, target: String) -> Self {
         Self {
             visible: true,
-            command_type,
+            _command_type: command_type,
             hostname,
             target,
             output_lines: Vec::new(),
@@ -155,10 +152,12 @@ pub struct TuiApp {
     user_navigated_away: bool,
     /// Terminal popup for streaming command output
     terminal_popup: Option<TerminalPopup>,
+    /// Dashboard configuration
+    config: DashboardConfig,
 }
 
 impl TuiApp {
-    pub fn new() -> Self {
+    pub fn new(config: DashboardConfig) -> Self {
         Self {
             host_widgets: HashMap::new(),
             current_host: None,
@@ -168,6 +167,7 @@ impl TuiApp {
             should_quit: false,
             user_navigated_away: false,
             terminal_popup: None,
+            config,
         }
     }
 
@@ -184,7 +184,6 @@ impl TuiApp {
         self.check_command_timeouts();
         
         // Check for rebuild completion by agent hash change
-        self.check_rebuild_completion(metric_store);
         
         if let Some(hostname) = self.current_host.clone() {
             // Only update widgets if we have metrics for this host
@@ -257,9 +256,9 @@ impl TuiApp {
         // Sort hosts alphabetically
         let mut sorted_hosts = hosts.clone();
         
-        // Keep hosts that are undergoing SystemRebuild even if they're offline
+        // Keep hosts that have ongoing commands even if they're offline
         for (hostname, host_widgets) in &self.host_widgets {
-            if let Some(CommandStatus::InProgress { command_type: CommandType::SystemRebuild, .. }) = &host_widgets.command_status {
+            if let Some(CommandStatus::InProgress { .. }) = &host_widgets.command_status {
                 if !sorted_hosts.contains(hostname) {
                     sorted_hosts.push(hostname.clone());
                 }
@@ -343,16 +342,20 @@ impl TuiApp {
                 KeyCode::Char('r') => {
                     match self.focused_panel {
                         PanelType::System => {
-                            // System rebuild command
+                            // Simple tmux popup with SSH rebuild using configured user and alias
                             if let Some(hostname) = self.current_host.clone() {
-                                self.start_command(&hostname, CommandType::SystemRebuild, hostname.clone());
-                                // Open terminal popup for real-time output
-                                self.terminal_popup = Some(TerminalPopup::new(
-                                    CommandType::SystemRebuild,
-                                    hostname.clone(),
-                                    "NixOS Rebuild".to_string()
-                                ));
-                                return Ok(Some(UiCommand::SystemRebuild { hostname }));
+                                // Launch tmux popup with SSH using config values
+                                std::process::Command::new("tmux")
+                                    .arg("popup")
+                                    .arg("-d")
+                                    .arg("#{pane_current_path}")
+                                    .arg("-xC")
+                                    .arg("-yC")
+                                    .arg("ssh")
+                                    .arg(&format!("{}@{}", self.config.ssh.rebuild_user, hostname))
+                                    .arg(&self.config.ssh.rebuild_alias)
+                                    .spawn()
+                                    .ok(); // Ignore errors, tmux will handle them
                             }
                         }
                         PanelType::Services => {
@@ -453,17 +456,6 @@ impl TuiApp {
         info!("Switched to host: {}", self.current_host.as_ref().unwrap());
     }
 
-    /// Check if a host is currently rebuilding
-    pub fn is_host_rebuilding(&self, hostname: &str) -> bool {
-        if let Some(host_widgets) = self.host_widgets.get(hostname) {
-            matches!(
-                &host_widgets.command_status,
-                Some(CommandStatus::InProgress { command_type: CommandType::SystemRebuild, .. })
-            )
-        } else {
-            false
-        }
-    }
 
     /// Switch to next panel (Shift+Tab) - only cycles through visible panels
     pub fn next_panel(&mut self) {
@@ -515,14 +507,10 @@ impl TuiApp {
     }
 
     /// Mark command as completed successfully
-    pub fn complete_command(&mut self, hostname: &str) {
+    pub fn _complete_command(&mut self, hostname: &str) {
         if let Some(host_widgets) = self.host_widgets.get_mut(hostname) {
-            if let Some(CommandStatus::InProgress { command_type, .. }) = &host_widgets.command_status {
-                host_widgets.command_status = Some(CommandStatus::Success {
-                    command_type: command_type.clone(),
-                    completed_at: Instant::now(),
-                });
-            }
+            // Simply clear the command status when completed
+            host_widgets.command_status = None;
         }
     }
 
@@ -533,22 +521,13 @@ impl TuiApp {
         let mut hosts_to_clear = Vec::new();
         
         for (hostname, host_widgets) in &self.host_widgets {
-            if let Some(CommandStatus::InProgress { command_type, start_time, .. }) = &host_widgets.command_status {
-                let timeout_duration = match command_type {
-                    CommandType::SystemRebuild => Duration::from_secs(300), // 5 minutes for rebuilds
-                    _ => Duration::from_secs(30), // 30 seconds for service commands
-                };
+            if let Some(CommandStatus::InProgress { command_type: _, start_time, .. }) = &host_widgets.command_status {
+                let timeout_duration = Duration::from_secs(30); // 30 seconds for service commands
                 
                 if now.duration_since(*start_time) > timeout_duration {
                     hosts_to_clear.push(hostname.clone());
                 }
             }
-            // Also clear success/failed status after display time
-            else if let Some(CommandStatus::Success { completed_at, .. }) = &host_widgets.command_status {
-                if now.duration_since(*completed_at) > Duration::from_secs(3) {
-                    hosts_to_clear.push(hostname.clone());
-                }
-            }
         }
         
         // Clear timed out commands
@@ -569,7 +548,7 @@ impl TuiApp {
     }
 
     /// Close terminal popup for a specific hostname
-    pub fn close_terminal_popup(&mut self, hostname: &str) {
+    pub fn _close_terminal_popup(&mut self, hostname: &str) {
         if let Some(ref mut popup) = self.terminal_popup {
             if popup.hostname == hostname {
                 popup.close();
@@ -578,32 +557,6 @@ impl TuiApp {
         }
     }
 
-    /// Check for rebuild completion by detecting agent hash changes
-    pub fn check_rebuild_completion(&mut self, metric_store: &MetricStore) {
-        let mut hosts_to_complete = Vec::new();
-        
-        for (hostname, host_widgets) in &self.host_widgets {
-            if let Some(CommandStatus::InProgress { command_type: CommandType::SystemRebuild, .. }) = &host_widgets.command_status {
-                // Check if agent hash has changed (indicating successful rebuild)
-                if let Some(agent_hash_metric) = metric_store.get_metric(hostname, "system_agent_hash") {
-                    if let cm_dashboard_shared::MetricValue::String(current_hash) = &agent_hash_metric.value {
-                        // Compare with stored hash (if we have one)
-                        if let Some(stored_hash) = host_widgets.system_widget.get_agent_hash() {
-                            if current_hash != stored_hash {
-                                // Agent hash changed - rebuild completed successfully
-                                hosts_to_complete.push(hostname.clone());
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        
-        // Mark rebuilds as completed
-        for hostname in hosts_to_complete {
-            self.complete_command(&hostname);
-        }
-    }
 
     /// Scroll the focused panel up or down
     pub fn scroll_focused_panel(&mut self, direction: i32) {
@@ -774,13 +727,9 @@ impl TuiApp {
             // Check if this host has a command status that affects the icon
             let (status_icon, status_color) = if let Some(host_widgets) = self.host_widgets.get(host) {
                 match &host_widgets.command_status {
-                    Some(CommandStatus::InProgress { command_type: CommandType::SystemRebuild, .. }) => {
-                        // Show blue circular arrow during rebuild
-                        ("↻", Theme::highlight())
-                    }
-                    Some(CommandStatus::Success { command_type: CommandType::SystemRebuild, .. }) => {
-                        // Show green checkmark for successful rebuild
-                        ("✓", Theme::success())
+                    Some(CommandStatus::InProgress { .. }) => {
+                        // Show working indicator for in-progress commands
+                        ("⏳", Theme::highlight())
                     }
                     _ => {
                         // Normal status icon based on metrics
@@ -950,7 +899,7 @@ impl TuiApp {
     /// Render terminal popup with streaming output
     fn render_terminal_popup(&self, frame: &mut Frame, area: Rect, popup: &TerminalPopup) {
         use ratatui::{
-            style::{Color, Modifier, Style},
+            style::{Color, Style},
             text::{Line, Span},
             widgets::{Block, Borders, Clear, Paragraph, Wrap},
         };

dashboard/src/ui/widgets/services.rs

@@ -146,7 +146,6 @@ impl ServicesWidget {
                         }
                     }
                 }
-                _ => {} // Success/Failed states will show normal status
             }
         }
         
@@ -561,7 +560,6 @@ impl ServicesWidget {
                                     StatusIcons::create_status_spans(*line_status, line_text)
                                 }
                             }
-                            _ => StatusIcons::create_status_spans(*line_status, line_text)
                         }
                     } else {
                         StatusIcons::create_status_spans(*line_status, line_text)

dashboard/src/ui/widgets/system.rs

@@ -129,7 +129,7 @@ impl SystemWidget {
     }
 
     /// Get the current agent hash for rebuild completion detection
-    pub fn get_agent_hash(&self) -> Option<&String> {
+    pub fn _get_agent_hash(&self) -> Option<&String> {
         self.agent_hash.as_ref()
     }