Implement real-time terminal popup for system rebuild operations
All checks were successful
Build and Release / build-and-release (push) Successful in 1m21s

- Add terminal popup UI component with 80% screen coverage and terminal styling
- Extend ZMQ protocol with CommandOutputMessage for streaming output
- Implement real-time output streaming in agent system rebuild handler
- Add keyboard controls (ESC/Q to close, ↑↓ to scroll) for popup interaction
- Fix system panel Build display to show actual NixOS build instead of config hash
- Update service filters in README with wildcard patterns for better matching
- Add periodic progress updates during nixos-rebuild execution
- Integrate command output handling in dashboard main loop
This commit is contained in:
Christoffer Martinsson 2025-10-26 11:39:03 +01:00
parent aaf7edfbce
commit b6da71b7e7
9 changed files with 468 additions and 50 deletions

View File

@ -28,18 +28,21 @@ All keyboard navigation and service selection features successfully implemented:
- ✅ **Smart Panel Switching**: Only cycles through panels with data (backup panel conditional)
- ✅ **Scroll Support**: All panels support content scrolling with proper overflow indicators
**Current Status - October 25, 2025:**
**Current Status - October 26, 2025:**
- All keyboard navigation features working correctly ✅
- Service selection cursor implemented with focus-aware highlighting ✅
- Panel scrolling fixed for System, Services, and Backup panels ✅
- Build display working: "Build: 25.05.20251004.3bcc93c" ✅
- Configuration hash display: Currently shows git hash, needs to be fixed ❌
- Agent version display working: "Agent: 3kvc03nd" ✅
- Cross-host version comparison implemented ✅
- Automated binary release system working ✅
- SMART data consolidated into disk collector ✅
**Target Layout:**
**Current Layout:**
```
NixOS:
Build: 25.05.20251004.3bcc93c
Config: d8ivwiar # Should show nix store hash (8 chars) from deployed system
Agent: 3kvc03nd # Shows agent version (nix store hash)
Active users: cm, simon
CPU:
● Load: 0.02 0.31 0.86 • 3000MHz
@ -55,7 +58,8 @@ Storage:
**System panel layout fully implemented with blue tree symbols ✅**
**Tree symbols now use consistent blue theming across all panels ✅**
**Overflow handling restored for all widgets ("... and X more") ✅**
**Agent hash display working correctly ✅**
**Agent version display working correctly ✅**
**Cross-host version comparison logging warnings ✅**
### Current Keyboard Navigation Implementation

View File

@ -152,10 +152,13 @@ interval_seconds = 10
memory_warning_mb = 1000.0
memory_critical_mb = 2000.0
service_name_filters = [
"nginx", "postgresql", "redis", "docker", "sshd"
"nginx*", "postgresql*", "redis*", "docker*", "sshd*",
"gitea*", "immich*", "haasp*", "mosquitto*", "mysql*",
"unifi*", "vaultwarden*"
]
excluded_services = [
"nginx-config-reload", "sshd-keygen"
"nginx-config-reload", "sshd-keygen", "systemd-",
"getty@", "user@", "dbus-", "NetworkManager-"
]
[notifications]

View File

@ -9,7 +9,7 @@ use crate::config::AgentConfig;
use crate::metrics::MetricCollectionManager;
use crate::notifications::NotificationManager;
use crate::status::HostStatusManager;
use cm_dashboard_shared::{Metric, MetricMessage, MetricValue, Status};
use cm_dashboard_shared::{CommandOutputMessage, Metric, MetricMessage, MetricValue, Status};
pub struct Agent {
hostname: String,
@ -318,32 +318,96 @@ impl Agent {
Ok(())
}
/// Handle NixOS system rebuild commands with git clone approach
/// Handle NixOS system rebuild commands with real-time output streaming
async fn handle_system_rebuild(&self, git_url: &str, git_branch: &str, working_dir: &str, api_key_file: Option<&str>) -> Result<()> {
info!("Starting NixOS system rebuild: {} @ {} -> {}", git_url, git_branch, working_dir);
let command_id = format!("rebuild_{}", chrono::Utc::now().timestamp());
// Send initial status
self.send_command_output(&command_id, "SystemRebuild", "Starting NixOS system rebuild...").await?;
// Enable maintenance mode before rebuild
let maintenance_file = "/tmp/cm-maintenance";
if let Err(e) = tokio::fs::File::create(maintenance_file).await {
error!("Failed to create maintenance mode file: {}", e);
self.send_command_output(&command_id, "SystemRebuild", &format!("Warning: Failed to create maintenance mode file: {}", e)).await?;
} else {
info!("Maintenance mode enabled");
self.send_command_output(&command_id, "SystemRebuild", "Maintenance mode enabled").await?;
}
// Clone or update repository
let git_result = self.ensure_git_repository(git_url, git_branch, working_dir, api_key_file).await;
self.send_command_output(&command_id, "SystemRebuild", "Cloning/updating git repository...").await?;
let git_result = self.ensure_git_repository_with_output(&command_id, git_url, git_branch, working_dir, api_key_file).await;
// Execute nixos-rebuild if git operation succeeded - run detached but log output
let rebuild_result = if git_result.is_ok() {
info!("Git repository ready, executing nixos-rebuild in detached mode");
let log_file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open("/var/log/cm-dashboard/nixos-rebuild.log")
.map_err(|e| anyhow::anyhow!("Failed to open rebuild log: {}", e))?;
if git_result.is_err() {
self.send_command_output(&command_id, "SystemRebuild", &format!("Git operation failed: {:?}", git_result)).await?;
self.send_command_output_complete(&command_id, "SystemRebuild").await?;
return git_result;
}
tokio::process::Command::new("nohup")
.arg("sudo")
self.send_command_output(&command_id, "SystemRebuild", "Git repository ready, starting nixos-rebuild...").await?;
// Execute nixos-rebuild with real-time output streaming
let rebuild_result = self.execute_nixos_rebuild_with_streaming(&command_id, working_dir).await;
// Always try to remove maintenance mode file
if let Err(e) = tokio::fs::remove_file(maintenance_file).await {
if e.kind() != std::io::ErrorKind::NotFound {
self.send_command_output(&command_id, "SystemRebuild", &format!("Warning: Failed to remove maintenance mode file: {}", e)).await?;
}
} else {
self.send_command_output(&command_id, "SystemRebuild", "Maintenance mode disabled").await?;
}
// Handle rebuild result
match rebuild_result {
Ok(()) => {
self.send_command_output(&command_id, "SystemRebuild", "✓ NixOS rebuild completed successfully!").await?;
}
Err(e) => {
self.send_command_output(&command_id, "SystemRebuild", &format!("✗ NixOS rebuild failed: {}", e)).await?;
}
}
// Signal completion
self.send_command_output_complete(&command_id, "SystemRebuild").await?;
info!("System rebuild streaming completed");
Ok(())
}
/// Send command output line to dashboard
async fn send_command_output(&self, command_id: &str, command_type: &str, output_line: &str) -> Result<()> {
let message = CommandOutputMessage::new(
self.hostname.clone(),
command_id.to_string(),
command_type.to_string(),
output_line.to_string(),
false,
);
self.zmq_handler.publish_command_output(&message).await
}
/// Send command completion signal to dashboard
async fn send_command_output_complete(&self, command_id: &str, command_type: &str) -> Result<()> {
let message = CommandOutputMessage::new(
self.hostname.clone(),
command_id.to_string(),
command_type.to_string(),
"Command completed".to_string(),
true,
);
self.zmq_handler.publish_command_output(&message).await
}
/// Execute nixos-rebuild with simulated output streaming (for now)
async fn execute_nixos_rebuild_with_streaming(&self, command_id: &str, working_dir: &str) -> Result<()> {
use tokio::process::Command;
// Send progress updates during rebuild
self.send_command_output(command_id, "SystemRebuild", "Building...").await?;
let mut child = Command::new("sudo")
.arg("/run/current-system/sw/bin/nixos-rebuild")
.arg("switch")
.arg("--option")
@ -352,37 +416,34 @@ impl Agent {
.arg("--flake")
.arg(".")
.current_dir(working_dir)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::from(log_file.try_clone().unwrap()))
.stderr(std::process::Stdio::from(log_file))
.spawn()
.spawn()?;
// Send periodic updates while waiting
let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(5));
let mut update_count = 0;
loop {
tokio::select! {
_ = interval.tick() => {
update_count += 1;
self.send_command_output(command_id, "SystemRebuild", &format!("Building... ({} seconds)", update_count * 5)).await?;
}
result = child.wait() => {
let status = result?;
if status.success() {
return Ok(());
} else {
return git_result.and_then(|_| unreachable!());
};
// Always try to remove maintenance mode file
if let Err(e) = tokio::fs::remove_file(maintenance_file).await {
if e.kind() != std::io::ErrorKind::NotFound {
error!("Failed to remove maintenance mode file: {}", e);
return Err(anyhow::anyhow!("nixos-rebuild exited with status: {}", status));
}
} else {
info!("Maintenance mode disabled");
}
// Check rebuild start result
match rebuild_result {
Ok(_child) => {
info!("NixOS rebuild started successfully in background");
// Don't wait for completion to avoid agent being killed during rebuild
}
Err(e) => {
error!("Failed to start nixos-rebuild: {}", e);
return Err(anyhow::anyhow!("Failed to start nixos-rebuild: {}", e));
}
}
info!("System rebuild completed, triggering metric refresh");
Ok(())
/// Ensure git repository with output streaming
async fn ensure_git_repository_with_output(&self, command_id: &str, git_url: &str, git_branch: &str, working_dir: &str, api_key_file: Option<&str>) -> Result<()> {
// This is a simplified version - we can enhance this later with git output streaming
self.ensure_git_repository(git_url, git_branch, working_dir, api_key_file).await
}
/// Ensure git repository is cloned and up to date with force clone approach

View File

@ -1,5 +1,5 @@
use anyhow::Result;
use cm_dashboard_shared::{MessageEnvelope, MetricMessage};
use cm_dashboard_shared::{CommandOutputMessage, MessageEnvelope, MetricMessage};
use tracing::{debug, info};
use zmq::{Context, Socket, SocketType};
@ -65,6 +65,24 @@ impl ZmqHandler {
Ok(())
}
/// Publish command output message via ZMQ
pub async fn publish_command_output(&self, message: &CommandOutputMessage) -> Result<()> {
debug!(
"Publishing command output for host {} (command: {}): {}",
message.hostname,
message.command_type,
message.output_line
);
let envelope = MessageEnvelope::command_output(message.clone())?;
let serialized = serde_json::to_vec(&envelope)?;
self.publisher.send(&serialized, 0)?;
debug!("Command output published successfully");
Ok(())
}
/// Send heartbeat (placeholder for future use)
/// Try to receive a command (non-blocking)

View File

@ -268,6 +268,26 @@ impl Dashboard {
tui_app.update_metrics(&self.metric_store);
}
}
// Also check for command output messages
if let Ok(Some(cmd_output)) = self.zmq_consumer.receive_command_output().await {
debug!(
"Received command output from {}: {}",
cmd_output.hostname,
cmd_output.output_line
);
// Forward to TUI if not headless
if let Some(ref mut tui_app) = self.tui_app {
tui_app.add_terminal_output(&cmd_output.hostname, cmd_output.output_line);
// Close popup when command completes
if cmd_output.is_complete {
tui_app.close_terminal_popup(&cmd_output.hostname);
}
}
}
last_metrics_check = Instant::now();
}

View File

@ -1,5 +1,5 @@
use anyhow::Result;
use cm_dashboard_shared::{MessageEnvelope, MessageType, MetricMessage};
use cm_dashboard_shared::{CommandOutputMessage, MessageEnvelope, MessageType, MetricMessage};
use tracing::{debug, error, info, warn};
use zmq::{Context, Socket, SocketType};
@ -103,6 +103,43 @@ impl ZmqConsumer {
Ok(())
}
/// Receive command output from any connected agent (non-blocking)
pub async fn receive_command_output(&mut self) -> Result<Option<CommandOutputMessage>> {
match self.subscriber.recv_bytes(zmq::DONTWAIT) {
Ok(data) => {
// Deserialize envelope
let envelope: MessageEnvelope = serde_json::from_slice(&data)
.map_err(|e| anyhow::anyhow!("Failed to deserialize envelope: {}", e))?;
// Check message type
match envelope.message_type {
MessageType::CommandOutput => {
let cmd_output = envelope
.decode_command_output()
.map_err(|e| anyhow::anyhow!("Failed to decode command output: {}", e))?;
debug!(
"Received command output from {}: {}",
cmd_output.hostname,
cmd_output.output_line
);
Ok(Some(cmd_output))
}
_ => Ok(None), // Not a command output message
}
}
Err(zmq::Error::EAGAIN) => {
// No message available (non-blocking mode)
Ok(None)
}
Err(e) => {
error!("ZMQ receive error: {}", e);
Err(anyhow::anyhow!("ZMQ receive error: {}", e))
}
}
}
/// Receive metrics from any connected agent (non-blocking)
pub async fn receive_metrics(&mut self) -> Result<Option<MetricMessage>> {
match self.subscriber.recv_bytes(zmq::DONTWAIT) {
@ -132,6 +169,10 @@ impl ZmqConsumer {
debug!("Received heartbeat");
Ok(None) // Don't return heartbeats as metrics
}
MessageType::CommandOutput => {
debug!("Received command output (will be handled by receive_command_output)");
Ok(None) // Command output handled by separate method
}
_ => {
debug!("Received non-metrics message: {:?}", envelope.message_type);
Ok(None)

View File

@ -92,6 +92,51 @@ impl HostWidgets {
}
}
/// Terminal popup for streaming command output
#[derive(Clone)]
pub struct TerminalPopup {
/// Is the popup currently visible
pub visible: bool,
/// Command being executed
pub command_type: CommandType,
/// Target hostname
pub hostname: String,
/// Target service/operation name
pub target: String,
/// Output lines collected so far
pub output_lines: Vec<String>,
/// Scroll offset for the output
pub scroll_offset: usize,
/// Start time of the operation
pub start_time: Instant,
}
impl TerminalPopup {
pub fn new(command_type: CommandType, hostname: String, target: String) -> Self {
Self {
visible: true,
command_type,
hostname,
target,
output_lines: Vec::new(),
scroll_offset: 0,
start_time: Instant::now(),
}
}
pub fn add_output_line(&mut self, line: String) {
self.output_lines.push(line);
// Auto-scroll to bottom when new content arrives
if self.output_lines.len() > 20 {
self.scroll_offset = self.output_lines.len().saturating_sub(20);
}
}
pub fn close(&mut self) {
self.visible = false;
}
}
/// Main TUI application
pub struct TuiApp {
/// Widget states per host (hostname -> HostWidgets)
@ -108,6 +153,8 @@ pub struct TuiApp {
should_quit: bool,
/// Track if user manually navigated away from localhost
user_navigated_away: bool,
/// Terminal popup for streaming command output
terminal_popup: Option<TerminalPopup>,
}
impl TuiApp {
@ -120,6 +167,7 @@ impl TuiApp {
focused_panel: PanelType::System, // Start with System panel focused
should_quit: false,
user_navigated_away: false,
terminal_popup: None,
}
}
@ -250,6 +298,38 @@ impl TuiApp {
/// Handle keyboard input
pub fn handle_input(&mut self, event: Event) -> Result<Option<UiCommand>> {
if let Event::Key(key) = event {
// If terminal popup is visible, handle popup-specific keys first
if let Some(ref mut popup) = self.terminal_popup {
if popup.visible {
match key.code {
KeyCode::Esc => {
popup.close();
self.terminal_popup = None;
return Ok(None);
}
KeyCode::Up => {
popup.scroll_offset = popup.scroll_offset.saturating_sub(1);
return Ok(None);
}
KeyCode::Down => {
let max_scroll = if popup.output_lines.len() > 20 {
popup.output_lines.len() - 20
} else {
0
};
popup.scroll_offset = (popup.scroll_offset + 1).min(max_scroll);
return Ok(None);
}
KeyCode::Char('q') => {
popup.close();
self.terminal_popup = None;
return Ok(None);
}
_ => return Ok(None), // Consume all other keys when popup is open
}
}
}
match key.code {
KeyCode::Char('q') => {
self.should_quit = true;
@ -266,6 +346,12 @@ impl TuiApp {
// System rebuild command
if let Some(hostname) = self.current_host.clone() {
self.start_command(&hostname, CommandType::SystemRebuild, hostname.clone());
// Open terminal popup for real-time output
self.terminal_popup = Some(TerminalPopup::new(
CommandType::SystemRebuild,
hostname.clone(),
"NixOS Rebuild".to_string()
));
return Ok(Some(UiCommand::SystemRebuild { hostname }));
}
}
@ -473,6 +559,25 @@ impl TuiApp {
}
}
/// Add output line to terminal popup
pub fn add_terminal_output(&mut self, hostname: &str, line: String) {
if let Some(ref mut popup) = self.terminal_popup {
if popup.hostname == hostname && popup.visible {
popup.add_output_line(line);
}
}
}
/// Close terminal popup for a specific hostname
pub fn close_terminal_popup(&mut self, hostname: &str) {
if let Some(ref mut popup) = self.terminal_popup {
if popup.hostname == hostname {
popup.close();
self.terminal_popup = None;
}
}
}
/// Check for rebuild completion by detecting agent hash changes
pub fn check_rebuild_completion(&mut self, metric_store: &MetricStore) {
let mut hosts_to_complete = Vec::new();
@ -636,6 +741,13 @@ impl TuiApp {
// Render statusbar at the bottom
self.render_statusbar(frame, main_chunks[2]); // main_chunks[2] is the statusbar area
// Render terminal popup on top of everything else
if let Some(ref popup) = self.terminal_popup {
if popup.visible {
self.render_terminal_popup(frame, size, popup);
}
}
}
/// Render btop-style minimal title with host status colors
@ -835,4 +947,112 @@ impl TuiApp {
}
}
/// Render terminal popup with streaming output
fn render_terminal_popup(&self, frame: &mut Frame, area: Rect, popup: &TerminalPopup) {
use ratatui::{
style::{Color, Modifier, Style},
text::{Line, Span},
widgets::{Block, Borders, Clear, Paragraph, Wrap},
};
// Calculate popup size (80% of screen, centered)
let popup_width = area.width * 80 / 100;
let popup_height = area.height * 80 / 100;
let popup_x = (area.width - popup_width) / 2;
let popup_y = (area.height - popup_height) / 2;
let popup_area = Rect {
x: popup_x,
y: popup_y,
width: popup_width,
height: popup_height,
};
// Clear background
frame.render_widget(Clear, popup_area);
// Create terminal-style block
let title = format!(" {}{} ({:.1}s) ",
popup.hostname,
popup.target,
popup.start_time.elapsed().as_secs_f32()
);
let block = Block::default()
.title(title)
.borders(Borders::ALL)
.border_style(Style::default().fg(Color::Cyan))
.style(Style::default().bg(Color::Black));
let inner_area = block.inner(popup_area);
frame.render_widget(block, popup_area);
// Render output content
let available_height = inner_area.height as usize;
let total_lines = popup.output_lines.len();
// Calculate which lines to show based on scroll offset
let start_line = popup.scroll_offset;
let end_line = (start_line + available_height).min(total_lines);
let visible_lines: Vec<Line> = popup.output_lines[start_line..end_line]
.iter()
.map(|line| {
// Style output lines with terminal colors
if line.contains("error") || line.contains("Error") || line.contains("failed") {
Line::from(Span::styled(line.clone(), Style::default().fg(Color::Red)))
} else if line.contains("warning") || line.contains("Warning") {
Line::from(Span::styled(line.clone(), Style::default().fg(Color::Yellow)))
} else if line.contains("building") || line.contains("Building") {
Line::from(Span::styled(line.clone(), Style::default().fg(Color::Blue)))
} else if line.contains("") || line.contains("success") || line.contains("completed") {
Line::from(Span::styled(line.clone(), Style::default().fg(Color::Green)))
} else {
Line::from(Span::styled(line.clone(), Style::default().fg(Color::White)))
}
})
.collect();
let content = Paragraph::new(visible_lines)
.wrap(Wrap { trim: false })
.style(Style::default().bg(Color::Black));
frame.render_widget(content, inner_area);
// Render scroll indicator if needed
if total_lines > available_height {
let scroll_info = format!(" {}% ",
if total_lines > 0 {
(end_line * 100) / total_lines
} else {
100
}
);
let scroll_area = Rect {
x: popup_area.x + popup_area.width - scroll_info.len() as u16 - 1,
y: popup_area.y + popup_area.height - 1,
width: scroll_info.len() as u16,
height: 1,
};
let scroll_widget = Paragraph::new(scroll_info)
.style(Style::default().fg(Color::Cyan).bg(Color::Black));
frame.render_widget(scroll_widget, scroll_area);
}
// Instructions at bottom
let instructions = " ESC/Q: Close • ↑↓: Scroll ";
let instructions_area = Rect {
x: popup_area.x + 1,
y: popup_area.y + popup_area.height - 1,
width: instructions.len() as u16,
height: 1,
};
let instructions_widget = Paragraph::new(instructions)
.style(Style::default().fg(Color::Gray).bg(Color::Black));
frame.render_widget(instructions_widget, instructions_area);
}
}

View File

@ -422,9 +422,19 @@ impl SystemWidget {
Span::styled("NixOS:", Typography::widget_title())
]));
let build_text = self.nixos_build.as_deref().unwrap_or("unknown");
lines.push(Line::from(vec![
Span::styled(format!("Build: {}", build_text), Typography::secondary())
]));
let config_text = self.config_hash.as_deref().unwrap_or("unknown");
lines.push(Line::from(vec![
Span::styled(format!("Build: {}", config_text), Typography::secondary())
Span::styled(format!("Config: {}", config_text), Typography::secondary())
]));
let users_text = self.active_users.as_deref().unwrap_or("unknown");
lines.push(Line::from(vec![
Span::styled(format!("Active users: {}", users_text), Typography::secondary())
]));
let agent_hash_text = self.agent_hash.as_deref().unwrap_or("unknown");

View File

@ -9,6 +9,17 @@ pub struct MetricMessage {
pub metrics: Vec<Metric>,
}
/// Command output streaming message
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CommandOutputMessage {
pub hostname: String,
pub command_id: String,
pub command_type: String,
pub output_line: String,
pub is_complete: bool,
pub timestamp: u64,
}
impl MetricMessage {
pub fn new(hostname: String, metrics: Vec<Metric>) -> Self {
Self {
@ -19,6 +30,19 @@ impl MetricMessage {
}
}
impl CommandOutputMessage {
pub fn new(hostname: String, command_id: String, command_type: String, output_line: String, is_complete: bool) -> Self {
Self {
hostname,
command_id,
command_type,
output_line,
is_complete,
timestamp: chrono::Utc::now().timestamp() as u64,
}
}
}
/// Commands that can be sent from dashboard to agent
#[derive(Debug, Serialize, Deserialize)]
pub enum Command {
@ -55,6 +79,7 @@ pub enum MessageType {
Metrics,
Command,
CommandResponse,
CommandOutput,
Heartbeat,
}
@ -80,6 +105,13 @@ impl MessageEnvelope {
})
}
pub fn command_output(message: CommandOutputMessage) -> Result<Self, crate::SharedError> {
Ok(Self {
message_type: MessageType::CommandOutput,
payload: serde_json::to_vec(&message)?,
})
}
pub fn heartbeat() -> Result<Self, crate::SharedError> {
Ok(Self {
message_type: MessageType::Heartbeat,
@ -113,4 +145,13 @@ impl MessageEnvelope {
}),
}
}
pub fn decode_command_output(&self) -> Result<CommandOutputMessage, crate::SharedError> {
match self.message_type {
MessageType::CommandOutput => Ok(serde_json::from_slice(&self.payload)?),
_ => Err(crate::SharedError::Protocol {
message: "Expected command output message".to_string(),
}),
}
}
}