Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| bc94f75328 | |||
| b6da71b7e7 | |||
| aaf7edfbce | |||
| bb72c42726 | |||
| af5f96ce2f | |||
| 8dffe18a23 | |||
| 0c544753f9 | |||
| c8e26b9bac | |||
| 60ef712fac | |||
| 1ed4666dfd |
@@ -91,4 +91,38 @@ jobs:
|
|||||||
curl -X POST \
|
curl -X POST \
|
||||||
-H "Authorization: token $GITEA_TOKEN" \
|
-H "Authorization: token $GITEA_TOKEN" \
|
||||||
-F "attachment=@release/cm-dashboard-linux-x86_64.tar.gz" \
|
-F "attachment=@release/cm-dashboard-linux-x86_64.tar.gz" \
|
||||||
"https://gitea.cmtec.se/api/v1/repos/cm/cm-dashboard/releases/$RELEASE_ID/assets?name=cm-dashboard-linux-x86_64.tar.gz"
|
"https://gitea.cmtec.se/api/v1/repos/cm/cm-dashboard/releases/$RELEASE_ID/assets?name=cm-dashboard-linux-x86_64.tar.gz"
|
||||||
|
|
||||||
|
- name: Update NixOS Configuration
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{ secrets.GITEATOKEN }}
|
||||||
|
run: |
|
||||||
|
VERSION="${{ steps.version.outputs.VERSION }}"
|
||||||
|
|
||||||
|
# Clone nixosbox repository
|
||||||
|
git clone https://$GITEA_TOKEN@gitea.cmtec.se/cm/nixosbox.git nixosbox-update
|
||||||
|
cd nixosbox-update
|
||||||
|
|
||||||
|
# Get hash for the new release tarball
|
||||||
|
TARBALL_URL="https://gitea.cmtec.se/cm/cm-dashboard/releases/download/$VERSION/cm-dashboard-linux-x86_64.tar.gz"
|
||||||
|
|
||||||
|
# Download tarball to get correct hash
|
||||||
|
curl -L -o cm-dashboard.tar.gz "$TARBALL_URL"
|
||||||
|
# Convert sha256 hex to base64 for Nix hash format using Python
|
||||||
|
NEW_HASH=$(sha256sum cm-dashboard.tar.gz | cut -d' ' -f1)
|
||||||
|
NIX_HASH="sha256-$(python3 -c "import base64, binascii; print(base64.b64encode(binascii.unhexlify('$NEW_HASH')).decode())")"
|
||||||
|
|
||||||
|
# Update the NixOS configuration
|
||||||
|
sed -i "s|version = \"v[^\"]*\"|version = \"$VERSION\"|" hosts/common/cm-dashboard.nix
|
||||||
|
sed -i "s|sha256 = \"sha256-[^\"]*\"|sha256 = \"$NIX_HASH\"|" hosts/common/cm-dashboard.nix
|
||||||
|
|
||||||
|
# Commit and push changes
|
||||||
|
git config user.name "Gitea Actions"
|
||||||
|
git config user.email "actions@gitea.cmtec.se"
|
||||||
|
git add hosts/common/cm-dashboard.nix
|
||||||
|
git commit -m "Auto-update cm-dashboard to $VERSION
|
||||||
|
|
||||||
|
- Update version to $VERSION with automated release
|
||||||
|
- Update tarball hash for new static binaries
|
||||||
|
- Automated update from cm-dashboard release workflow"
|
||||||
|
git push
|
||||||
14
CLAUDE.md
14
CLAUDE.md
@@ -28,18 +28,21 @@ All keyboard navigation and service selection features successfully implemented:
|
|||||||
- ✅ **Smart Panel Switching**: Only cycles through panels with data (backup panel conditional)
|
- ✅ **Smart Panel Switching**: Only cycles through panels with data (backup panel conditional)
|
||||||
- ✅ **Scroll Support**: All panels support content scrolling with proper overflow indicators
|
- ✅ **Scroll Support**: All panels support content scrolling with proper overflow indicators
|
||||||
|
|
||||||
**Current Status - October 25, 2025:**
|
**Current Status - October 26, 2025:**
|
||||||
- All keyboard navigation features working correctly ✅
|
- All keyboard navigation features working correctly ✅
|
||||||
- Service selection cursor implemented with focus-aware highlighting ✅
|
- Service selection cursor implemented with focus-aware highlighting ✅
|
||||||
- Panel scrolling fixed for System, Services, and Backup panels ✅
|
- Panel scrolling fixed for System, Services, and Backup panels ✅
|
||||||
- Build display working: "Build: 25.05.20251004.3bcc93c" ✅
|
- Build display working: "Build: 25.05.20251004.3bcc93c" ✅
|
||||||
- Configuration hash display: Currently shows git hash, needs to be fixed ❌
|
- Agent version display working: "Agent: 3kvc03nd" ✅
|
||||||
|
- Cross-host version comparison implemented ✅
|
||||||
|
- Automated binary release system working ✅
|
||||||
|
- SMART data consolidated into disk collector ✅
|
||||||
|
|
||||||
**Target Layout:**
|
**Current Layout:**
|
||||||
```
|
```
|
||||||
NixOS:
|
NixOS:
|
||||||
Build: 25.05.20251004.3bcc93c
|
Build: 25.05.20251004.3bcc93c
|
||||||
Config: d8ivwiar # Should show nix store hash (8 chars) from deployed system
|
Agent: 3kvc03nd # Shows agent version (nix store hash)
|
||||||
Active users: cm, simon
|
Active users: cm, simon
|
||||||
CPU:
|
CPU:
|
||||||
● Load: 0.02 0.31 0.86 • 3000MHz
|
● Load: 0.02 0.31 0.86 • 3000MHz
|
||||||
@@ -55,7 +58,8 @@ Storage:
|
|||||||
**System panel layout fully implemented with blue tree symbols ✅**
|
**System panel layout fully implemented with blue tree symbols ✅**
|
||||||
**Tree symbols now use consistent blue theming across all panels ✅**
|
**Tree symbols now use consistent blue theming across all panels ✅**
|
||||||
**Overflow handling restored for all widgets ("... and X more") ✅**
|
**Overflow handling restored for all widgets ("... and X more") ✅**
|
||||||
**Agent hash display working correctly ✅**
|
**Agent version display working correctly ✅**
|
||||||
|
**Cross-host version comparison logging warnings ✅**
|
||||||
|
|
||||||
### Current Keyboard Navigation Implementation
|
### Current Keyboard Navigation Implementation
|
||||||
|
|
||||||
|
|||||||
@@ -152,10 +152,13 @@ interval_seconds = 10
|
|||||||
memory_warning_mb = 1000.0
|
memory_warning_mb = 1000.0
|
||||||
memory_critical_mb = 2000.0
|
memory_critical_mb = 2000.0
|
||||||
service_name_filters = [
|
service_name_filters = [
|
||||||
"nginx", "postgresql", "redis", "docker", "sshd"
|
"nginx*", "postgresql*", "redis*", "docker*", "sshd*",
|
||||||
|
"gitea*", "immich*", "haasp*", "mosquitto*", "mysql*",
|
||||||
|
"unifi*", "vaultwarden*"
|
||||||
]
|
]
|
||||||
excluded_services = [
|
excluded_services = [
|
||||||
"nginx-config-reload", "sshd-keygen"
|
"nginx-config-reload", "sshd-keygen", "systemd-",
|
||||||
|
"getty@", "user@", "dbus-", "NetworkManager-"
|
||||||
]
|
]
|
||||||
|
|
||||||
[notifications]
|
[notifications]
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ use crate::config::AgentConfig;
|
|||||||
use crate::metrics::MetricCollectionManager;
|
use crate::metrics::MetricCollectionManager;
|
||||||
use crate::notifications::NotificationManager;
|
use crate::notifications::NotificationManager;
|
||||||
use crate::status::HostStatusManager;
|
use crate::status::HostStatusManager;
|
||||||
use cm_dashboard_shared::{Metric, MetricMessage};
|
use cm_dashboard_shared::{CommandOutputMessage, Metric, MetricMessage, MetricValue, Status};
|
||||||
|
|
||||||
pub struct Agent {
|
pub struct Agent {
|
||||||
hostname: String,
|
hostname: String,
|
||||||
@@ -162,6 +162,10 @@ impl Agent {
|
|||||||
let host_status_metric = self.host_status_manager.get_host_status_metric();
|
let host_status_metric = self.host_status_manager.get_host_status_metric();
|
||||||
metrics.push(host_status_metric);
|
metrics.push(host_status_metric);
|
||||||
|
|
||||||
|
// Add agent version metric for cross-host version comparison
|
||||||
|
let version_metric = self.get_agent_version_metric();
|
||||||
|
metrics.push(version_metric);
|
||||||
|
|
||||||
if metrics.is_empty() {
|
if metrics.is_empty() {
|
||||||
debug!("No metrics to broadcast");
|
debug!("No metrics to broadcast");
|
||||||
return Ok(());
|
return Ok(());
|
||||||
@@ -183,6 +187,39 @@ impl Agent {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Create agent version metric for cross-host version comparison
|
||||||
|
fn get_agent_version_metric(&self) -> Metric {
|
||||||
|
// Get version from executable path (same logic as main.rs get_version)
|
||||||
|
let version = self.get_agent_version();
|
||||||
|
|
||||||
|
Metric::new(
|
||||||
|
"agent_version".to_string(),
|
||||||
|
MetricValue::String(version),
|
||||||
|
Status::Ok,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get agent version from executable path
|
||||||
|
fn get_agent_version(&self) -> String {
|
||||||
|
match std::env::current_exe() {
|
||||||
|
Ok(exe_path) => {
|
||||||
|
let exe_str = exe_path.to_string_lossy();
|
||||||
|
|
||||||
|
// Extract Nix store hash from path
|
||||||
|
if let Some(hash_part) = exe_str.strip_prefix("/nix/store/") {
|
||||||
|
if let Some(hash) = hash_part.split('-').next() {
|
||||||
|
if hash.len() >= 8 {
|
||||||
|
return hash[..8].to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
"unknown".to_string()
|
||||||
|
},
|
||||||
|
Err(_) => "unknown".to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn handle_commands(&mut self) -> Result<()> {
|
async fn handle_commands(&mut self) -> Result<()> {
|
||||||
// Try to receive commands (non-blocking)
|
// Try to receive commands (non-blocking)
|
||||||
match self.zmq_handler.try_receive_command() {
|
match self.zmq_handler.try_receive_command() {
|
||||||
@@ -281,73 +318,167 @@ impl Agent {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handle NixOS system rebuild commands with git clone approach
|
/// Handle NixOS system rebuild commands with real-time output streaming
|
||||||
async fn handle_system_rebuild(&self, git_url: &str, git_branch: &str, working_dir: &str, api_key_file: Option<&str>) -> Result<()> {
|
async fn handle_system_rebuild(&self, git_url: &str, git_branch: &str, working_dir: &str, api_key_file: Option<&str>) -> Result<()> {
|
||||||
info!("Starting NixOS system rebuild: {} @ {} -> {}", git_url, git_branch, working_dir);
|
info!("Starting NixOS system rebuild: {} @ {} -> {}", git_url, git_branch, working_dir);
|
||||||
|
|
||||||
|
let command_id = format!("rebuild_{}", chrono::Utc::now().timestamp());
|
||||||
|
|
||||||
|
// Send initial status
|
||||||
|
self.send_command_output(&command_id, "SystemRebuild", "Starting NixOS system rebuild...").await?;
|
||||||
|
|
||||||
// Enable maintenance mode before rebuild
|
// Enable maintenance mode before rebuild
|
||||||
let maintenance_file = "/tmp/cm-maintenance";
|
let maintenance_file = "/tmp/cm-maintenance";
|
||||||
if let Err(e) = tokio::fs::File::create(maintenance_file).await {
|
if let Err(e) = tokio::fs::File::create(maintenance_file).await {
|
||||||
error!("Failed to create maintenance mode file: {}", e);
|
self.send_command_output(&command_id, "SystemRebuild", &format!("Warning: Failed to create maintenance mode file: {}", e)).await?;
|
||||||
} else {
|
} else {
|
||||||
info!("Maintenance mode enabled");
|
self.send_command_output(&command_id, "SystemRebuild", "Maintenance mode enabled").await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clone or update repository
|
// Clone or update repository
|
||||||
let git_result = self.ensure_git_repository(git_url, git_branch, working_dir, api_key_file).await;
|
self.send_command_output(&command_id, "SystemRebuild", "Cloning/updating git repository...").await?;
|
||||||
|
let git_result = self.ensure_git_repository_with_output(&command_id, git_url, git_branch, working_dir, api_key_file).await;
|
||||||
|
|
||||||
// Execute nixos-rebuild if git operation succeeded - run detached but log output
|
if git_result.is_err() {
|
||||||
let rebuild_result = if git_result.is_ok() {
|
self.send_command_output(&command_id, "SystemRebuild", &format!("Git operation failed: {:?}", git_result)).await?;
|
||||||
info!("Git repository ready, executing nixos-rebuild in detached mode");
|
self.send_command_output_complete(&command_id, "SystemRebuild").await?;
|
||||||
let log_file = std::fs::OpenOptions::new()
|
return git_result;
|
||||||
.create(true)
|
}
|
||||||
.append(true)
|
|
||||||
.open("/var/log/cm-dashboard/nixos-rebuild.log")
|
self.send_command_output(&command_id, "SystemRebuild", "Git repository ready, starting nixos-rebuild...").await?;
|
||||||
.map_err(|e| anyhow::anyhow!("Failed to open rebuild log: {}", e))?;
|
|
||||||
|
// Execute nixos-rebuild with real-time output streaming
|
||||||
tokio::process::Command::new("nohup")
|
let rebuild_result = self.execute_nixos_rebuild_with_streaming(&command_id, working_dir).await;
|
||||||
.arg("sudo")
|
|
||||||
.arg("/run/current-system/sw/bin/nixos-rebuild")
|
|
||||||
.arg("switch")
|
|
||||||
.arg("--option")
|
|
||||||
.arg("sandbox")
|
|
||||||
.arg("false")
|
|
||||||
.arg("--flake")
|
|
||||||
.arg(".")
|
|
||||||
.current_dir(working_dir)
|
|
||||||
.stdin(std::process::Stdio::null())
|
|
||||||
.stdout(std::process::Stdio::from(log_file.try_clone().unwrap()))
|
|
||||||
.stderr(std::process::Stdio::from(log_file))
|
|
||||||
.spawn()
|
|
||||||
} else {
|
|
||||||
return git_result.and_then(|_| unreachable!());
|
|
||||||
};
|
|
||||||
|
|
||||||
// Always try to remove maintenance mode file
|
// Always try to remove maintenance mode file
|
||||||
if let Err(e) = tokio::fs::remove_file(maintenance_file).await {
|
if let Err(e) = tokio::fs::remove_file(maintenance_file).await {
|
||||||
if e.kind() != std::io::ErrorKind::NotFound {
|
if e.kind() != std::io::ErrorKind::NotFound {
|
||||||
error!("Failed to remove maintenance mode file: {}", e);
|
self.send_command_output(&command_id, "SystemRebuild", &format!("Warning: Failed to remove maintenance mode file: {}", e)).await?;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
info!("Maintenance mode disabled");
|
self.send_command_output(&command_id, "SystemRebuild", "Maintenance mode disabled").await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check rebuild start result
|
// Handle rebuild result
|
||||||
match rebuild_result {
|
match rebuild_result {
|
||||||
Ok(_child) => {
|
Ok(()) => {
|
||||||
info!("NixOS rebuild started successfully in background");
|
self.send_command_output(&command_id, "SystemRebuild", "✓ NixOS rebuild completed successfully!").await?;
|
||||||
// Don't wait for completion to avoid agent being killed during rebuild
|
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!("Failed to start nixos-rebuild: {}", e);
|
self.send_command_output(&command_id, "SystemRebuild", &format!("✗ NixOS rebuild failed: {}", e)).await?;
|
||||||
return Err(anyhow::anyhow!("Failed to start nixos-rebuild: {}", e));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("System rebuild completed, triggering metric refresh");
|
// Signal completion
|
||||||
|
self.send_command_output_complete(&command_id, "SystemRebuild").await?;
|
||||||
|
|
||||||
|
info!("System rebuild streaming completed");
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Send command output line to dashboard
|
||||||
|
async fn send_command_output(&self, command_id: &str, command_type: &str, output_line: &str) -> Result<()> {
|
||||||
|
let message = CommandOutputMessage::new(
|
||||||
|
self.hostname.clone(),
|
||||||
|
command_id.to_string(),
|
||||||
|
command_type.to_string(),
|
||||||
|
output_line.to_string(),
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
self.zmq_handler.publish_command_output(&message).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Send command completion signal to dashboard
|
||||||
|
async fn send_command_output_complete(&self, command_id: &str, command_type: &str) -> Result<()> {
|
||||||
|
let message = CommandOutputMessage::new(
|
||||||
|
self.hostname.clone(),
|
||||||
|
command_id.to_string(),
|
||||||
|
command_type.to_string(),
|
||||||
|
"Command completed".to_string(),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
self.zmq_handler.publish_command_output(&message).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Execute nixos-rebuild with real-time output streaming
|
||||||
|
async fn execute_nixos_rebuild_with_streaming(&self, command_id: &str, working_dir: &str) -> Result<()> {
|
||||||
|
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||||
|
use tokio::process::Command;
|
||||||
|
|
||||||
|
let mut child = Command::new("sudo")
|
||||||
|
.arg("/run/current-system/sw/bin/nixos-rebuild")
|
||||||
|
.arg("switch")
|
||||||
|
.arg("--option")
|
||||||
|
.arg("sandbox")
|
||||||
|
.arg("false")
|
||||||
|
.arg("--flake")
|
||||||
|
.arg(".")
|
||||||
|
.current_dir(working_dir)
|
||||||
|
.stdout(std::process::Stdio::piped())
|
||||||
|
.stderr(std::process::Stdio::piped())
|
||||||
|
.spawn()?;
|
||||||
|
|
||||||
|
// Get stdout and stderr handles
|
||||||
|
let stdout = child.stdout.take().expect("Failed to get stdout");
|
||||||
|
let stderr = child.stderr.take().expect("Failed to get stderr");
|
||||||
|
|
||||||
|
// Create readers for both streams
|
||||||
|
let stdout_reader = BufReader::new(stdout);
|
||||||
|
let stderr_reader = BufReader::new(stderr);
|
||||||
|
|
||||||
|
let mut stdout_lines = stdout_reader.lines();
|
||||||
|
let mut stderr_lines = stderr_reader.lines();
|
||||||
|
|
||||||
|
// Stream output lines in real-time
|
||||||
|
loop {
|
||||||
|
tokio::select! {
|
||||||
|
// Read from stdout
|
||||||
|
line = stdout_lines.next_line() => {
|
||||||
|
match line {
|
||||||
|
Ok(Some(line)) => {
|
||||||
|
self.send_command_output(command_id, "SystemRebuild", &line).await?;
|
||||||
|
}
|
||||||
|
Ok(None) => {
|
||||||
|
// stdout closed
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
self.send_command_output(command_id, "SystemRebuild", &format!("stdout error: {}", e)).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Read from stderr
|
||||||
|
line = stderr_lines.next_line() => {
|
||||||
|
match line {
|
||||||
|
Ok(Some(line)) => {
|
||||||
|
self.send_command_output(command_id, "SystemRebuild", &line).await?;
|
||||||
|
}
|
||||||
|
Ok(None) => {
|
||||||
|
// stderr closed
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
self.send_command_output(command_id, "SystemRebuild", &format!("stderr error: {}", e)).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Wait for process completion
|
||||||
|
result = child.wait() => {
|
||||||
|
let status = result?;
|
||||||
|
if status.success() {
|
||||||
|
return Ok(());
|
||||||
|
} else {
|
||||||
|
return Err(anyhow::anyhow!("nixos-rebuild exited with status: {}", status));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ensure git repository with output streaming
|
||||||
|
async fn ensure_git_repository_with_output(&self, command_id: &str, git_url: &str, git_branch: &str, working_dir: &str, api_key_file: Option<&str>) -> Result<()> {
|
||||||
|
// This is a simplified version - we can enhance this later with git output streaming
|
||||||
|
self.ensure_git_repository(git_url, git_branch, working_dir, api_key_file).await
|
||||||
|
}
|
||||||
|
|
||||||
/// Ensure git repository is cloned and up to date with force clone approach
|
/// Ensure git repository is cloned and up to date with force clone approach
|
||||||
async fn ensure_git_repository(&self, git_url: &str, git_branch: &str, working_dir: &str, api_key_file: Option<&str>) -> Result<()> {
|
async fn ensure_git_repository(&self, git_url: &str, git_branch: &str, working_dir: &str, api_key_file: Option<&str>) -> Result<()> {
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|||||||
@@ -41,11 +41,11 @@ pub struct DiskCollector {
|
|||||||
|
|
||||||
impl DiskCollector {
|
impl DiskCollector {
|
||||||
pub fn new(config: DiskConfig) -> Self {
|
pub fn new(config: DiskConfig) -> Self {
|
||||||
// Create hysteresis thresholds for disk temperature
|
// Create hysteresis thresholds for disk temperature from config
|
||||||
let temperature_thresholds = HysteresisThresholds::with_custom_gaps(
|
let temperature_thresholds = HysteresisThresholds::with_custom_gaps(
|
||||||
60.0, // warning at 60°C
|
config.temperature_warning_celsius,
|
||||||
5.0, // 5°C gap for recovery
|
5.0, // 5°C gap for recovery
|
||||||
70.0, // critical at 70°C
|
config.temperature_critical_celsius,
|
||||||
5.0, // 5°C gap for recovery
|
5.0, // 5°C gap for recovery
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -219,18 +219,12 @@ impl DiskCollector {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Parse wear level from SMART output (SSD wear leveling)
|
/// Parse wear level from SMART output (SSD wear leveling)
|
||||||
|
/// Supports both NVMe and SATA SSD wear indicators
|
||||||
fn parse_wear_level_from_smart(&self, smart_output: &str) -> Option<f32> {
|
fn parse_wear_level_from_smart(&self, smart_output: &str) -> Option<f32> {
|
||||||
for line in smart_output.lines() {
|
for line in smart_output.lines() {
|
||||||
// Look for wear leveling indicators
|
let line = line.trim();
|
||||||
if line.contains("Wear_Leveling_Count") || line.contains("Media_Wearout_Indicator") {
|
|
||||||
let parts: Vec<&str> = line.split_whitespace().collect();
|
// NVMe drives - direct percentage used
|
||||||
if parts.len() >= 10 {
|
|
||||||
if let Ok(wear) = parts[9].parse::<f32>() {
|
|
||||||
return Some(100.0 - wear); // Convert to percentage used
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// NVMe drives might show percentage used directly
|
|
||||||
if line.contains("Percentage Used:") {
|
if line.contains("Percentage Used:") {
|
||||||
if let Some(wear_part) = line.split("Percentage Used:").nth(1) {
|
if let Some(wear_part) = line.split("Percentage Used:").nth(1) {
|
||||||
if let Some(wear_str) = wear_part.split('%').next() {
|
if let Some(wear_str) = wear_part.split('%').next() {
|
||||||
@@ -240,6 +234,38 @@ impl DiskCollector {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SATA SSD attributes - parse SMART table format
|
||||||
|
// Format: ID ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE
|
||||||
|
let parts: Vec<&str> = line.split_whitespace().collect();
|
||||||
|
if parts.len() >= 10 {
|
||||||
|
// SSD Life Left / Percent Lifetime Remaining (higher = less wear)
|
||||||
|
if line.contains("SSD_Life_Left") || line.contains("Percent_Lifetime_Remain") {
|
||||||
|
if let Ok(remaining) = parts[3].parse::<f32>() { // VALUE column
|
||||||
|
return Some(100.0 - remaining); // Convert remaining to used
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Media Wearout Indicator (lower = more wear, normalize to 0-100)
|
||||||
|
if line.contains("Media_Wearout_Indicator") {
|
||||||
|
if let Ok(remaining) = parts[3].parse::<f32>() { // VALUE column
|
||||||
|
return Some(100.0 - remaining); // Convert remaining to used
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wear Leveling Count (higher = less wear, but varies by manufacturer)
|
||||||
|
if line.contains("Wear_Leveling_Count") {
|
||||||
|
if let Ok(wear_count) = parts[3].parse::<f32>() { // VALUE column
|
||||||
|
// Most SSDs: 100 = new, decreases with wear
|
||||||
|
if wear_count <= 100.0 {
|
||||||
|
return Some(100.0 - wear_count);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Total LBAs Written - calculate against typical endurance if available
|
||||||
|
// This is more complex and manufacturer-specific, so we skip for now
|
||||||
|
}
|
||||||
}
|
}
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ pub mod disk;
|
|||||||
pub mod error;
|
pub mod error;
|
||||||
pub mod memory;
|
pub mod memory;
|
||||||
pub mod nixos;
|
pub mod nixos;
|
||||||
pub mod smart;
|
|
||||||
pub mod systemd;
|
pub mod systemd;
|
||||||
|
|
||||||
pub use error::CollectorError;
|
pub use error::CollectorError;
|
||||||
|
|||||||
@@ -1,178 +0,0 @@
|
|||||||
use async_trait::async_trait;
|
|
||||||
use cm_dashboard_shared::{Metric, MetricValue, Status, StatusTracker};
|
|
||||||
use std::process::Stdio;
|
|
||||||
use tokio::process::Command;
|
|
||||||
use tracing::{debug, warn};
|
|
||||||
|
|
||||||
use super::{Collector, CollectorError};
|
|
||||||
|
|
||||||
pub struct SmartCollector;
|
|
||||||
|
|
||||||
impl SmartCollector {
|
|
||||||
/// Get list of storage devices to monitor
|
|
||||||
async fn get_devices(&self) -> Result<Vec<String>, CollectorError> {
|
|
||||||
let output = Command::new("lsblk")
|
|
||||||
.args(["-d", "-n", "-o", "NAME,TYPE"])
|
|
||||||
.stdout(Stdio::piped())
|
|
||||||
.stderr(Stdio::null())
|
|
||||||
.output()
|
|
||||||
.await
|
|
||||||
.map_err(|e| CollectorError::SystemRead {
|
|
||||||
path: "lsblk".to_string(),
|
|
||||||
error: e.to_string()
|
|
||||||
})?;
|
|
||||||
|
|
||||||
if !output.status.success() {
|
|
||||||
return Ok(Vec::new()); // Return empty if lsblk fails
|
|
||||||
}
|
|
||||||
|
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
||||||
let mut devices = Vec::new();
|
|
||||||
|
|
||||||
for line in stdout.lines() {
|
|
||||||
let parts: Vec<&str> = line.split_whitespace().collect();
|
|
||||||
if parts.len() >= 2 && parts[1] == "disk" {
|
|
||||||
let device_name = parts[0];
|
|
||||||
if device_name.starts_with("nvme") || device_name.starts_with("sd") {
|
|
||||||
devices.push(format!("/dev/{}", device_name));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(devices)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Collect SMART data for a single device
|
|
||||||
async fn collect_device_smart(&self, device: &str, status_tracker: &mut StatusTracker) -> Result<Vec<Metric>, CollectorError> {
|
|
||||||
debug!("Collecting SMART data for device: {}", device);
|
|
||||||
|
|
||||||
let output = Command::new("sudo")
|
|
||||||
.args(["smartctl", "-H", "-A", device]) // Health and attributes only
|
|
||||||
.stdout(Stdio::piped())
|
|
||||||
.stderr(Stdio::null())
|
|
||||||
.output()
|
|
||||||
.await
|
|
||||||
.map_err(|e| CollectorError::SystemRead {
|
|
||||||
path: "lsblk".to_string(),
|
|
||||||
error: e.to_string()
|
|
||||||
})?;
|
|
||||||
|
|
||||||
if !output.status.success() {
|
|
||||||
warn!("smartctl failed for device: {}", device);
|
|
||||||
return Ok(Vec::new());
|
|
||||||
}
|
|
||||||
|
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
||||||
self.parse_smart_output(device, &stdout, status_tracker)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Parse smartctl output and create metrics
|
|
||||||
fn parse_smart_output(&self, device: &str, output: &str, status_tracker: &mut StatusTracker) -> Result<Vec<Metric>, CollectorError> {
|
|
||||||
let mut metrics = Vec::new();
|
|
||||||
let device_name = device.trim_start_matches("/dev/");
|
|
||||||
|
|
||||||
let mut health_ok = true;
|
|
||||||
let mut temperature: Option<f32> = None;
|
|
||||||
|
|
||||||
for line in output.lines() {
|
|
||||||
let line = line.trim();
|
|
||||||
|
|
||||||
// Parse health status
|
|
||||||
if line.contains("SMART overall-health self-assessment") {
|
|
||||||
if line.contains("FAILED") {
|
|
||||||
health_ok = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse temperature from various formats
|
|
||||||
if (line.contains("Temperature") || line.contains("Airflow_Temperature")) && temperature.is_none() {
|
|
||||||
if let Some(temp) = self.extract_temperature(line) {
|
|
||||||
temperature = Some(temp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create health metric
|
|
||||||
let health_status = if health_ok {
|
|
||||||
Status::Ok
|
|
||||||
} else {
|
|
||||||
Status::Critical
|
|
||||||
};
|
|
||||||
|
|
||||||
metrics.push(Metric::new(
|
|
||||||
format!("smart_health_{}", device_name),
|
|
||||||
MetricValue::String(if health_ok { "PASSED".to_string() } else { "FAILED".to_string() }),
|
|
||||||
health_status,
|
|
||||||
));
|
|
||||||
|
|
||||||
// Create temperature metric if available
|
|
||||||
if let Some(temp) = temperature {
|
|
||||||
let temp_status = if temp >= 70.0 {
|
|
||||||
Status::Critical
|
|
||||||
} else if temp >= 60.0 {
|
|
||||||
Status::Warning
|
|
||||||
} else {
|
|
||||||
Status::Ok
|
|
||||||
};
|
|
||||||
|
|
||||||
metrics.push(Metric::new(
|
|
||||||
format!("smart_temperature_{}", device_name),
|
|
||||||
MetricValue::Float(temp),
|
|
||||||
temp_status,
|
|
||||||
).with_unit("celsius".to_string()));
|
|
||||||
}
|
|
||||||
|
|
||||||
debug!("Collected {} SMART metrics for {}", metrics.len(), device);
|
|
||||||
Ok(metrics)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Extract temperature value from smartctl output line
|
|
||||||
fn extract_temperature(&self, line: &str) -> Option<f32> {
|
|
||||||
let parts: Vec<&str> = line.split_whitespace().collect();
|
|
||||||
|
|
||||||
for (i, part) in parts.iter().enumerate() {
|
|
||||||
if let Ok(temp) = part.parse::<f32>() {
|
|
||||||
// Check if this looks like a temperature value (reasonable range)
|
|
||||||
if temp > 0.0 && temp < 150.0 {
|
|
||||||
// Check context around the number
|
|
||||||
if i + 1 < parts.len() {
|
|
||||||
let next = parts[i + 1].to_lowercase();
|
|
||||||
if next.contains("celsius") || next.contains("°c") || next == "c" {
|
|
||||||
return Some(temp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// For SMART attribute lines, temperature is often the 10th column
|
|
||||||
if parts.len() >= 10 && (line.contains("Temperature") || line.contains("Airflow_Temperature")) {
|
|
||||||
return Some(temp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl Collector for SmartCollector {
|
|
||||||
async fn collect(&self, status_tracker: &mut StatusTracker) -> Result<Vec<Metric>, CollectorError> {
|
|
||||||
debug!("Starting SMART data collection");
|
|
||||||
|
|
||||||
let devices = self.get_devices().await?;
|
|
||||||
let mut all_metrics = Vec::new();
|
|
||||||
|
|
||||||
for device in devices {
|
|
||||||
match self.collect_device_smart(&device, status_tracker).await {
|
|
||||||
Ok(mut metrics) => {
|
|
||||||
all_metrics.append(&mut metrics);
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
warn!("Failed to collect SMART data for {}: {}", device, e);
|
|
||||||
// Continue with other devices
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
debug!("Collected {} total SMART metrics", all_metrics.len());
|
|
||||||
Ok(all_metrics)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use cm_dashboard_shared::{MessageEnvelope, MetricMessage};
|
use cm_dashboard_shared::{CommandOutputMessage, MessageEnvelope, MetricMessage};
|
||||||
use tracing::{debug, info};
|
use tracing::{debug, info};
|
||||||
use zmq::{Context, Socket, SocketType};
|
use zmq::{Context, Socket, SocketType};
|
||||||
|
|
||||||
@@ -65,6 +65,24 @@ impl ZmqHandler {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Publish command output message via ZMQ
|
||||||
|
pub async fn publish_command_output(&self, message: &CommandOutputMessage) -> Result<()> {
|
||||||
|
debug!(
|
||||||
|
"Publishing command output for host {} (command: {}): {}",
|
||||||
|
message.hostname,
|
||||||
|
message.command_type,
|
||||||
|
message.output_line
|
||||||
|
);
|
||||||
|
|
||||||
|
let envelope = MessageEnvelope::command_output(message.clone())?;
|
||||||
|
let serialized = serde_json::to_vec(&envelope)?;
|
||||||
|
|
||||||
|
self.publisher.send(&serialized, 0)?;
|
||||||
|
|
||||||
|
debug!("Command output published successfully");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// Send heartbeat (placeholder for future use)
|
/// Send heartbeat (placeholder for future use)
|
||||||
|
|
||||||
/// Try to receive a command (non-blocking)
|
/// Try to receive a command (non-blocking)
|
||||||
|
|||||||
@@ -36,7 +36,6 @@ pub struct CollectorConfig {
|
|||||||
pub memory: MemoryConfig,
|
pub memory: MemoryConfig,
|
||||||
pub disk: DiskConfig,
|
pub disk: DiskConfig,
|
||||||
pub systemd: SystemdConfig,
|
pub systemd: SystemdConfig,
|
||||||
pub smart: SmartConfig,
|
|
||||||
pub backup: BackupConfig,
|
pub backup: BackupConfig,
|
||||||
pub network: NetworkConfig,
|
pub network: NetworkConfig,
|
||||||
pub nixos: NixOSConfig,
|
pub nixos: NixOSConfig,
|
||||||
@@ -75,6 +74,11 @@ pub struct DiskConfig {
|
|||||||
pub usage_critical_percent: f32,
|
pub usage_critical_percent: f32,
|
||||||
/// Filesystem configurations
|
/// Filesystem configurations
|
||||||
pub filesystems: Vec<FilesystemConfig>,
|
pub filesystems: Vec<FilesystemConfig>,
|
||||||
|
/// SMART monitoring thresholds
|
||||||
|
pub temperature_warning_celsius: f32,
|
||||||
|
pub temperature_critical_celsius: f32,
|
||||||
|
pub wear_warning_percent: f32,
|
||||||
|
pub wear_critical_percent: f32,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Filesystem configuration entry
|
/// Filesystem configuration entry
|
||||||
@@ -102,16 +106,6 @@ pub struct SystemdConfig {
|
|||||||
pub host_user_mapping: String,
|
pub host_user_mapping: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// SMART collector configuration
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct SmartConfig {
|
|
||||||
pub enabled: bool,
|
|
||||||
pub interval_seconds: u64,
|
|
||||||
pub temperature_warning_celsius: f32,
|
|
||||||
pub temperature_critical_celsius: f32,
|
|
||||||
pub wear_warning_percent: f32,
|
|
||||||
pub wear_critical_percent: f32,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// NixOS collector configuration
|
/// NixOS collector configuration
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
|||||||
@@ -13,10 +13,26 @@ mod status;
|
|||||||
|
|
||||||
use agent::Agent;
|
use agent::Agent;
|
||||||
|
|
||||||
|
/// Get version showing cm-dashboard-agent package hash for easy deployment verification
|
||||||
|
fn get_version() -> &'static str {
|
||||||
|
// Get the path of the current executable
|
||||||
|
let exe_path = std::env::current_exe().expect("Failed to get executable path");
|
||||||
|
let exe_str = exe_path.to_string_lossy();
|
||||||
|
|
||||||
|
// Extract Nix store hash from path like /nix/store/HASH-cm-dashboard-v0.1.8/bin/cm-dashboard-agent
|
||||||
|
let hash_part = exe_str.strip_prefix("/nix/store/").expect("Not a nix store path");
|
||||||
|
let hash = hash_part.split('-').next().expect("Invalid nix store path format");
|
||||||
|
assert!(hash.len() >= 8, "Hash too short");
|
||||||
|
|
||||||
|
// Return first 8 characters of nix store hash
|
||||||
|
let short_hash = hash[..8].to_string();
|
||||||
|
Box::leak(short_hash.into_boxed_str())
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Parser)]
|
#[derive(Parser)]
|
||||||
#[command(name = "cm-dashboard-agent")]
|
#[command(name = "cm-dashboard-agent")]
|
||||||
#[command(about = "CM Dashboard metrics agent with individual metric collection")]
|
#[command(about = "CM Dashboard metrics agent with individual metric collection")]
|
||||||
#[command(version)]
|
#[command(version = get_version())]
|
||||||
struct Cli {
|
struct Cli {
|
||||||
/// Increase logging verbosity (-v, -vv)
|
/// Increase logging verbosity (-v, -vv)
|
||||||
#[arg(short, long, action = clap::ArgAction::Count)]
|
#[arg(short, long, action = clap::ArgAction::Count)]
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ use tracing::{error, info};
|
|||||||
|
|
||||||
use crate::collectors::{
|
use crate::collectors::{
|
||||||
backup::BackupCollector, cpu::CpuCollector, disk::DiskCollector, memory::MemoryCollector,
|
backup::BackupCollector, cpu::CpuCollector, disk::DiskCollector, memory::MemoryCollector,
|
||||||
nixos::NixOSCollector, smart::SmartCollector, systemd::SystemdCollector, Collector,
|
nixos::NixOSCollector, systemd::SystemdCollector, Collector,
|
||||||
};
|
};
|
||||||
use crate::config::{AgentConfig, CollectorConfig};
|
use crate::config::{AgentConfig, CollectorConfig};
|
||||||
|
|
||||||
@@ -61,14 +61,6 @@ impl MetricCollectionManager {
|
|||||||
info!("BENCHMARK: Backup collector only");
|
info!("BENCHMARK: Backup collector only");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Some("smart") => {
|
|
||||||
// SMART collector only
|
|
||||||
if config.smart.enabled {
|
|
||||||
let smart_collector = SmartCollector;
|
|
||||||
collectors.push(Box::new(smart_collector));
|
|
||||||
info!("BENCHMARK: SMART collector only");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some("none") => {
|
Some("none") => {
|
||||||
// No collectors - test agent loop only
|
// No collectors - test agent loop only
|
||||||
info!("BENCHMARK: No collectors enabled");
|
info!("BENCHMARK: No collectors enabled");
|
||||||
@@ -110,11 +102,6 @@ impl MetricCollectionManager {
|
|||||||
info!("NixOS collector initialized");
|
info!("NixOS collector initialized");
|
||||||
}
|
}
|
||||||
|
|
||||||
if config.smart.enabled {
|
|
||||||
let smart_collector = SmartCollector;
|
|
||||||
collectors.push(Box::new(smart_collector));
|
|
||||||
info!("SMART collector initialized");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -236,6 +236,13 @@ impl Dashboard {
|
|||||||
self.metric_store
|
self.metric_store
|
||||||
.update_metrics(&metric_message.hostname, metric_message.metrics);
|
.update_metrics(&metric_message.hostname, metric_message.metrics);
|
||||||
|
|
||||||
|
// Check for agent version mismatches across hosts
|
||||||
|
if let Some((current_version, outdated_hosts)) = self.metric_store.get_version_mismatches() {
|
||||||
|
for outdated_host in &outdated_hosts {
|
||||||
|
warn!("Host {} has outdated agent version (current: {})", outdated_host, current_version);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Update TUI with new hosts and metrics (only if not headless)
|
// Update TUI with new hosts and metrics (only if not headless)
|
||||||
if let Some(ref mut tui_app) = self.tui_app {
|
if let Some(ref mut tui_app) = self.tui_app {
|
||||||
let mut connected_hosts = self
|
let mut connected_hosts = self
|
||||||
@@ -261,6 +268,26 @@ impl Dashboard {
|
|||||||
tui_app.update_metrics(&self.metric_store);
|
tui_app.update_metrics(&self.metric_store);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Also check for command output messages
|
||||||
|
if let Ok(Some(cmd_output)) = self.zmq_consumer.receive_command_output().await {
|
||||||
|
debug!(
|
||||||
|
"Received command output from {}: {}",
|
||||||
|
cmd_output.hostname,
|
||||||
|
cmd_output.output_line
|
||||||
|
);
|
||||||
|
|
||||||
|
// Forward to TUI if not headless
|
||||||
|
if let Some(ref mut tui_app) = self.tui_app {
|
||||||
|
tui_app.add_terminal_output(&cmd_output.hostname, cmd_output.output_line);
|
||||||
|
|
||||||
|
// Close popup when command completes
|
||||||
|
if cmd_output.is_complete {
|
||||||
|
tui_app.close_terminal_popup(&cmd_output.hostname);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
last_metrics_check = Instant::now();
|
last_metrics_check = Instant::now();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use cm_dashboard_shared::{MessageEnvelope, MessageType, MetricMessage};
|
use cm_dashboard_shared::{CommandOutputMessage, MessageEnvelope, MessageType, MetricMessage};
|
||||||
use tracing::{debug, error, info, warn};
|
use tracing::{debug, error, info, warn};
|
||||||
use zmq::{Context, Socket, SocketType};
|
use zmq::{Context, Socket, SocketType};
|
||||||
|
|
||||||
@@ -103,6 +103,43 @@ impl ZmqConsumer {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Receive command output from any connected agent (non-blocking)
|
||||||
|
pub async fn receive_command_output(&mut self) -> Result<Option<CommandOutputMessage>> {
|
||||||
|
match self.subscriber.recv_bytes(zmq::DONTWAIT) {
|
||||||
|
Ok(data) => {
|
||||||
|
// Deserialize envelope
|
||||||
|
let envelope: MessageEnvelope = serde_json::from_slice(&data)
|
||||||
|
.map_err(|e| anyhow::anyhow!("Failed to deserialize envelope: {}", e))?;
|
||||||
|
|
||||||
|
// Check message type
|
||||||
|
match envelope.message_type {
|
||||||
|
MessageType::CommandOutput => {
|
||||||
|
let cmd_output = envelope
|
||||||
|
.decode_command_output()
|
||||||
|
.map_err(|e| anyhow::anyhow!("Failed to decode command output: {}", e))?;
|
||||||
|
|
||||||
|
debug!(
|
||||||
|
"Received command output from {}: {}",
|
||||||
|
cmd_output.hostname,
|
||||||
|
cmd_output.output_line
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(Some(cmd_output))
|
||||||
|
}
|
||||||
|
_ => Ok(None), // Not a command output message
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(zmq::Error::EAGAIN) => {
|
||||||
|
// No message available (non-blocking mode)
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("ZMQ receive error: {}", e);
|
||||||
|
Err(anyhow::anyhow!("ZMQ receive error: {}", e))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Receive metrics from any connected agent (non-blocking)
|
/// Receive metrics from any connected agent (non-blocking)
|
||||||
pub async fn receive_metrics(&mut self) -> Result<Option<MetricMessage>> {
|
pub async fn receive_metrics(&mut self) -> Result<Option<MetricMessage>> {
|
||||||
match self.subscriber.recv_bytes(zmq::DONTWAIT) {
|
match self.subscriber.recv_bytes(zmq::DONTWAIT) {
|
||||||
@@ -132,6 +169,10 @@ impl ZmqConsumer {
|
|||||||
debug!("Received heartbeat");
|
debug!("Received heartbeat");
|
||||||
Ok(None) // Don't return heartbeats as metrics
|
Ok(None) // Don't return heartbeats as metrics
|
||||||
}
|
}
|
||||||
|
MessageType::CommandOutput => {
|
||||||
|
debug!("Received command output (will be handled by receive_command_output)");
|
||||||
|
Ok(None) // Command output handled by separate method
|
||||||
|
}
|
||||||
_ => {
|
_ => {
|
||||||
debug!("Received non-metrics message: {:?}", envelope.message_type);
|
debug!("Received non-metrics message: {:?}", envelope.message_type);
|
||||||
Ok(None)
|
Ok(None)
|
||||||
|
|||||||
@@ -124,4 +124,52 @@ impl MetricStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Get agent versions from all hosts for cross-host comparison
|
||||||
|
pub fn get_agent_versions(&self) -> HashMap<String, String> {
|
||||||
|
let mut versions = HashMap::new();
|
||||||
|
|
||||||
|
for (hostname, metrics) in &self.current_metrics {
|
||||||
|
if let Some(version_metric) = metrics.get("agent_version") {
|
||||||
|
if let cm_dashboard_shared::MetricValue::String(version) = &version_metric.value {
|
||||||
|
versions.insert(hostname.clone(), version.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
versions
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check for agent version mismatches across hosts
|
||||||
|
pub fn get_version_mismatches(&self) -> Option<(String, Vec<String>)> {
|
||||||
|
let versions = self.get_agent_versions();
|
||||||
|
|
||||||
|
if versions.len() < 2 {
|
||||||
|
return None; // Need at least 2 hosts to compare
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find the most common version (assume it's the "current" version)
|
||||||
|
let mut version_counts = HashMap::new();
|
||||||
|
for version in versions.values() {
|
||||||
|
*version_counts.entry(version.clone()).or_insert(0) += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
let most_common_version = version_counts
|
||||||
|
.iter()
|
||||||
|
.max_by_key(|(_, count)| *count)
|
||||||
|
.map(|(version, _)| version.clone())?;
|
||||||
|
|
||||||
|
// Find hosts with different versions
|
||||||
|
let outdated_hosts: Vec<String> = versions
|
||||||
|
.iter()
|
||||||
|
.filter(|(_, version)| *version != &most_common_version)
|
||||||
|
.map(|(hostname, _)| hostname.clone())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
if outdated_hosts.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some((most_common_version, outdated_hosts))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -92,6 +92,51 @@ impl HostWidgets {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Terminal popup for streaming command output
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct TerminalPopup {
|
||||||
|
/// Is the popup currently visible
|
||||||
|
pub visible: bool,
|
||||||
|
/// Command being executed
|
||||||
|
pub command_type: CommandType,
|
||||||
|
/// Target hostname
|
||||||
|
pub hostname: String,
|
||||||
|
/// Target service/operation name
|
||||||
|
pub target: String,
|
||||||
|
/// Output lines collected so far
|
||||||
|
pub output_lines: Vec<String>,
|
||||||
|
/// Scroll offset for the output
|
||||||
|
pub scroll_offset: usize,
|
||||||
|
/// Start time of the operation
|
||||||
|
pub start_time: Instant,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TerminalPopup {
|
||||||
|
pub fn new(command_type: CommandType, hostname: String, target: String) -> Self {
|
||||||
|
Self {
|
||||||
|
visible: true,
|
||||||
|
command_type,
|
||||||
|
hostname,
|
||||||
|
target,
|
||||||
|
output_lines: Vec::new(),
|
||||||
|
scroll_offset: 0,
|
||||||
|
start_time: Instant::now(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn add_output_line(&mut self, line: String) {
|
||||||
|
self.output_lines.push(line);
|
||||||
|
// Auto-scroll to bottom when new content arrives
|
||||||
|
if self.output_lines.len() > 20 {
|
||||||
|
self.scroll_offset = self.output_lines.len().saturating_sub(20);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn close(&mut self) {
|
||||||
|
self.visible = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Main TUI application
|
/// Main TUI application
|
||||||
pub struct TuiApp {
|
pub struct TuiApp {
|
||||||
/// Widget states per host (hostname -> HostWidgets)
|
/// Widget states per host (hostname -> HostWidgets)
|
||||||
@@ -108,6 +153,8 @@ pub struct TuiApp {
|
|||||||
should_quit: bool,
|
should_quit: bool,
|
||||||
/// Track if user manually navigated away from localhost
|
/// Track if user manually navigated away from localhost
|
||||||
user_navigated_away: bool,
|
user_navigated_away: bool,
|
||||||
|
/// Terminal popup for streaming command output
|
||||||
|
terminal_popup: Option<TerminalPopup>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TuiApp {
|
impl TuiApp {
|
||||||
@@ -120,6 +167,7 @@ impl TuiApp {
|
|||||||
focused_panel: PanelType::System, // Start with System panel focused
|
focused_panel: PanelType::System, // Start with System panel focused
|
||||||
should_quit: false,
|
should_quit: false,
|
||||||
user_navigated_away: false,
|
user_navigated_away: false,
|
||||||
|
terminal_popup: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -250,6 +298,38 @@ impl TuiApp {
|
|||||||
/// Handle keyboard input
|
/// Handle keyboard input
|
||||||
pub fn handle_input(&mut self, event: Event) -> Result<Option<UiCommand>> {
|
pub fn handle_input(&mut self, event: Event) -> Result<Option<UiCommand>> {
|
||||||
if let Event::Key(key) = event {
|
if let Event::Key(key) = event {
|
||||||
|
// If terminal popup is visible, handle popup-specific keys first
|
||||||
|
if let Some(ref mut popup) = self.terminal_popup {
|
||||||
|
if popup.visible {
|
||||||
|
match key.code {
|
||||||
|
KeyCode::Esc => {
|
||||||
|
popup.close();
|
||||||
|
self.terminal_popup = None;
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
KeyCode::Up => {
|
||||||
|
popup.scroll_offset = popup.scroll_offset.saturating_sub(1);
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
KeyCode::Down => {
|
||||||
|
let max_scroll = if popup.output_lines.len() > 20 {
|
||||||
|
popup.output_lines.len() - 20
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
popup.scroll_offset = (popup.scroll_offset + 1).min(max_scroll);
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
KeyCode::Char('q') => {
|
||||||
|
popup.close();
|
||||||
|
self.terminal_popup = None;
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
_ => return Ok(None), // Consume all other keys when popup is open
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
match key.code {
|
match key.code {
|
||||||
KeyCode::Char('q') => {
|
KeyCode::Char('q') => {
|
||||||
self.should_quit = true;
|
self.should_quit = true;
|
||||||
@@ -266,6 +346,12 @@ impl TuiApp {
|
|||||||
// System rebuild command
|
// System rebuild command
|
||||||
if let Some(hostname) = self.current_host.clone() {
|
if let Some(hostname) = self.current_host.clone() {
|
||||||
self.start_command(&hostname, CommandType::SystemRebuild, hostname.clone());
|
self.start_command(&hostname, CommandType::SystemRebuild, hostname.clone());
|
||||||
|
// Open terminal popup for real-time output
|
||||||
|
self.terminal_popup = Some(TerminalPopup::new(
|
||||||
|
CommandType::SystemRebuild,
|
||||||
|
hostname.clone(),
|
||||||
|
"NixOS Rebuild".to_string()
|
||||||
|
));
|
||||||
return Ok(Some(UiCommand::SystemRebuild { hostname }));
|
return Ok(Some(UiCommand::SystemRebuild { hostname }));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -473,6 +559,25 @@ impl TuiApp {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Add output line to terminal popup
|
||||||
|
pub fn add_terminal_output(&mut self, hostname: &str, line: String) {
|
||||||
|
if let Some(ref mut popup) = self.terminal_popup {
|
||||||
|
if popup.hostname == hostname && popup.visible {
|
||||||
|
popup.add_output_line(line);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Close terminal popup for a specific hostname
|
||||||
|
pub fn close_terminal_popup(&mut self, hostname: &str) {
|
||||||
|
if let Some(ref mut popup) = self.terminal_popup {
|
||||||
|
if popup.hostname == hostname {
|
||||||
|
popup.close();
|
||||||
|
self.terminal_popup = None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Check for rebuild completion by detecting agent hash changes
|
/// Check for rebuild completion by detecting agent hash changes
|
||||||
pub fn check_rebuild_completion(&mut self, metric_store: &MetricStore) {
|
pub fn check_rebuild_completion(&mut self, metric_store: &MetricStore) {
|
||||||
let mut hosts_to_complete = Vec::new();
|
let mut hosts_to_complete = Vec::new();
|
||||||
@@ -636,6 +741,13 @@ impl TuiApp {
|
|||||||
|
|
||||||
// Render statusbar at the bottom
|
// Render statusbar at the bottom
|
||||||
self.render_statusbar(frame, main_chunks[2]); // main_chunks[2] is the statusbar area
|
self.render_statusbar(frame, main_chunks[2]); // main_chunks[2] is the statusbar area
|
||||||
|
|
||||||
|
// Render terminal popup on top of everything else
|
||||||
|
if let Some(ref popup) = self.terminal_popup {
|
||||||
|
if popup.visible {
|
||||||
|
self.render_terminal_popup(frame, size, popup);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Render btop-style minimal title with host status colors
|
/// Render btop-style minimal title with host status colors
|
||||||
@@ -835,4 +947,112 @@ impl TuiApp {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Render terminal popup with streaming output
|
||||||
|
fn render_terminal_popup(&self, frame: &mut Frame, area: Rect, popup: &TerminalPopup) {
|
||||||
|
use ratatui::{
|
||||||
|
style::{Color, Modifier, Style},
|
||||||
|
text::{Line, Span},
|
||||||
|
widgets::{Block, Borders, Clear, Paragraph, Wrap},
|
||||||
|
};
|
||||||
|
|
||||||
|
// Calculate popup size (80% of screen, centered)
|
||||||
|
let popup_width = area.width * 80 / 100;
|
||||||
|
let popup_height = area.height * 80 / 100;
|
||||||
|
let popup_x = (area.width - popup_width) / 2;
|
||||||
|
let popup_y = (area.height - popup_height) / 2;
|
||||||
|
|
||||||
|
let popup_area = Rect {
|
||||||
|
x: popup_x,
|
||||||
|
y: popup_y,
|
||||||
|
width: popup_width,
|
||||||
|
height: popup_height,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Clear background
|
||||||
|
frame.render_widget(Clear, popup_area);
|
||||||
|
|
||||||
|
// Create terminal-style block
|
||||||
|
let title = format!(" {} → {} ({:.1}s) ",
|
||||||
|
popup.hostname,
|
||||||
|
popup.target,
|
||||||
|
popup.start_time.elapsed().as_secs_f32()
|
||||||
|
);
|
||||||
|
|
||||||
|
let block = Block::default()
|
||||||
|
.title(title)
|
||||||
|
.borders(Borders::ALL)
|
||||||
|
.border_style(Style::default().fg(Color::Cyan))
|
||||||
|
.style(Style::default().bg(Color::Black));
|
||||||
|
|
||||||
|
let inner_area = block.inner(popup_area);
|
||||||
|
frame.render_widget(block, popup_area);
|
||||||
|
|
||||||
|
// Render output content
|
||||||
|
let available_height = inner_area.height as usize;
|
||||||
|
let total_lines = popup.output_lines.len();
|
||||||
|
|
||||||
|
// Calculate which lines to show based on scroll offset
|
||||||
|
let start_line = popup.scroll_offset;
|
||||||
|
let end_line = (start_line + available_height).min(total_lines);
|
||||||
|
|
||||||
|
let visible_lines: Vec<Line> = popup.output_lines[start_line..end_line]
|
||||||
|
.iter()
|
||||||
|
.map(|line| {
|
||||||
|
// Style output lines with terminal colors
|
||||||
|
if line.contains("error") || line.contains("Error") || line.contains("failed") {
|
||||||
|
Line::from(Span::styled(line.clone(), Style::default().fg(Color::Red)))
|
||||||
|
} else if line.contains("warning") || line.contains("Warning") {
|
||||||
|
Line::from(Span::styled(line.clone(), Style::default().fg(Color::Yellow)))
|
||||||
|
} else if line.contains("building") || line.contains("Building") {
|
||||||
|
Line::from(Span::styled(line.clone(), Style::default().fg(Color::Blue)))
|
||||||
|
} else if line.contains("✓") || line.contains("success") || line.contains("completed") {
|
||||||
|
Line::from(Span::styled(line.clone(), Style::default().fg(Color::Green)))
|
||||||
|
} else {
|
||||||
|
Line::from(Span::styled(line.clone(), Style::default().fg(Color::White)))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let content = Paragraph::new(visible_lines)
|
||||||
|
.wrap(Wrap { trim: false })
|
||||||
|
.style(Style::default().bg(Color::Black));
|
||||||
|
|
||||||
|
frame.render_widget(content, inner_area);
|
||||||
|
|
||||||
|
// Render scroll indicator if needed
|
||||||
|
if total_lines > available_height {
|
||||||
|
let scroll_info = format!(" {}% ",
|
||||||
|
if total_lines > 0 {
|
||||||
|
(end_line * 100) / total_lines
|
||||||
|
} else {
|
||||||
|
100
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
let scroll_area = Rect {
|
||||||
|
x: popup_area.x + popup_area.width - scroll_info.len() as u16 - 1,
|
||||||
|
y: popup_area.y + popup_area.height - 1,
|
||||||
|
width: scroll_info.len() as u16,
|
||||||
|
height: 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
let scroll_widget = Paragraph::new(scroll_info)
|
||||||
|
.style(Style::default().fg(Color::Cyan).bg(Color::Black));
|
||||||
|
frame.render_widget(scroll_widget, scroll_area);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Instructions at bottom
|
||||||
|
let instructions = " ESC/Q: Close • ↑↓: Scroll ";
|
||||||
|
let instructions_area = Rect {
|
||||||
|
x: popup_area.x + 1,
|
||||||
|
y: popup_area.y + popup_area.height - 1,
|
||||||
|
width: instructions.len() as u16,
|
||||||
|
height: 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
let instructions_widget = Paragraph::new(instructions)
|
||||||
|
.style(Style::default().fg(Color::Gray).bg(Color::Black));
|
||||||
|
frame.render_widget(instructions_widget, instructions_area);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -339,9 +339,9 @@ impl Widget for SystemWidget {
|
|||||||
self.active_users = Some(users.clone());
|
self.active_users = Some(users.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
"system_agent_hash" => {
|
"agent_version" => {
|
||||||
if let MetricValue::String(hash) = &metric.value {
|
if let MetricValue::String(version) = &metric.value {
|
||||||
self.agent_hash = Some(hash.clone());
|
self.agent_hash = Some(version.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -422,9 +422,19 @@ impl SystemWidget {
|
|||||||
Span::styled("NixOS:", Typography::widget_title())
|
Span::styled("NixOS:", Typography::widget_title())
|
||||||
]));
|
]));
|
||||||
|
|
||||||
|
let build_text = self.nixos_build.as_deref().unwrap_or("unknown");
|
||||||
|
lines.push(Line::from(vec![
|
||||||
|
Span::styled(format!("Build: {}", build_text), Typography::secondary())
|
||||||
|
]));
|
||||||
|
|
||||||
let config_text = self.config_hash.as_deref().unwrap_or("unknown");
|
let config_text = self.config_hash.as_deref().unwrap_or("unknown");
|
||||||
lines.push(Line::from(vec![
|
lines.push(Line::from(vec![
|
||||||
Span::styled(format!("Build: {}", config_text), Typography::secondary())
|
Span::styled(format!("Config: {}", config_text), Typography::secondary())
|
||||||
|
]));
|
||||||
|
|
||||||
|
let users_text = self.active_users.as_deref().unwrap_or("unknown");
|
||||||
|
lines.push(Line::from(vec![
|
||||||
|
Span::styled(format!("Active users: {}", users_text), Typography::secondary())
|
||||||
]));
|
]));
|
||||||
|
|
||||||
let agent_hash_text = self.agent_hash.as_deref().unwrap_or("unknown");
|
let agent_hash_text = self.agent_hash.as_deref().unwrap_or("unknown");
|
||||||
|
|||||||
@@ -9,6 +9,17 @@ pub struct MetricMessage {
|
|||||||
pub metrics: Vec<Metric>,
|
pub metrics: Vec<Metric>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Command output streaming message
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct CommandOutputMessage {
|
||||||
|
pub hostname: String,
|
||||||
|
pub command_id: String,
|
||||||
|
pub command_type: String,
|
||||||
|
pub output_line: String,
|
||||||
|
pub is_complete: bool,
|
||||||
|
pub timestamp: u64,
|
||||||
|
}
|
||||||
|
|
||||||
impl MetricMessage {
|
impl MetricMessage {
|
||||||
pub fn new(hostname: String, metrics: Vec<Metric>) -> Self {
|
pub fn new(hostname: String, metrics: Vec<Metric>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
@@ -19,6 +30,19 @@ impl MetricMessage {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl CommandOutputMessage {
|
||||||
|
pub fn new(hostname: String, command_id: String, command_type: String, output_line: String, is_complete: bool) -> Self {
|
||||||
|
Self {
|
||||||
|
hostname,
|
||||||
|
command_id,
|
||||||
|
command_type,
|
||||||
|
output_line,
|
||||||
|
is_complete,
|
||||||
|
timestamp: chrono::Utc::now().timestamp() as u64,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Commands that can be sent from dashboard to agent
|
/// Commands that can be sent from dashboard to agent
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
pub enum Command {
|
pub enum Command {
|
||||||
@@ -55,6 +79,7 @@ pub enum MessageType {
|
|||||||
Metrics,
|
Metrics,
|
||||||
Command,
|
Command,
|
||||||
CommandResponse,
|
CommandResponse,
|
||||||
|
CommandOutput,
|
||||||
Heartbeat,
|
Heartbeat,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -80,6 +105,13 @@ impl MessageEnvelope {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn command_output(message: CommandOutputMessage) -> Result<Self, crate::SharedError> {
|
||||||
|
Ok(Self {
|
||||||
|
message_type: MessageType::CommandOutput,
|
||||||
|
payload: serde_json::to_vec(&message)?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn heartbeat() -> Result<Self, crate::SharedError> {
|
pub fn heartbeat() -> Result<Self, crate::SharedError> {
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
message_type: MessageType::Heartbeat,
|
message_type: MessageType::Heartbeat,
|
||||||
@@ -113,4 +145,13 @@ impl MessageEnvelope {
|
|||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn decode_command_output(&self) -> Result<CommandOutputMessage, crate::SharedError> {
|
||||||
|
match self.message_type {
|
||||||
|
MessageType::CommandOutput => Ok(serde_json::from_slice(&self.payload)?),
|
||||||
|
_ => Err(crate::SharedError::Protocol {
|
||||||
|
message: "Expected command output message".to_string(),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user