Implement comprehensive backup monitoring and fix timestamp issues

- Add BackupCollector for reading TOML status files with disk space metrics
- Implement BackupWidget with disk usage display and service status details
- Fix backup script disk space parsing by adding missing capture_output=True
- Update backup widget to show actual disk usage instead of repository size
- Fix timestamp parsing to use backup completion time instead of start time
- Resolve timezone issues by using UTC timestamps in backup script
- Add disk identification metrics (product name, serial number) to backup status
- Enhance UI layout with proper backup monitoring integration
This commit is contained in:
2025-10-18 18:33:41 +02:00
parent 8a36472a3d
commit 125111ee99
19 changed files with 2788 additions and 1020 deletions

View File

@@ -1,20 +1,22 @@
use cm_dashboard_shared::{Metric, Status};
use ratatui::{
layout::{Constraint, Direction, Layout, Rect},
style::{Color, Style},
widgets::{Block, Borders, List, ListItem, Paragraph},
widgets::Paragraph,
Frame,
};
use std::collections::HashMap;
use tracing::debug;
use super::Widget;
use crate::ui::theme::Theme;
use crate::ui::theme::{Theme, Typography, Components, StatusIcons};
use ratatui::style::Style;
/// Services widget displaying individual systemd service statuses
/// Services widget displaying hierarchical systemd service statuses
pub struct ServicesWidget {
/// Individual service statuses
services: HashMap<String, ServiceInfo>,
/// Parent services (nginx, docker, etc.)
parent_services: HashMap<String, ServiceInfo>,
/// Sub-services grouped by parent (nginx -> [gitea, mariehall, ...], docker -> [container1, ...])
sub_services: HashMap<String, Vec<(String, ServiceInfo)>>,
/// Aggregated status
status: Status,
/// Last update indicator
@@ -26,86 +28,183 @@ struct ServiceInfo {
status: String,
memory_mb: Option<f32>,
disk_gb: Option<f32>,
latency_ms: Option<f32>,
widget_status: Status,
}
impl ServicesWidget {
pub fn new() -> Self {
Self {
services: HashMap::new(),
parent_services: HashMap::new(),
sub_services: HashMap::new(),
status: Status::Unknown,
has_data: false,
}
}
/// Get status color for display (btop-style)
fn get_status_color(&self) -> Color {
Theme::status_color(self.status)
}
/// Extract service name from metric name
fn extract_service_name(metric_name: &str) -> Option<String> {
/// Extract service name and determine if it's a parent or sub-service
fn extract_service_info(metric_name: &str) -> Option<(String, Option<String>)> {
if metric_name.starts_with("service_") {
if let Some(end_pos) = metric_name.rfind("_status")
.or_else(|| metric_name.rfind("_memory_mb"))
.or_else(|| metric_name.rfind("_disk_gb")) {
let service_name = &metric_name[8..end_pos]; // Remove "service_" prefix
return Some(service_name.to_string());
.or_else(|| metric_name.rfind("_disk_gb"))
.or_else(|| metric_name.rfind("_latency_ms")) {
let service_part = &metric_name[8..end_pos]; // Remove "service_" prefix
// Check for sub-services patterns
if service_part.starts_with("nginx_") {
// nginx sub-services: service_nginx_gitea_latency_ms -> ("nginx", "gitea")
let sub_service = service_part.strip_prefix("nginx_").unwrap_or(service_part);
return Some(("nginx".to_string(), Some(sub_service.to_string())));
} else if service_part.starts_with("docker_") {
// docker sub-services: service_docker_container1_status -> ("docker", "container1")
let sub_service = service_part.strip_prefix("docker_").unwrap_or(service_part);
return Some(("docker".to_string(), Some(sub_service.to_string())));
} else {
// Regular parent service: service_nginx_status -> ("nginx", None)
return Some((service_part.to_string(), None));
}
}
}
None
}
/// Format service info for display
fn format_service_info(&self, name: &str, info: &ServiceInfo) -> String {
let status_icon = match info.widget_status {
Status::Ok => "",
Status::Warning => "⚠️",
Status::Critical => "",
Status::Unknown => "",
};
/// Format disk size with appropriate units (kB/MB/GB)
fn format_disk_size(size_gb: f32) -> String {
let size_mb = size_gb * 1024.0; // Convert GB to MB
let memory_str = if let Some(memory) = info.memory_mb {
format!(" Mem:{:.1}MB", memory)
if size_mb >= 1024.0 {
// Show as GB
format!("{:.1}GB", size_gb)
} else if size_mb >= 1.0 {
// Show as MB
format!("{:.0}MB", size_mb)
} else if size_mb >= 0.001 {
// Convert to kB
let size_kb = size_mb * 1024.0;
format!("{:.0}kB", size_kb)
} else {
"".to_string()
};
let disk_str = if let Some(disk) = info.disk_gb {
format!(" Disk:{:.1}GB", disk)
} else {
"".to_string()
};
format!("{} {} ({}){}{}", status_icon, name, info.status, memory_str, disk_str)
// Show very small sizes as bytes
let size_bytes = size_mb * 1024.0 * 1024.0;
format!("{:.0}B", size_bytes)
}
}
/// Format service info in clean service list format
fn format_btop_process_line(&self, name: &str, info: &ServiceInfo, _index: usize) -> String {
/// Format parent service line - returns text without icon for span formatting
fn format_parent_service_line(&self, name: &str, info: &ServiceInfo) -> String {
let memory_str = info.memory_mb.map_or("0M".to_string(), |m| format!("{:.0}M", m));
let disk_str = info.disk_gb.map_or("0G".to_string(), |d| format!("{:.1}G", d));
let disk_str = info.disk_gb.map_or("0".to_string(), |d| Self::format_disk_size(d));
// Truncate long service names to fit layout
let short_name = if name.len() > 23 {
format!("{}...", &name[..20])
// Truncate long service names to fit layout (account for icon space)
let short_name = if name.len() > 22 {
format!("{}...", &name[..19])
} else {
name.to_string()
};
// Status with color indicator
// Parent services always show active/inactive status
let status_str = match info.widget_status {
Status::Ok => "active",
Status::Warning => "⚠️ inactive",
Status::Critical => "failed",
Status::Unknown => "unknown",
Status::Ok => "active".to_string(),
Status::Warning => "inactive".to_string(),
Status::Critical => "failed".to_string(),
Status::Unknown => "unknown".to_string(),
};
format!("{:<25} {:<10} {:<8} {:<8}",
format!("{:<24} {:<10} {:<8} {:<8}",
short_name,
status_str,
memory_str,
disk_str)
}
/// Format sub-service line (indented, no memory/disk columns) - returns text without icon for span formatting
fn format_sub_service_line(&self, name: &str, info: &ServiceInfo) -> String {
// Truncate long sub-service names to fit layout (accounting for indentation)
let short_name = if name.len() > 18 {
format!("{}...", &name[..15])
} else {
name.to_string()
};
// Sub-services show latency if available, otherwise status
let status_str = if let Some(latency) = info.latency_ms {
if latency < 0.0 {
"timeout".to_string()
} else {
format!("{:.0}ms", latency)
}
} else {
match info.widget_status {
Status::Ok => "active".to_string(),
Status::Warning => "inactive".to_string(),
Status::Critical => "failed".to_string(),
Status::Unknown => "unknown".to_string(),
}
};
// Indent sub-services with " ├─ " prefix (no memory/disk columns)
format!(" ├─ {:<20} {:<10}",
short_name,
status_str)
}
/// Create spans for sub-service with icon next to name
fn create_sub_service_spans(&self, name: &str, info: &ServiceInfo) -> Vec<ratatui::text::Span<'static>> {
// Truncate long sub-service names to fit layout (accounting for indentation)
let short_name = if name.len() > 18 {
format!("{}...", &name[..15])
} else {
name.to_string()
};
// Sub-services show latency if available, otherwise status
let status_str = if let Some(latency) = info.latency_ms {
if latency < 0.0 {
"timeout".to_string()
} else {
format!("{:.0}ms", latency)
}
} else {
match info.widget_status {
Status::Ok => "active".to_string(),
Status::Warning => "inactive".to_string(),
Status::Critical => "failed".to_string(),
Status::Unknown => "unknown".to_string(),
}
};
let status_color = match info.widget_status {
Status::Ok => Theme::success(),
Status::Warning => Theme::warning(),
Status::Critical => Theme::error(),
Status::Unknown => Theme::muted_text(),
};
let icon = StatusIcons::get_icon(info.widget_status);
vec![
// Indentation and tree prefix
ratatui::text::Span::styled(
" ├─ ".to_string(),
Style::default().fg(Theme::secondary_text()).bg(Theme::background())
),
// Status icon
ratatui::text::Span::styled(
format!("{} ", icon),
Style::default().fg(status_color).bg(Theme::background())
),
// Service name
ratatui::text::Span::styled(
format!("{:<18} ", short_name),
Style::default().fg(Theme::secondary_text()).bg(Theme::background())
),
// Status/latency text
ratatui::text::Span::styled(
status_str,
Style::default().fg(Theme::secondary_text()).bg(Theme::background())
),
]
}
}
impl Widget for ServicesWidget {
@@ -116,73 +215,180 @@ impl Widget for ServicesWidget {
// Process individual service metrics
for metric in metrics {
if let Some(service_name) = Self::extract_service_name(&metric.name) {
let service_info = self.services.entry(service_name).or_insert(ServiceInfo {
status: "unknown".to_string(),
memory_mb: None,
disk_gb: None,
widget_status: Status::Unknown,
});
if metric.name.ends_with("_status") {
service_info.status = metric.value.as_string();
service_info.widget_status = metric.status;
} else if metric.name.ends_with("_memory_mb") {
if let Some(memory) = metric.value.as_f32() {
service_info.memory_mb = Some(memory);
if let Some((parent_service, sub_service)) = Self::extract_service_info(&metric.name) {
match sub_service {
None => {
// Parent service metric
let service_info = self.parent_services.entry(parent_service).or_insert(ServiceInfo {
status: "unknown".to_string(),
memory_mb: None,
disk_gb: None,
latency_ms: None,
widget_status: Status::Unknown,
});
if metric.name.ends_with("_status") {
service_info.status = metric.value.as_string();
service_info.widget_status = metric.status;
} else if metric.name.ends_with("_memory_mb") {
if let Some(memory) = metric.value.as_f32() {
service_info.memory_mb = Some(memory);
}
} else if metric.name.ends_with("_disk_gb") {
if let Some(disk) = metric.value.as_f32() {
service_info.disk_gb = Some(disk);
}
}
}
} else if metric.name.ends_with("_disk_gb") {
if let Some(disk) = metric.value.as_f32() {
service_info.disk_gb = Some(disk);
Some(sub_name) => {
// Sub-service metric
let sub_service_list = self.sub_services.entry(parent_service).or_insert_with(Vec::new);
// Find existing sub-service or create new one
let sub_service_info = if let Some(pos) = sub_service_list.iter().position(|(name, _)| name == &sub_name) {
&mut sub_service_list[pos].1
} else {
sub_service_list.push((sub_name.clone(), ServiceInfo {
status: "unknown".to_string(),
memory_mb: None,
disk_gb: None,
latency_ms: None,
widget_status: Status::Unknown,
}));
&mut sub_service_list.last_mut().unwrap().1
};
if metric.name.ends_with("_status") {
sub_service_info.status = metric.value.as_string();
sub_service_info.widget_status = metric.status;
} else if metric.name.ends_with("_memory_mb") {
if let Some(memory) = metric.value.as_f32() {
sub_service_info.memory_mb = Some(memory);
}
} else if metric.name.ends_with("_disk_gb") {
if let Some(disk) = metric.value.as_f32() {
sub_service_info.disk_gb = Some(disk);
}
} else if metric.name.ends_with("_latency_ms") {
if let Some(latency) = metric.value.as_f32() {
sub_service_info.latency_ms = Some(latency);
sub_service_info.widget_status = metric.status;
}
}
}
}
}
}
// Aggregate status from all services
let statuses: Vec<Status> = self.services.values()
.map(|info| info.widget_status)
.collect();
// Aggregate status from all parent and sub-services
let mut all_statuses = Vec::new();
self.status = if statuses.is_empty() {
// Add parent service statuses
all_statuses.extend(self.parent_services.values().map(|info| info.widget_status));
// Add sub-service statuses
for sub_list in self.sub_services.values() {
all_statuses.extend(sub_list.iter().map(|(_, info)| info.widget_status));
}
self.status = if all_statuses.is_empty() {
Status::Unknown
} else {
Status::aggregate(&statuses)
Status::aggregate(&all_statuses)
};
self.has_data = !self.services.is_empty();
self.has_data = !self.parent_services.is_empty() || !self.sub_services.is_empty();
debug!("Services widget updated: {} services, status={:?}",
self.services.len(), self.status);
debug!("Services widget updated: {} parent services, {} sub-service groups, status={:?}",
self.parent_services.len(), self.sub_services.len(), self.status);
}
fn render(&mut self, frame: &mut Frame, area: Rect) {
let services_block = Block::default().title("services").borders(Borders::ALL).style(Style::default().fg(Theme::border()).bg(Theme::background())).title_style(Style::default().fg(Theme::primary_text()));
let services_block = Components::widget_block("services");
let inner_area = services_block.inner(area);
frame.render_widget(services_block, area);
let content_chunks = Layout::default().direction(Direction::Vertical).constraints([Constraint::Length(1), Constraint::Min(0)]).split(inner_area);
let header = format!("{:<25} {:<10} {:<8} {:<8}", "Service:", "Status:", "MemB", "DiskGB");
let header_para = Paragraph::new(header).style(Style::default().fg(Theme::muted_text()).bg(Theme::background()));
let content_chunks = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Length(1), Constraint::Min(0)])
.split(inner_area);
// Header
let header = format!("{:<25} {:<10} {:<8} {:<8}", "Service:", "Status:", "RAM:", "Disk:");
let header_para = Paragraph::new(header).style(Typography::muted());
frame.render_widget(header_para, content_chunks[0]);
if self.services.is_empty() { let empty_text = Paragraph::new("No process data").style(Style::default().fg(Theme::muted_text()).bg(Theme::background())); frame.render_widget(empty_text, content_chunks[1]); return; }
let mut services: Vec<_> = self.services.iter().collect();
services.sort_by(|(_, a), (_, b)| b.memory_mb.unwrap_or(0.0).partial_cmp(&a.memory_mb.unwrap_or(0.0)).unwrap_or(std::cmp::Ordering::Equal));
let available_lines = content_chunks[1].height as usize;
let service_chunks = Layout::default().direction(Direction::Vertical).constraints(vec![Constraint::Length(1); available_lines.min(services.len())]).split(content_chunks[1]);
for (i, (name, info)) in services.iter().take(available_lines).enumerate() {
let service_line = self.format_btop_process_line(name, info, i);
let color = match info.widget_status { Status::Ok => Theme::primary_text(), Status::Warning => Theme::warning(), Status::Critical => Theme::error(), Status::Unknown => Theme::muted_text(), };
let service_para = Paragraph::new(service_line).style(Style::default().fg(color).bg(Theme::background()));
frame.render_widget(service_para, service_chunks[i]);
// Check if we have any services to display
if self.parent_services.is_empty() && self.sub_services.is_empty() {
let empty_text = Paragraph::new("No process data").style(Typography::muted());
frame.render_widget(empty_text, content_chunks[1]);
return;
}
// Build hierarchical service list for display
let mut display_lines = Vec::new();
// Sort parent services alphabetically for consistent order
let mut parent_services: Vec<_> = self.parent_services.iter().collect();
parent_services.sort_by(|(a, _), (b, _)| a.cmp(b));
for (parent_name, parent_info) in parent_services {
// Add parent service line
let parent_line = self.format_parent_service_line(parent_name, parent_info);
display_lines.push((parent_line, parent_info.widget_status, false, None)); // false = not sub-service
// Add sub-services for this parent (if any)
if let Some(sub_list) = self.sub_services.get(parent_name) {
// Sort sub-services by name for consistent display
let mut sorted_subs = sub_list.clone();
sorted_subs.sort_by(|(a, _), (b, _)| a.cmp(b));
for (sub_name, sub_info) in sorted_subs {
// Store sub-service info for custom span rendering
display_lines.push((sub_name.clone(), sub_info.widget_status, true, Some(sub_info.clone()))); // true = sub-service
}
}
}
// Render all lines within available space
let available_lines = content_chunks[1].height as usize;
let lines_to_show = available_lines.min(display_lines.len());
if lines_to_show > 0 {
let service_chunks = Layout::default()
.direction(Direction::Vertical)
.constraints(vec![Constraint::Length(1); lines_to_show])
.split(content_chunks[1]);
for (i, (line_text, line_status, is_sub, sub_info)) in display_lines.iter().take(lines_to_show).enumerate() {
let spans = if *is_sub && sub_info.is_some() {
// Use custom sub-service span creation
self.create_sub_service_spans(line_text, sub_info.as_ref().unwrap())
} else {
// Use regular status spans for parent services
StatusIcons::create_status_spans(*line_status, line_text)
};
let service_para = Paragraph::new(ratatui::text::Line::from(spans));
frame.render_widget(service_para, service_chunks[i]);
}
}
// Show indicator if there are more services than we can display
if display_lines.len() > available_lines {
let more_count = display_lines.len() - available_lines;
if available_lines > 0 {
let last_line_area = Rect {
x: content_chunks[1].x,
y: content_chunks[1].y + (available_lines - 1) as u16,
width: content_chunks[1].width,
height: 1,
};
let more_text = format!("... and {} more services", more_count);
let more_para = Paragraph::new(more_text).style(Typography::muted());
frame.render_widget(more_para, last_line_area);
}
}
}
fn get_name(&self) -> &str {
"Services"
}
fn has_data(&self) -> bool {
self.has_data
}
}