Implement per-service disk usage monitoring

Replaced system-wide disk usage with accurate per-service tracking by scanning
service-specific directories. Services like sshd now correctly show minimal
disk usage instead of misleading system totals.

- Rename storage widget and add drive capacity/usage columns
- Move host display to main dashboard title for cleaner layout
- Replace separate alert displays with color-coded row highlighting
- Add per-service disk usage collection using du command
- Update services widget formatting to handle small disk values
- Restructure into workspace with dedicated agent and dashboard packages

agent/src/collectors/backup.rs

@@ -0,0 +1,388 @@
+use async_trait::async_trait;
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use serde_json::json;
+use std::process::Stdio;
+use std::time::Duration;
+use tokio::process::Command;
+use tokio::time::timeout;
+
+use super::{AgentType, Collector, CollectorError, CollectorOutput};
+
+#[derive(Debug, Clone)]
+pub struct BackupCollector {
+    pub enabled: bool,
+    pub interval: Duration,
+    pub restic_repo: Option<String>,
+    pub backup_service: String,
+    pub timeout_ms: u64,
+}
+
+impl BackupCollector {
+    pub fn new(
+        enabled: bool,
+        interval_ms: u64,
+        restic_repo: Option<String>,
+        backup_service: String,
+    ) -> Self {
+        Self {
+            enabled,
+            interval: Duration::from_millis(interval_ms),
+            restic_repo,
+            backup_service,
+            timeout_ms: 30000, // 30 second timeout for backup operations
+        }
+    }
+
+    async fn get_restic_snapshots(&self) -> Result<ResticStats, CollectorError> {
+        let repo = self
+            .restic_repo
+            .as_ref()
+            .ok_or_else(|| CollectorError::ConfigError {
+                message: "No restic repository configured".to_string(),
+            })?;
+
+        let timeout_duration = Duration::from_millis(self.timeout_ms);
+
+        // Get restic snapshots
+        let output = timeout(
+            timeout_duration,
+            Command::new("restic")
+                .args(["-r", repo, "snapshots", "--json"])
+                .stdout(Stdio::piped())
+                .stderr(Stdio::piped())
+                .output(),
+        )
+        .await
+        .map_err(|_| CollectorError::Timeout {
+            duration_ms: self.timeout_ms,
+        })?
+        .map_err(|e| CollectorError::CommandFailed {
+            command: format!("restic -r {} snapshots --json", repo),
+            message: e.to_string(),
+        })?;
+
+        if !output.status.success() {
+            let stderr = String::from_utf8_lossy(&output.stderr);
+            return Err(CollectorError::CommandFailed {
+                command: format!("restic -r {} snapshots --json", repo),
+                message: stderr.to_string(),
+            });
+        }
+
+        let stdout = String::from_utf8_lossy(&output.stdout);
+        let snapshots: Vec<ResticSnapshot> =
+            serde_json::from_str(&stdout).map_err(|e| CollectorError::ParseError {
+                message: format!("Failed to parse restic snapshots: {}", e),
+            })?;
+
+        // Get repository stats
+        let stats_output = timeout(
+            timeout_duration,
+            Command::new("restic")
+                .args(["-r", repo, "stats", "--json"])
+                .stdout(Stdio::piped())
+                .stderr(Stdio::piped())
+                .output(),
+        )
+        .await
+        .map_err(|_| CollectorError::Timeout {
+            duration_ms: self.timeout_ms,
+        })?
+        .map_err(|e| CollectorError::CommandFailed {
+            command: format!("restic -r {} stats --json", repo),
+            message: e.to_string(),
+        })?;
+
+        let repo_size_gb = if stats_output.status.success() {
+            let stats_stdout = String::from_utf8_lossy(&stats_output.stdout);
+            let stats: Result<ResticStats, _> = serde_json::from_str(&stats_stdout);
+            stats
+                .ok()
+                .map(|s| s.total_size as f32 / (1024.0 * 1024.0 * 1024.0))
+                .unwrap_or(0.0)
+        } else {
+            0.0
+        };
+
+        // Find most recent snapshot
+        let last_success = snapshots.iter().map(|s| s.time).max();
+
+        Ok(ResticStats {
+            total_size: (repo_size_gb * 1024.0 * 1024.0 * 1024.0) as u64,
+            snapshot_count: snapshots.len() as u32,
+            last_success,
+        })
+    }
+
+    async fn get_backup_service_status(&self) -> Result<BackupServiceData, CollectorError> {
+        let timeout_duration = Duration::from_millis(self.timeout_ms);
+
+        // Get systemctl status for backup service
+        let status_output = timeout(
+            timeout_duration,
+            Command::new("systemctl")
+                .args([
+                    "show",
+                    &self.backup_service,
+                    "--property=ActiveState,SubState,MainPID",
+                ])
+                .stdout(Stdio::piped())
+                .stderr(Stdio::piped())
+                .output(),
+        )
+        .await
+        .map_err(|_| CollectorError::Timeout {
+            duration_ms: self.timeout_ms,
+        })?
+        .map_err(|e| CollectorError::CommandFailed {
+            command: format!("systemctl show {}", self.backup_service),
+            message: e.to_string(),
+        })?;
+
+        let enabled = if status_output.status.success() {
+            let status_stdout = String::from_utf8_lossy(&status_output.stdout);
+            status_stdout.contains("ActiveState=active")
+                || status_stdout.contains("SubState=running")
+        } else {
+            false
+        };
+
+        // Check for backup timer or service logs for last message
+        let last_message = self.get_last_backup_log_message().await.ok();
+
+        // Check for pending backup jobs (simplified - could check systemd timers)
+        let pending_jobs = 0; // TODO: Implement proper pending job detection
+
+        Ok(BackupServiceData {
+            enabled,
+            pending_jobs,
+            last_message,
+        })
+    }
+
+    async fn get_last_backup_log_message(&self) -> Result<String, CollectorError> {
+        let output = Command::new("journalctl")
+            .args([
+                "-u",
+                &self.backup_service,
+                "--lines=1",
+                "--no-pager",
+                "--output=cat",
+            ])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .output()
+            .await
+            .map_err(|e| CollectorError::CommandFailed {
+                command: format!("journalctl -u {} --lines=1", self.backup_service),
+                message: e.to_string(),
+            })?;
+
+        if output.status.success() {
+            let stdout = String::from_utf8_lossy(&output.stdout);
+            let message = stdout.trim().to_string();
+            if !message.is_empty() {
+                return Ok(message);
+            }
+        }
+
+        Err(CollectorError::ParseError {
+            message: "No log messages found".to_string(),
+        })
+    }
+
+    async fn get_backup_logs_for_failures(&self) -> Result<Option<DateTime<Utc>>, CollectorError> {
+        let output = Command::new("journalctl")
+            .args([
+                "-u",
+                &self.backup_service,
+                "--since",
+                "1 week ago",
+                "--grep=failed\\|error\\|ERROR",
+                "--output=json",
+                "--lines=1",
+            ])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .output()
+            .await
+            .map_err(|e| CollectorError::CommandFailed {
+                command: format!(
+                    "journalctl -u {} --since='1 week ago' --grep=failed",
+                    self.backup_service
+                ),
+                message: e.to_string(),
+            })?;
+
+        if output.status.success() {
+            let stdout = String::from_utf8_lossy(&output.stdout);
+            if let Ok(log_entry) = serde_json::from_str::<JournalEntry>(&stdout) {
+                if let Ok(timestamp) = log_entry.realtime_timestamp.parse::<i64>() {
+                    let dt =
+                        DateTime::from_timestamp_micros(timestamp).unwrap_or_else(|| Utc::now());
+                    return Ok(Some(dt));
+                }
+            }
+        }
+
+        Ok(None)
+    }
+
+    fn determine_backup_status(
+        &self,
+        restic_stats: &Result<ResticStats, CollectorError>,
+        service_data: &BackupServiceData,
+        last_failure: Option<DateTime<Utc>>,
+    ) -> BackupStatus {
+        match restic_stats {
+            Ok(stats) => {
+                if let Some(last_success) = stats.last_success {
+                    let hours_since_backup =
+                        Utc::now().signed_duration_since(last_success).num_hours();
+
+                    if hours_since_backup > 48 {
+                        BackupStatus::Warning // More than 2 days since last backup
+                    } else if let Some(failure) = last_failure {
+                        if failure > last_success {
+                            BackupStatus::Failed // Failure after last success
+                        } else {
+                            BackupStatus::Healthy
+                        }
+                    } else {
+                        BackupStatus::Healthy
+                    }
+                } else {
+                    BackupStatus::Warning // No successful backups found
+                }
+            }
+            Err(_) => {
+                if service_data.enabled {
+                    BackupStatus::Failed // Service enabled but can't access repo
+                } else {
+                    BackupStatus::Unknown // Service disabled
+                }
+            }
+        }
+    }
+}
+
+#[async_trait]
+impl Collector for BackupCollector {
+    fn name(&self) -> &str {
+        "backup"
+    }
+
+    fn agent_type(&self) -> AgentType {
+        AgentType::Backup
+    }
+
+    fn collect_interval(&self) -> Duration {
+        self.interval
+    }
+
+    fn is_enabled(&self) -> bool {
+        self.enabled
+    }
+
+    fn requires_root(&self) -> bool {
+        false // Depends on restic repo permissions
+    }
+
+    async fn collect(&self) -> Result<CollectorOutput, CollectorError> {
+        // Get restic repository stats
+        let restic_stats = self.get_restic_snapshots().await;
+
+        // Get backup service status
+        let service_data = self
+            .get_backup_service_status()
+            .await
+            .unwrap_or(BackupServiceData {
+                enabled: false,
+                pending_jobs: 0,
+                last_message: None,
+            });
+
+        // Check for recent failures
+        let last_failure = self.get_backup_logs_for_failures().await.unwrap_or(None);
+
+        // Determine overall backup status
+        let overall_status =
+            self.determine_backup_status(&restic_stats, &service_data, last_failure);
+
+        let (backup_info, _size_gb) = match &restic_stats {
+            Ok(stats) => (
+                BackupInfo {
+                    last_success: stats.last_success,
+                    last_failure,
+                    size_gb: stats.total_size as f32 / (1024.0 * 1024.0 * 1024.0),
+                    snapshot_count: stats.snapshot_count,
+                },
+                stats.total_size as f32 / (1024.0 * 1024.0 * 1024.0),
+            ),
+            Err(_) => (
+                BackupInfo {
+                    last_success: None,
+                    last_failure,
+                    size_gb: 0.0,
+                    snapshot_count: 0,
+                },
+                0.0,
+            ),
+        };
+
+        let backup_metrics = json!({
+            "overall_status": overall_status,
+            "backup": backup_info,
+            "service": service_data,
+            "timestamp": Utc::now()
+        });
+
+        Ok(CollectorOutput {
+            agent_type: AgentType::Backup,
+            data: backup_metrics,
+            timestamp: Utc::now(),
+        })
+    }
+}
+
+#[derive(Debug, Deserialize)]
+struct ResticSnapshot {
+    time: DateTime<Utc>,
+}
+
+#[derive(Debug, Deserialize)]
+struct ResticStats {
+    total_size: u64,
+    snapshot_count: u32,
+    last_success: Option<DateTime<Utc>>,
+}
+
+#[derive(Debug, Serialize)]
+struct BackupServiceData {
+    enabled: bool,
+    pending_jobs: u32,
+    last_message: Option<String>,
+}
+
+#[derive(Debug, Serialize)]
+struct BackupInfo {
+    last_success: Option<DateTime<Utc>>,
+    last_failure: Option<DateTime<Utc>>,
+    size_gb: f32,
+    snapshot_count: u32,
+}
+
+#[derive(Debug, Serialize)]
+enum BackupStatus {
+    Healthy,
+    Warning,
+    Failed,
+    Unknown,
+}
+
+#[derive(Debug, Deserialize)]
+struct JournalEntry {
+    #[serde(rename = "__REALTIME_TIMESTAMP")]
+    realtime_timestamp: String,
+}

agent/src/collectors/error.rs

@@ -0,0 +1,53 @@
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum CollectorError {
+    #[error("Command execution failed: {command} - {message}")]
+    CommandFailed { command: String, message: String },
+
+    #[error("Permission denied: {message}")]
+    PermissionDenied { message: String },
+
+    #[error("Data parsing error: {message}")]
+    ParseError { message: String },
+
+    #[error("Timeout after {duration_ms}ms")]
+    Timeout { duration_ms: u64 },
+
+    #[error("IO error: {message}")]
+    IoError { message: String },
+
+    #[error("Configuration error: {message}")]
+    ConfigError { message: String },
+
+    #[error("Service not found: {service}")]
+    ServiceNotFound { service: String },
+
+    #[error("Device not found: {device}")]
+    DeviceNotFound { device: String },
+
+    #[error("External dependency error: {dependency} - {message}")]
+    ExternalDependency { dependency: String, message: String },
+}
+
+impl From<std::io::Error> for CollectorError {
+    fn from(err: std::io::Error) -> Self {
+        CollectorError::IoError {
+            message: err.to_string(),
+        }
+    }
+}
+
+impl From<serde_json::Error> for CollectorError {
+    fn from(err: serde_json::Error) -> Self {
+        CollectorError::ParseError {
+            message: err.to_string(),
+        }
+    }
+}
+
+impl From<tokio::time::error::Elapsed> for CollectorError {
+    fn from(_: tokio::time::error::Elapsed) -> Self {
+        CollectorError::Timeout { duration_ms: 0 }
+    }
+}

agent/src/collectors/mod.rs

@@ -0,0 +1,49 @@
+use async_trait::async_trait;
+use chrono::{DateTime, Utc};
+use serde_json::Value;
+use std::time::Duration;
+
+pub mod backup;
+pub mod error;
+pub mod service;
+pub mod smart;
+
+pub use error::CollectorError;
+
+#[derive(Debug, Clone)]
+pub enum AgentType {
+    Smart,
+    Service,
+    Backup,
+}
+
+impl AgentType {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            AgentType::Smart => "smart",
+            AgentType::Service => "service",
+            AgentType::Backup => "backup",
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct CollectorOutput {
+    pub agent_type: AgentType,
+    pub data: Value,
+    pub timestamp: DateTime<Utc>,
+}
+
+#[async_trait]
+pub trait Collector: Send + Sync {
+    fn name(&self) -> &str;
+    fn agent_type(&self) -> AgentType;
+    fn collect_interval(&self) -> Duration;
+    async fn collect(&self) -> Result<CollectorOutput, CollectorError>;
+    fn is_enabled(&self) -> bool {
+        true
+    }
+    fn requires_root(&self) -> bool {
+        false
+    }
+}

agent/src/collectors/service.rs

@@ -0,0 +1,603 @@
+use async_trait::async_trait;
+use chrono::Utc;
+use serde::Serialize;
+use serde_json::json;
+use std::collections::HashMap;
+use std::process::Stdio;
+use std::time::Duration;
+use tokio::fs;
+use tokio::process::Command;
+use tokio::time::timeout;
+
+use super::{AgentType, Collector, CollectorError, CollectorOutput};
+
+#[derive(Debug, Clone)]
+pub struct ServiceCollector {
+    pub enabled: bool,
+    pub interval: Duration,
+    pub services: Vec<String>,
+    pub timeout_ms: u64,
+}
+
+impl ServiceCollector {
+    pub fn new(enabled: bool, interval_ms: u64, services: Vec<String>) -> Self {
+        Self {
+            enabled,
+            interval: Duration::from_millis(interval_ms),
+            services,
+            timeout_ms: 10000, // 10 second timeout for service checks
+        }
+    }
+
+    async fn get_service_status(&self, service: &str) -> Result<ServiceData, CollectorError> {
+        let timeout_duration = Duration::from_millis(self.timeout_ms);
+
+        // Get systemctl status
+        let status_output = timeout(
+            timeout_duration,
+            Command::new("systemctl")
+                .args(["show", service, "--property=ActiveState,SubState,MainPID"])
+                .stdout(Stdio::piped())
+                .stderr(Stdio::piped())
+                .output(),
+        )
+        .await
+        .map_err(|_| CollectorError::Timeout {
+            duration_ms: self.timeout_ms,
+        })?
+        .map_err(|e| CollectorError::CommandFailed {
+            command: format!("systemctl show {}", service),
+            message: e.to_string(),
+        })?;
+
+        if !status_output.status.success() {
+            return Err(CollectorError::ServiceNotFound {
+                service: service.to_string(),
+            });
+        }
+
+        let status_stdout = String::from_utf8_lossy(&status_output.stdout);
+        let mut active_state = None;
+        let mut sub_state = None;
+        let mut main_pid = None;
+
+        for line in status_stdout.lines() {
+            if let Some(value) = line.strip_prefix("ActiveState=") {
+                active_state = Some(value.to_string());
+            } else if let Some(value) = line.strip_prefix("SubState=") {
+                sub_state = Some(value.to_string());
+            } else if let Some(value) = line.strip_prefix("MainPID=") {
+                main_pid = value.parse::<u32>().ok();
+            }
+        }
+
+        let status = self.determine_service_status(&active_state, &sub_state);
+
+        // Get resource usage if service is running
+        let (memory_used_mb, cpu_percent) = if let Some(pid) = main_pid {
+            self.get_process_resources(pid).await.unwrap_or((0.0, 0.0))
+        } else {
+            (0.0, 0.0)
+        };
+
+        // Get memory quota from systemd if available
+        let memory_quota_mb = self.get_service_memory_limit(service).await.unwrap_or(0.0);
+        
+        // Get disk usage for this service
+        let disk_used_gb = self.get_service_disk_usage(service).await.unwrap_or(0.0);
+
+        Ok(ServiceData {
+            name: service.to_string(),
+            status,
+            memory_used_mb,
+            memory_quota_mb,
+            cpu_percent,
+            sandbox_limit: None, // TODO: Implement sandbox limit detection
+            disk_used_gb,
+        })
+    }
+
+    fn determine_service_status(
+        &self,
+        active_state: &Option<String>,
+        sub_state: &Option<String>,
+    ) -> ServiceStatus {
+        match (active_state.as_deref(), sub_state.as_deref()) {
+            (Some("active"), Some("running")) => ServiceStatus::Running,
+            (Some("active"), Some("exited")) => ServiceStatus::Running, // One-shot services
+            (Some("reloading"), _) | (Some("activating"), _) => ServiceStatus::Restarting,
+            (Some("failed"), _) | (Some("inactive"), Some("failed")) => ServiceStatus::Stopped,
+            (Some("inactive"), _) => ServiceStatus::Stopped,
+            _ => ServiceStatus::Degraded,
+        }
+    }
+
+    async fn get_process_resources(&self, pid: u32) -> Result<(f32, f32), CollectorError> {
+        // Read /proc/{pid}/stat for CPU and memory info
+        let stat_path = format!("/proc/{}/stat", pid);
+        let stat_content =
+            fs::read_to_string(&stat_path)
+                .await
+                .map_err(|e| CollectorError::IoError {
+                    message: e.to_string(),
+                })?;
+
+        let stat_fields: Vec<&str> = stat_content.split_whitespace().collect();
+        if stat_fields.len() < 24 {
+            return Err(CollectorError::ParseError {
+                message: format!("Invalid /proc/{}/stat format", pid),
+            });
+        }
+
+        // Field 23 is RSS (Resident Set Size) in pages
+        let rss_pages: u64 = stat_fields[23]
+            .parse()
+            .map_err(|e| CollectorError::ParseError {
+                message: format!("Failed to parse RSS from /proc/{}/stat: {}", pid, e),
+            })?;
+
+        // Convert pages to MB (assuming 4KB pages)
+        let memory_mb = (rss_pages * 4) as f32 / 1024.0;
+
+        // For CPU, we'd need to track over time - simplified to 0 for now
+        // TODO: Implement proper CPU percentage calculation
+        let cpu_percent = 0.0;
+
+        Ok((memory_mb, cpu_percent))
+    }
+
+    async fn get_service_disk_usage(&self, service: &str) -> Result<f32, CollectorError> {
+        // For systemd services, check if they have private /var directories or specific data paths
+        // This is a simplified implementation - could be enhanced to check actual service-specific paths
+        
+        // Common service data directories to check
+        let potential_paths = vec![
+            format!("/var/lib/{}", service),
+            format!("/var/cache/{}", service),
+            format!("/var/log/{}", service),
+            format!("/opt/{}", service),
+            format!("/srv/{}", service),
+        ];
+        
+        let mut total_usage = 0.0;
+        
+        for path in potential_paths {
+            if let Ok(usage) = self.get_directory_size(&path).await {
+                total_usage += usage;
+            }
+        }
+        
+        Ok(total_usage)
+    }
+    
+    async fn get_directory_size(&self, path: &str) -> Result<f32, CollectorError> {
+        let output = Command::new("du")
+            .args(["-s", "-k", path])  // Use kilobytes instead of forcing GB
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .output()
+            .await
+            .map_err(|e| CollectorError::CommandFailed {
+                command: format!("du -s -k {}", path),
+                message: e.to_string(),
+            })?;
+
+        if !output.status.success() {
+            // Directory doesn't exist or permission denied - return 0
+            return Ok(0.0);
+        }
+
+        let stdout = String::from_utf8_lossy(&output.stdout);
+        if let Some(line) = stdout.lines().next() {
+            if let Some(size_str) = line.split_whitespace().next() {
+                let size_kb = size_str.parse::<f32>().unwrap_or(0.0);
+                let size_gb = size_kb / (1024.0 * 1024.0); // Convert KB to GB
+                return Ok(size_gb);
+            }
+        }
+        
+        Ok(0.0)
+    }
+
+    async fn get_service_memory_limit(&self, service: &str) -> Result<f32, CollectorError> {
+        let output = Command::new("systemctl")
+            .args(["show", service, "--property=MemoryMax"])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .output()
+            .await
+            .map_err(|e| CollectorError::CommandFailed {
+                command: format!("systemctl show {} --property=MemoryMax", service),
+                message: e.to_string(),
+            })?;
+
+        let stdout = String::from_utf8_lossy(&output.stdout);
+        for line in stdout.lines() {
+            if let Some(value) = line.strip_prefix("MemoryMax=") {
+                if value == "infinity" {
+                    return Ok(0.0); // No limit
+                }
+                if let Ok(bytes) = value.parse::<u64>() {
+                    return Ok(bytes as f32 / (1024.0 * 1024.0)); // Convert to MB
+                }
+            }
+        }
+
+        Ok(0.0) // No limit or couldn't parse
+    }
+
+    async fn get_system_memory_info(&self) -> Result<SystemMemoryInfo, CollectorError> {
+        let meminfo =
+            fs::read_to_string("/proc/meminfo")
+                .await
+                .map_err(|e| CollectorError::IoError {
+                    message: e.to_string(),
+                })?;
+
+        let mut memory_info = HashMap::new();
+        for line in meminfo.lines() {
+            if let Some((key, value)) = line.split_once(':') {
+                let value = value.trim().trim_end_matches(" kB");
+                if let Ok(kb) = value.parse::<u64>() {
+                    memory_info.insert(key.to_string(), kb);
+                }
+            }
+        }
+
+        let total_kb = memory_info.get("MemTotal").copied().unwrap_or(0);
+        let available_kb = memory_info.get("MemAvailable").copied().unwrap_or(0);
+        let used_kb = total_kb.saturating_sub(available_kb);
+
+        Ok(SystemMemoryInfo {
+            total_mb: total_kb as f32 / 1024.0,
+            used_mb: used_kb as f32 / 1024.0,
+        })
+    }
+
+    async fn get_disk_usage(&self) -> Result<DiskUsage, CollectorError> {
+        let output = Command::new("df")
+            .args(["-BG", "--output=size,used,avail", "/"])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .output()
+            .await
+            .map_err(|e| CollectorError::CommandFailed {
+                command: "df -BG --output=size,used,avail /".to_string(),
+                message: e.to_string(),
+            })?;
+
+        if !output.status.success() {
+            let stderr = String::from_utf8_lossy(&output.stderr);
+            return Err(CollectorError::CommandFailed {
+                command: "df -BG --output=size,used,avail /".to_string(),
+                message: stderr.to_string(),
+            });
+        }
+
+        let stdout = String::from_utf8_lossy(&output.stdout);
+        let lines: Vec<&str> = stdout.lines().collect();
+
+        if lines.len() < 2 {
+            return Err(CollectorError::ParseError {
+                message: "Unexpected df output format".to_string(),
+            });
+        }
+
+        let data_line = lines[1].trim();
+        let parts: Vec<&str> = data_line.split_whitespace().collect();
+        if parts.len() < 3 {
+            return Err(CollectorError::ParseError {
+                message: format!("Unexpected df data format: {}", data_line),
+            });
+        }
+
+        let parse_size = |s: &str| -> Result<f32, CollectorError> {
+            s.trim_end_matches('G')
+                .parse::<f32>()
+                .map_err(|e| CollectorError::ParseError {
+                    message: format!("Failed to parse disk size '{}': {}", s, e),
+                })
+        };
+
+        Ok(DiskUsage {
+            total_gb: parse_size(parts[0])?,
+            used_gb: parse_size(parts[1])?,
+        })
+    }
+
+    async fn get_cpu_load(&self) -> Result<(f32, f32, f32), CollectorError> {
+        let loadavg =
+            fs::read_to_string("/proc/loadavg")
+                .await
+                .map_err(|e| CollectorError::IoError {
+                    message: e.to_string(),
+                })?;
+
+        let parts: Vec<&str> = loadavg.split_whitespace().collect();
+        if parts.len() < 3 {
+            return Err(CollectorError::ParseError {
+                message: "Unexpected /proc/loadavg format".to_string(),
+            });
+        }
+
+        let parse = |s: &str| -> Result<f32, CollectorError> {
+            s.parse::<f32>().map_err(|e| CollectorError::ParseError {
+                message: format!("Failed to parse load average '{}': {}", s, e),
+            })
+        };
+
+        Ok((parse(parts[0])?, parse(parts[1])?, parse(parts[2])?))
+    }
+
+    async fn get_cpu_frequency_mhz(&self) -> Option<f32> {
+        let candidates = [
+            "/sys/devices/system/cpu/cpufreq/policy0/scaling_cur_freq",
+            "/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq",
+        ];
+
+        for path in candidates {
+            if let Ok(content) = fs::read_to_string(path).await {
+                if let Ok(khz) = content.trim().parse::<f32>() {
+                    if khz > 0.0 {
+                        return Some(khz / 1000.0);
+                    }
+                }
+            }
+        }
+
+        if let Ok(content) = fs::read_to_string("/proc/cpuinfo").await {
+            for line in content.lines() {
+                if let Some(rest) = line.strip_prefix("cpu MHz") {
+                    if let Some(value) = rest.split(':').nth(1) {
+                        if let Ok(mhz) = value.trim().parse::<f32>() {
+                            if mhz > 0.0 {
+                                return Some(mhz);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        None
+    }
+
+    async fn get_cpu_temperature_c(&self) -> Option<f32> {
+        let mut entries = fs::read_dir("/sys/class/thermal").await.ok()?;
+        let mut fallback: Option<f32> = None;
+
+        while let Ok(Some(entry)) = entries.next_entry().await {
+            let path = entry.path();
+            let type_path = path.join("type");
+            let temp_path = path.join("temp");
+
+            let label = fs::read_to_string(&type_path).await.ok()?.to_lowercase();
+            let raw = match fs::read_to_string(&temp_path).await {
+                Ok(value) => value,
+                Err(_) => continue,
+            };
+
+            let milli: f32 = match raw.trim().parse() {
+                Ok(value) => value,
+                Err(_) => continue,
+            };
+
+            let temp_c = milli / 1000.0;
+            if label.contains("cpu") || label.contains("pkg") {
+                if temp_c > 0.0 {
+                    return Some(temp_c);
+                }
+            }
+
+            if fallback.is_none() && temp_c > 0.0 {
+                fallback = Some(temp_c);
+            }
+        }
+
+        fallback
+    }
+
+    async fn get_gpu_metrics(&self) -> (Option<f32>, Option<f32>) {
+        let output = Command::new("nvidia-smi")
+            .args([
+                "--query-gpu=utilization.gpu,temperature.gpu",
+                "--format=csv,noheader,nounits",
+            ])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .output()
+            .await;
+
+        match output {
+            Ok(result) if result.status.success() => {
+                let stdout = String::from_utf8_lossy(&result.stdout);
+                if let Some(line) = stdout.lines().next() {
+                    let parts: Vec<&str> = line.split(',').map(|s| s.trim()).collect();
+                    if parts.len() >= 2 {
+                        let load = parts[0].parse::<f32>().ok();
+                        let temp = parts[1].parse::<f32>().ok();
+                        return (load, temp);
+                    }
+                }
+                (None, None)
+            }
+            Ok(_) | Err(_) => {
+                let util_output = Command::new("/opt/vc/bin/vcgencmd")
+                    .arg("measure_temp")
+                    .stdout(Stdio::piped())
+                    .stderr(Stdio::piped())
+                    .output()
+                    .await;
+
+                if let Ok(result) = util_output {
+                    if result.status.success() {
+                        let stdout = String::from_utf8_lossy(&result.stdout);
+                        if let Some(value) = stdout
+                            .trim()
+                            .strip_prefix("temp=")
+                            .and_then(|s| s.strip_suffix("'C"))
+                        {
+                            if let Ok(temp_c) = value.parse::<f32>() {
+                                return (None, Some(temp_c));
+                            }
+                        }
+                    }
+                }
+
+                (None, None)
+            }
+        }
+    }
+}
+
+#[async_trait]
+impl Collector for ServiceCollector {
+    fn name(&self) -> &str {
+        "service"
+    }
+
+    fn agent_type(&self) -> AgentType {
+        AgentType::Service
+    }
+
+    fn collect_interval(&self) -> Duration {
+        self.interval
+    }
+
+    fn is_enabled(&self) -> bool {
+        self.enabled
+    }
+
+    fn requires_root(&self) -> bool {
+        false // Most systemctl commands work without root
+    }
+
+    async fn collect(&self) -> Result<CollectorOutput, CollectorError> {
+        let mut services = Vec::new();
+        let mut healthy = 0;
+        let mut degraded = 0;
+        let mut failed = 0;
+        let mut total_memory_used = 0.0;
+        let mut total_memory_quota = 0.0;
+        let mut total_disk_used = 0.0;
+
+        // Collect data from all configured services
+        for service in &self.services {
+            match self.get_service_status(service).await {
+                Ok(service_data) => {
+                    match service_data.status {
+                        ServiceStatus::Running => healthy += 1,
+                        ServiceStatus::Degraded | ServiceStatus::Restarting => degraded += 1,
+                        ServiceStatus::Stopped => failed += 1,
+                    }
+
+                    total_memory_used += service_data.memory_used_mb;
+                    if service_data.memory_quota_mb > 0.0 {
+                        total_memory_quota += service_data.memory_quota_mb;
+                    }
+                    total_disk_used += service_data.disk_used_gb;
+
+                    services.push(service_data);
+                }
+                Err(e) => {
+                    failed += 1;
+                    // Add a placeholder service entry for failed collection
+                    services.push(ServiceData {
+                        name: service.clone(),
+                        status: ServiceStatus::Stopped,
+                        memory_used_mb: 0.0,
+                        memory_quota_mb: 0.0,
+                        cpu_percent: 0.0,
+                        sandbox_limit: None,
+                        disk_used_gb: 0.0,
+                    });
+                    tracing::warn!("Failed to collect metrics for service {}: {}", service, e);
+                }
+            }
+        }
+
+        // Get system memory info for quota calculation
+        let system_memory = self
+            .get_system_memory_info()
+            .await
+            .unwrap_or(SystemMemoryInfo {
+                total_mb: 0.0,
+                used_mb: 0.0,
+            });
+
+        let _disk_usage = self.get_disk_usage().await.unwrap_or(DiskUsage {
+            total_gb: 0.0,
+            used_gb: 0.0,
+        });
+
+        let (cpu_load_1, cpu_load_5, cpu_load_15) =
+            self.get_cpu_load().await.unwrap_or((0.0, 0.0, 0.0));
+        let cpu_freq_mhz = self.get_cpu_frequency_mhz().await;
+        let cpu_temp_c = self.get_cpu_temperature_c().await;
+        let (gpu_load_percent, gpu_temp_c) = self.get_gpu_metrics().await;
+
+        // If no specific quotas are set, use system memory as reference
+        if total_memory_quota == 0.0 {
+            total_memory_quota = system_memory.total_mb;
+        }
+
+        let service_metrics = json!({
+            "summary": {
+                "healthy": healthy,
+                "degraded": degraded,
+                "failed": failed,
+                "memory_used_mb": total_memory_used,
+                "memory_quota_mb": total_memory_quota,
+                "system_memory_used_mb": system_memory.used_mb,
+                "system_memory_total_mb": system_memory.total_mb,
+                "disk_used_gb": total_disk_used,
+                "disk_total_gb": total_disk_used, // For services, total = used (no quota concept)
+                "cpu_load_1": cpu_load_1,
+                "cpu_load_5": cpu_load_5,
+                "cpu_load_15": cpu_load_15,
+                "cpu_freq_mhz": cpu_freq_mhz,
+                "cpu_temp_c": cpu_temp_c,
+                "gpu_load_percent": gpu_load_percent,
+                "gpu_temp_c": gpu_temp_c,
+            },
+            "services": services,
+            "timestamp": Utc::now()
+        });
+
+        Ok(CollectorOutput {
+            agent_type: AgentType::Service,
+            data: service_metrics,
+            timestamp: Utc::now(),
+        })
+    }
+}
+
+#[derive(Debug, Clone, Serialize)]
+struct ServiceData {
+    name: String,
+    status: ServiceStatus,
+    memory_used_mb: f32,
+    memory_quota_mb: f32,
+    cpu_percent: f32,
+    sandbox_limit: Option<f32>,
+    disk_used_gb: f32,
+}
+
+#[derive(Debug, Clone, Serialize)]
+enum ServiceStatus {
+    Running,
+    Degraded,
+    Restarting,
+    Stopped,
+}
+
+struct SystemMemoryInfo {
+    total_mb: f32,
+    used_mb: f32,
+}
+
+#[allow(dead_code)]
+struct DiskUsage {
+    total_gb: f32,
+    used_gb: f32,
+}

agent/src/collectors/smart.rs

@@ -0,0 +1,447 @@
+use async_trait::async_trait;
+use chrono::Utc;
+use serde::{Deserialize, Serialize};
+use serde_json::json;
+use std::io::ErrorKind;
+use std::process::Stdio;
+use std::time::Duration;
+use tokio::process::Command;
+use tokio::time::timeout;
+
+use super::{AgentType, Collector, CollectorError, CollectorOutput};
+
+#[derive(Debug, Clone)]
+pub struct SmartCollector {
+    pub enabled: bool,
+    pub interval: Duration,
+    pub devices: Vec<String>,
+    pub timeout_ms: u64,
+}
+
+impl SmartCollector {
+    pub fn new(enabled: bool, interval_ms: u64, devices: Vec<String>) -> Self {
+        Self {
+            enabled,
+            interval: Duration::from_millis(interval_ms),
+            devices,
+            timeout_ms: 30000, // 30 second timeout for smartctl
+        }
+    }
+
+    async fn get_smart_data(&self, device: &str) -> Result<SmartDeviceData, CollectorError> {
+        let timeout_duration = Duration::from_millis(self.timeout_ms);
+
+        let command_result = timeout(
+            timeout_duration,
+            Command::new("smartctl")
+                .args(["-a", "-j", &format!("/dev/{}", device)])
+                .stdout(Stdio::piped())
+                .stderr(Stdio::piped())
+                .output(),
+        )
+        .await
+        .map_err(|_| CollectorError::Timeout {
+            duration_ms: self.timeout_ms,
+        })?;
+
+        let output = command_result.map_err(|e| match e.kind() {
+            ErrorKind::NotFound => CollectorError::ExternalDependency {
+                dependency: "smartctl".to_string(),
+                message: e.to_string(),
+            },
+            ErrorKind::PermissionDenied => CollectorError::PermissionDenied {
+                message: e.to_string(),
+            },
+            _ => CollectorError::CommandFailed {
+                command: format!("smartctl -a -j /dev/{}", device),
+                message: e.to_string(),
+            },
+        })?;
+
+        if !output.status.success() {
+            let stderr = String::from_utf8_lossy(&output.stderr);
+            let stderr_lower = stderr.to_lowercase();
+
+            if stderr_lower.contains("permission denied") {
+                return Err(CollectorError::PermissionDenied {
+                    message: stderr.to_string(),
+                });
+            }
+
+            if stderr_lower.contains("no such device") || stderr_lower.contains("cannot open") {
+                return Err(CollectorError::DeviceNotFound {
+                    device: device.to_string(),
+                });
+            }
+
+            return Err(CollectorError::CommandFailed {
+                command: format!("smartctl -a -j /dev/{}", device),
+                message: stderr.to_string(),
+            });
+        }
+
+        let stdout = String::from_utf8_lossy(&output.stdout);
+        let smart_output: SmartCtlOutput =
+            serde_json::from_str(&stdout).map_err(|e| CollectorError::ParseError {
+                message: format!("Failed to parse smartctl output for {}: {}", device, e),
+            })?;
+
+        Ok(SmartDeviceData::from_smartctl_output(device, smart_output))
+    }
+
+    async fn get_drive_usage(&self, device: &str) -> Result<(Option<f32>, Option<f32>), CollectorError> {
+        // Get capacity first
+        let capacity = match self.get_drive_capacity(device).await {
+            Ok(cap) => Some(cap),
+            Err(_) => None,
+        };
+
+        // Try to get usage information
+        // For simplicity, we'll use the root filesystem usage for now
+        // In the future, this could be enhanced to map drives to specific mount points
+        let usage = if device.contains("nvme0n1") || device.contains("sda") {
+            // This is likely the main system drive, use root filesystem usage
+            match self.get_disk_usage().await {
+                Ok(disk_usage) => Some(disk_usage.used_gb),
+                Err(_) => None,
+            }
+        } else {
+            // For other drives, we don't have usage info yet
+            None
+        };
+
+        Ok((capacity, usage))
+    }
+
+    async fn get_drive_capacity(&self, device: &str) -> Result<f32, CollectorError> {
+        let output = Command::new("lsblk")
+            .args(["-J", "-o", "NAME,SIZE", &format!("/dev/{}", device)])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .output()
+            .await
+            .map_err(|e| CollectorError::CommandFailed {
+                command: format!("lsblk -J -o NAME,SIZE /dev/{}", device),
+                message: e.to_string(),
+            })?;
+
+        if !output.status.success() {
+            let stderr = String::from_utf8_lossy(&output.stderr);
+            return Err(CollectorError::CommandFailed {
+                command: format!("lsblk -J -o NAME,SIZE /dev/{}", device),
+                message: stderr.to_string(),
+            });
+        }
+
+        let stdout = String::from_utf8_lossy(&output.stdout);
+        let lsblk_output: serde_json::Value = serde_json::from_str(&stdout)
+            .map_err(|e| CollectorError::ParseError {
+                message: format!("Failed to parse lsblk JSON: {}", e),
+            })?;
+
+        // Extract size from the first blockdevice
+        if let Some(blockdevices) = lsblk_output["blockdevices"].as_array() {
+            if let Some(device_info) = blockdevices.first() {
+                if let Some(size_str) = device_info["size"].as_str() {
+                    return self.parse_lsblk_size(size_str);
+                }
+            }
+        }
+
+        Err(CollectorError::ParseError {
+            message: format!("No size information found for device {}", device),
+        })
+    }
+
+    fn parse_lsblk_size(&self, size_str: &str) -> Result<f32, CollectorError> {
+        // Parse sizes like "953,9G", "1T", "512M"
+        let size_str = size_str.replace(',', ".");  // Handle European decimal separator
+        
+        if let Some(pos) = size_str.find(|c: char| c.is_alphabetic()) {
+            let (number_part, unit_part) = size_str.split_at(pos);
+            let number: f32 = number_part.parse()
+                .map_err(|e| CollectorError::ParseError {
+                    message: format!("Failed to parse size number '{}': {}", number_part, e),
+                })?;
+            
+            let multiplier = match unit_part.to_uppercase().as_str() {
+                "T" | "TB" => 1024.0,
+                "G" | "GB" => 1.0,
+                "M" | "MB" => 1.0 / 1024.0,
+                "K" | "KB" => 1.0 / (1024.0 * 1024.0),
+                _ => return Err(CollectorError::ParseError {
+                    message: format!("Unknown size unit: {}", unit_part),
+                }),
+            };
+            
+            Ok(number * multiplier)
+        } else {
+            Err(CollectorError::ParseError {
+                message: format!("Invalid size format: {}", size_str),
+            })
+        }
+    }
+
+    async fn get_disk_usage(&self) -> Result<DiskUsage, CollectorError> {
+        let output = Command::new("df")
+            .args(["-BG", "--output=size,used,avail", "/"])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .output()
+            .await
+            .map_err(|e| CollectorError::CommandFailed {
+                command: "df -BG --output=size,used,avail /".to_string(),
+                message: e.to_string(),
+            })?;
+
+        if !output.status.success() {
+            let stderr = String::from_utf8_lossy(&output.stderr);
+            return Err(CollectorError::CommandFailed {
+                command: "df -BG --output=size,used,avail /".to_string(),
+                message: stderr.to_string(),
+            });
+        }
+
+        let stdout = String::from_utf8_lossy(&output.stdout);
+        let lines: Vec<&str> = stdout.lines().collect();
+
+        if lines.len() < 2 {
+            return Err(CollectorError::ParseError {
+                message: "Unexpected df output format".to_string(),
+            });
+        }
+
+        // Skip header line, parse data line
+        let data_line = lines[1].trim();
+        let parts: Vec<&str> = data_line.split_whitespace().collect();
+
+        if parts.len() < 3 {
+            return Err(CollectorError::ParseError {
+                message: format!("Unexpected df data format: {}", data_line),
+            });
+        }
+
+        let parse_size = |s: &str| -> Result<f32, CollectorError> {
+            s.trim_end_matches('G')
+                .parse::<f32>()
+                .map_err(|e| CollectorError::ParseError {
+                    message: format!("Failed to parse disk size '{}': {}", s, e),
+                })
+        };
+
+        Ok(DiskUsage {
+            total_gb: parse_size(parts[0])?,
+            used_gb: parse_size(parts[1])?,
+            available_gb: parse_size(parts[2])?,
+        })
+    }
+}
+
+#[async_trait]
+impl Collector for SmartCollector {
+    fn name(&self) -> &str {
+        "smart"
+    }
+
+    fn agent_type(&self) -> AgentType {
+        AgentType::Smart
+    }
+
+    fn collect_interval(&self) -> Duration {
+        self.interval
+    }
+
+    fn is_enabled(&self) -> bool {
+        self.enabled
+    }
+
+    fn requires_root(&self) -> bool {
+        true // smartctl typically requires root access
+    }
+
+    async fn collect(&self) -> Result<CollectorOutput, CollectorError> {
+        let mut drives = Vec::new();
+        let mut issues = Vec::new();
+        let mut healthy = 0;
+        let mut warning = 0;
+        let mut critical = 0;
+
+        // Collect data from all configured devices
+        for device in &self.devices {
+            match self.get_smart_data(device).await {
+                Ok(mut drive_data) => {
+                    // Try to get capacity and usage for this drive
+                    if let Ok((capacity, usage)) = self.get_drive_usage(device).await {
+                        drive_data.capacity_gb = capacity;
+                        drive_data.used_gb = usage;
+                    }
+                    match drive_data.health_status.as_str() {
+                        "PASSED" => healthy += 1,
+                        "FAILED" => {
+                            critical += 1;
+                            issues.push(format!("{}: SMART status FAILED", device));
+                        }
+                        _ => {
+                            warning += 1;
+                            issues.push(format!("{}: Unknown SMART status", device));
+                        }
+                    }
+                    drives.push(drive_data);
+                }
+                Err(e) => {
+                    warning += 1;
+                    issues.push(format!("{}: {}", device, e));
+                }
+            }
+        }
+
+        // Get disk usage information
+        let disk_usage = self.get_disk_usage().await?;
+
+        let status = if critical > 0 {
+            "CRITICAL"
+        } else if warning > 0 {
+            "WARNING"
+        } else {
+            "HEALTHY"
+        };
+
+        let smart_metrics = json!({
+            "status": status,
+            "drives": drives,
+            "summary": {
+                "healthy": healthy,
+                "warning": warning,
+                "critical": critical,
+                "capacity_total_gb": disk_usage.total_gb,
+                "capacity_used_gb": disk_usage.used_gb,
+                "capacity_available_gb": disk_usage.available_gb
+            },
+            "issues": issues,
+            "timestamp": Utc::now()
+        });
+
+        Ok(CollectorOutput {
+            agent_type: AgentType::Smart,
+            data: smart_metrics,
+            timestamp: Utc::now(),
+        })
+    }
+}
+
+#[derive(Debug, Clone, Serialize)]
+struct SmartDeviceData {
+    name: String,
+    temperature_c: f32,
+    wear_level: f32,
+    power_on_hours: u64,
+    available_spare: f32,
+    health_status: String,
+    capacity_gb: Option<f32>,
+    used_gb: Option<f32>,
+}
+
+impl SmartDeviceData {
+    fn from_smartctl_output(device: &str, output: SmartCtlOutput) -> Self {
+        let temperature_c = output.temperature.and_then(|t| t.current).unwrap_or(0.0);
+
+        let wear_level = output
+            .nvme_smart_health_information_log
+            .as_ref()
+            .and_then(|nvme| nvme.percentage_used)
+            .unwrap_or(0.0);
+
+        let power_on_hours = output.power_on_time.and_then(|p| p.hours).unwrap_or(0);
+
+        let available_spare = output
+            .nvme_smart_health_information_log
+            .as_ref()
+            .and_then(|nvme| nvme.available_spare)
+            .unwrap_or(100.0);
+
+        let health_status = output
+            .smart_status
+            .and_then(|s| s.passed)
+            .map(|passed| {
+                if passed {
+                    "PASSED".to_string()
+                } else {
+                    "FAILED".to_string()
+                }
+            })
+            .unwrap_or_else(|| "UNKNOWN".to_string());
+
+        Self {
+            name: device.to_string(),
+            temperature_c,
+            wear_level,
+            power_on_hours,
+            available_spare,
+            health_status,
+            capacity_gb: None, // Will be set later by the collector
+            used_gb: None,     // Will be set later by the collector
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+struct DiskUsage {
+    total_gb: f32,
+    used_gb: f32,
+    available_gb: f32,
+}
+
+// Minimal smartctl JSON output structure - only the fields we need
+#[derive(Debug, Deserialize)]
+struct SmartCtlOutput {
+    temperature: Option<Temperature>,
+    power_on_time: Option<PowerOnTime>,
+    smart_status: Option<SmartStatus>,
+    nvme_smart_health_information_log: Option<NvmeSmartLog>,
+}
+
+#[derive(Debug, Deserialize)]
+struct Temperature {
+    current: Option<f32>,
+}
+
+#[derive(Debug, Deserialize)]
+struct PowerOnTime {
+    hours: Option<u64>,
+}
+
+#[derive(Debug, Deserialize)]
+struct SmartStatus {
+    passed: Option<bool>,
+}
+
+#[derive(Debug, Deserialize)]
+struct NvmeSmartLog {
+    percentage_used: Option<f32>,
+    available_spare: Option<f32>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_lsblk_size() {
+        let collector = SmartCollector::new(true, 5000, vec![]);
+        
+        // Test gigabyte sizes
+        assert!((collector.parse_lsblk_size("953,9G").unwrap() - 953.9).abs() < 0.1);
+        assert!((collector.parse_lsblk_size("1G").unwrap() - 1.0).abs() < 0.1);
+        
+        // Test terabyte sizes
+        assert!((collector.parse_lsblk_size("1T").unwrap() - 1024.0).abs() < 0.1);
+        assert!((collector.parse_lsblk_size("2,5T").unwrap() - 2560.0).abs() < 0.1);
+        
+        // Test megabyte sizes
+        assert!((collector.parse_lsblk_size("512M").unwrap() - 0.5).abs() < 0.1);
+        
+        // Test error cases
+        assert!(collector.parse_lsblk_size("invalid").is_err());
+        assert!(collector.parse_lsblk_size("1X").is_err());
+    }
+}