Add comprehensive timeouts to all blocking system commands
Fixes random host disconnections caused by blocking operations preventing timely ZMQ packet transmission. Changes: - Add run_command_with_timeout() wrapper using tokio for async command execution - Apply 10s timeout to smartctl (prevents 30+ second hangs on failing drives) - Apply 5s timeout to du, lsblk, systemctl list commands - Apply 3s timeout to systemctl show/is-active, df, ip commands - Apply 2s timeout to hostname command - Use system 'timeout' command for sync operations where async not needed Critical fixes: - smartctl: Failing drives could block for 30+ seconds per drive - du: Large directories (Docker, PostgreSQL) could block 10-30+ seconds - systemctl/docker: Commands could block indefinitely during system issues With 1-second collection interval and 10-second heartbeat timeout, any blocking operation >10s causes false "host offline" alerts. These timeouts ensure collection completes quickly even during system degradation.
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
use async_trait::async_trait;
|
||||
use cm_dashboard_shared::{AgentData};
|
||||
|
||||
use std::process::{Command, Output};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
|
||||
pub mod backup;
|
||||
pub mod cpu;
|
||||
@@ -13,6 +15,20 @@ pub mod systemd;
|
||||
|
||||
pub use error::CollectorError;
|
||||
|
||||
/// Run a command with a timeout to prevent blocking
|
||||
pub async fn run_command_with_timeout(mut cmd: Command, timeout_secs: u64) -> std::io::Result<Output> {
|
||||
let timeout_duration = Duration::from_secs(timeout_secs);
|
||||
|
||||
match timeout(timeout_duration, tokio::task::spawn_blocking(move || cmd.output())).await {
|
||||
Ok(Ok(result)) => result,
|
||||
Ok(Err(e)) => Err(std::io::Error::new(std::io::ErrorKind::Other, e)),
|
||||
Err(_) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
format!("Command timed out after {} seconds", timeout_secs)
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Base trait for all collectors with direct structured data output
|
||||
#[async_trait]
|
||||
|
||||
Reference in New Issue
Block a user