Add ARK game servers to systemd service monitoring
This commit is contained in:
parent
ca160c9627
commit
f67779be9d
@ -51,7 +51,7 @@ impl SystemdCollector {
|
||||
/// Get monitored services, discovering them if needed or cache is expired
|
||||
fn get_monitored_services(&self) -> Result<Vec<String>> {
|
||||
let mut state = self.state.write().unwrap();
|
||||
|
||||
|
||||
// Check if we need to discover services
|
||||
let needs_discovery = match state.last_discovery_time {
|
||||
None => true, // First time
|
||||
@ -60,15 +60,18 @@ impl SystemdCollector {
|
||||
elapsed >= state.discovery_interval_seconds
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
if needs_discovery {
|
||||
debug!("Discovering systemd services (cache expired or first run)");
|
||||
match self.discover_services() {
|
||||
Ok(services) => {
|
||||
state.monitored_services = services;
|
||||
state.last_discovery_time = Some(Instant::now());
|
||||
debug!("Auto-discovered {} services to monitor: {:?}",
|
||||
state.monitored_services.len(), state.monitored_services);
|
||||
debug!(
|
||||
"Auto-discovered {} services to monitor: {:?}",
|
||||
state.monitored_services.len(),
|
||||
state.monitored_services
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("Failed to discover services, using cached list: {}", e);
|
||||
@ -76,14 +79,14 @@ impl SystemdCollector {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Ok(state.monitored_services.clone())
|
||||
}
|
||||
|
||||
/// Get nginx site metrics, checking them if cache is expired
|
||||
fn get_nginx_site_metrics(&self) -> Vec<Metric> {
|
||||
let mut state = self.state.write().unwrap();
|
||||
|
||||
|
||||
// Check if we need to refresh nginx site metrics
|
||||
let needs_refresh = match state.last_nginx_check_time {
|
||||
None => true, // First time
|
||||
@ -92,17 +95,20 @@ impl SystemdCollector {
|
||||
elapsed >= state.nginx_check_interval_seconds
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
if needs_refresh {
|
||||
// Only check nginx sites if nginx service is active
|
||||
if state.monitored_services.iter().any(|s| s.contains("nginx")) {
|
||||
debug!("Refreshing nginx site latency metrics (interval: {}s)", state.nginx_check_interval_seconds);
|
||||
debug!(
|
||||
"Refreshing nginx site latency metrics (interval: {}s)",
|
||||
state.nginx_check_interval_seconds
|
||||
);
|
||||
let fresh_metrics = self.get_nginx_sites();
|
||||
state.nginx_site_metrics = fresh_metrics;
|
||||
state.last_nginx_check_time = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
state.nginx_site_metrics.clone()
|
||||
}
|
||||
|
||||
@ -126,7 +132,7 @@ impl SystemdCollector {
|
||||
// Skip setup/certificate services that don't need monitoring (from legacy)
|
||||
let excluded_services = [
|
||||
"mosquitto-certs",
|
||||
"immich-setup",
|
||||
"immich-setup",
|
||||
"phpfpm-kryddorten",
|
||||
"phpfpm-mariehall2",
|
||||
"acme-haasp.net",
|
||||
@ -160,7 +166,7 @@ impl SystemdCollector {
|
||||
"rclone",
|
||||
// Container runtimes
|
||||
"docker",
|
||||
// CI/CD services
|
||||
// CI/CD services
|
||||
"gitea-actions",
|
||||
"gitea-runner",
|
||||
"actions-runner",
|
||||
@ -176,6 +182,8 @@ impl SystemdCollector {
|
||||
"haasp",
|
||||
// Backup services
|
||||
"backup",
|
||||
// Game servers
|
||||
"ark",
|
||||
];
|
||||
|
||||
for line in output_str.lines() {
|
||||
@ -183,26 +191,32 @@ impl SystemdCollector {
|
||||
if fields.len() >= 4 && fields[0].ends_with(".service") {
|
||||
let service_name = fields[0].trim_end_matches(".service");
|
||||
debug!("Processing service: '{}'", service_name);
|
||||
|
||||
|
||||
// Skip excluded services first
|
||||
let mut is_excluded = false;
|
||||
for excluded in &excluded_services {
|
||||
if service_name.contains(excluded) {
|
||||
debug!("EXCLUDING service '{}' because it matches pattern '{}'", service_name, excluded);
|
||||
debug!(
|
||||
"EXCLUDING service '{}' because it matches pattern '{}'",
|
||||
service_name, excluded
|
||||
);
|
||||
is_excluded = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if is_excluded {
|
||||
debug!("Skipping excluded service: '{}'", service_name);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// Check if this service matches our interesting patterns
|
||||
for pattern in &interesting_services {
|
||||
if service_name.contains(pattern) || pattern.contains(service_name) {
|
||||
debug!("INCLUDING service '{}' because it matches pattern '{}'", service_name, pattern);
|
||||
debug!(
|
||||
"INCLUDING service '{}' because it matches pattern '{}'",
|
||||
service_name, pattern
|
||||
);
|
||||
services.push(service_name.to_string());
|
||||
break;
|
||||
}
|
||||
@ -214,7 +228,8 @@ impl SystemdCollector {
|
||||
if !services.iter().any(|s| s.contains("ssh")) {
|
||||
for line in output_str.lines() {
|
||||
let fields: Vec<&str> = line.split_whitespace().collect();
|
||||
if fields.len() >= 4 && (fields[0] == "sshd.service" || fields[0] == "ssh.service") {
|
||||
if fields.len() >= 4 && (fields[0] == "sshd.service" || fields[0] == "ssh.service")
|
||||
{
|
||||
let service_name = fields[0].trim_end_matches(".service");
|
||||
services.push(service_name.to_string());
|
||||
break;
|
||||
@ -276,7 +291,6 @@ impl SystemdCollector {
|
||||
None
|
||||
}
|
||||
|
||||
|
||||
/// Get service disk usage by examining service working directory
|
||||
fn get_service_disk_usage(&self, service: &str) -> Option<f32> {
|
||||
// Try to get working directory from systemctl
|
||||
@ -301,12 +315,18 @@ impl SystemdCollector {
|
||||
let service_dirs = match service {
|
||||
// Container and virtualization services
|
||||
s if s.contains("docker") => vec!["/var/lib/docker", "/var/lib/docker/containers"],
|
||||
|
||||
|
||||
// Web services and applications
|
||||
s if s.contains("gitea") => vec!["/var/lib/gitea", "/opt/gitea", "/home/git", "/data/gitea"],
|
||||
s if s.contains("gitea") => {
|
||||
vec!["/var/lib/gitea", "/opt/gitea", "/home/git", "/data/gitea"]
|
||||
}
|
||||
s if s.contains("nginx") => vec!["/var/log/nginx", "/var/www", "/usr/share/nginx"],
|
||||
s if s.contains("apache") || s.contains("httpd") => vec!["/var/log/apache2", "/var/www", "/etc/apache2"],
|
||||
s if s.contains("immich") => vec!["/var/lib/immich", "/opt/immich", "/usr/src/app/upload"],
|
||||
s if s.contains("apache") || s.contains("httpd") => {
|
||||
vec!["/var/log/apache2", "/var/www", "/etc/apache2"]
|
||||
}
|
||||
s if s.contains("immich") => {
|
||||
vec!["/var/lib/immich", "/opt/immich", "/usr/src/app/upload"]
|
||||
}
|
||||
s if s.contains("nextcloud") => vec!["/var/www/nextcloud", "/var/nextcloud"],
|
||||
s if s.contains("owncloud") => vec!["/var/www/owncloud", "/var/owncloud"],
|
||||
s if s.contains("plex") => vec!["/var/lib/plexmediaserver", "/opt/plex"],
|
||||
@ -315,27 +335,31 @@ impl SystemdCollector {
|
||||
s if s.contains("vaultwarden") => vec!["/var/lib/vaultwarden", "/opt/vaultwarden"],
|
||||
s if s.contains("grafana") => vec!["/var/lib/grafana", "/etc/grafana"],
|
||||
s if s.contains("prometheus") => vec!["/var/lib/prometheus", "/etc/prometheus"],
|
||||
|
||||
|
||||
// Database services
|
||||
s if s.contains("postgres") => vec!["/var/lib/postgresql", "/var/lib/postgres"],
|
||||
s if s.contains("mysql") => vec!["/var/lib/mysql"],
|
||||
s if s.contains("mariadb") => vec!["/var/lib/mysql", "/var/lib/mariadb"],
|
||||
s if s.contains("redis") => vec!["/var/lib/redis", "/var/redis"],
|
||||
s if s.contains("mongodb") || s.contains("mongo") => vec!["/var/lib/mongodb", "/var/lib/mongo"],
|
||||
|
||||
s if s.contains("mongodb") || s.contains("mongo") => {
|
||||
vec!["/var/lib/mongodb", "/var/lib/mongo"]
|
||||
}
|
||||
|
||||
// Message queues and communication
|
||||
s if s.contains("mosquitto") => vec!["/var/lib/mosquitto", "/etc/mosquitto"],
|
||||
s if s.contains("postfix") => vec!["/var/spool/postfix", "/var/lib/postfix"],
|
||||
s if s.contains("ssh") => vec!["/var/log/auth.log", "/etc/ssh"],
|
||||
|
||||
|
||||
// Download and sync services
|
||||
s if s.contains("transmission") => vec!["/var/lib/transmission-daemon", "/var/transmission"],
|
||||
s if s.contains("transmission") => {
|
||||
vec!["/var/lib/transmission-daemon", "/var/transmission"]
|
||||
}
|
||||
s if s.contains("syncthing") => vec!["/var/lib/syncthing", "/home/syncthing"],
|
||||
|
||||
|
||||
// System services - check logs and config
|
||||
s if s.contains("systemd") => vec!["/var/log/journal"],
|
||||
s if s.contains("cron") => vec!["/var/spool/cron", "/var/log/cron"],
|
||||
|
||||
|
||||
// Default fallbacks for any service
|
||||
_ => vec![],
|
||||
};
|
||||
@ -365,14 +389,9 @@ impl SystemdCollector {
|
||||
None
|
||||
}
|
||||
|
||||
|
||||
/// Get directory size in GB with permission-aware logging
|
||||
fn get_directory_size(&self, dir: &str) -> Option<f32> {
|
||||
let output = Command::new("du")
|
||||
.arg("-sb")
|
||||
.arg(dir)
|
||||
.output()
|
||||
.ok()?;
|
||||
let output = Command::new("du").arg("-sb").arg(dir).output().ok()?;
|
||||
|
||||
if !output.status.success() {
|
||||
// Log permission errors for debugging but don't spam logs
|
||||
@ -449,9 +468,13 @@ impl SystemdCollector {
|
||||
// Try service-specific known directories
|
||||
let service_dirs = match service {
|
||||
s if s.contains("docker") => vec!["/var/lib/docker", "/var/lib/docker/containers"],
|
||||
s if s.contains("gitea") => vec!["/var/lib/gitea", "/opt/gitea", "/home/git", "/data/gitea"],
|
||||
s if s.contains("gitea") => {
|
||||
vec!["/var/lib/gitea", "/opt/gitea", "/home/git", "/data/gitea"]
|
||||
}
|
||||
s if s.contains("nginx") => vec!["/var/log/nginx", "/var/www", "/usr/share/nginx"],
|
||||
s if s.contains("immich") => vec!["/var/lib/immich", "/opt/immich", "/usr/src/app/upload"],
|
||||
s if s.contains("immich") => {
|
||||
vec!["/var/lib/immich", "/opt/immich", "/usr/src/app/upload"]
|
||||
}
|
||||
s if s.contains("postgres") => vec!["/var/lib/postgresql", "/var/lib/postgres"],
|
||||
s if s.contains("mysql") => vec!["/var/lib/mysql"],
|
||||
s if s.contains("redis") => vec!["/var/lib/redis", "/var/redis"],
|
||||
@ -626,9 +649,10 @@ impl SystemdCollector {
|
||||
if let Ok(link) = std::fs::read_link(entry.path()) {
|
||||
if let Some(path_str) = link.to_str() {
|
||||
// Skip special files, focus on regular files
|
||||
if !path_str.starts_with("/dev/") &&
|
||||
!path_str.starts_with("/proc/") &&
|
||||
!path_str.starts_with("[") {
|
||||
if !path_str.starts_with("/dev/")
|
||||
&& !path_str.starts_with("/proc/")
|
||||
&& !path_str.starts_with("[")
|
||||
{
|
||||
if let Ok(metadata) = std::fs::metadata(&link) {
|
||||
total_size += metadata.len();
|
||||
found_any = true;
|
||||
@ -651,15 +675,15 @@ impl SystemdCollector {
|
||||
fn estimate_service_disk_usage(&self, service: &str) -> Option<f32> {
|
||||
// Get memory usage to help estimate disk usage
|
||||
let memory_mb = self.get_service_memory(service).unwrap_or(0.0);
|
||||
|
||||
|
||||
let estimated_gb = match service {
|
||||
// Database services typically have significant disk usage
|
||||
s if s.contains("mysql") || s.contains("postgres") || s.contains("redis") => {
|
||||
(memory_mb / 100.0).max(0.1) // Estimate based on memory
|
||||
},
|
||||
}
|
||||
// Web services and applications
|
||||
s if s.contains("nginx") || s.contains("apache") => 0.05, // ~50MB for configs/logs
|
||||
s if s.contains("gitea") => (memory_mb / 50.0).max(0.5), // Code repositories
|
||||
s if s.contains("gitea") => (memory_mb / 50.0).max(0.5), // Code repositories
|
||||
s if s.contains("docker") => 1.0, // Docker has significant overhead
|
||||
// System services
|
||||
s if s.contains("ssh") || s.contains("postfix") => 0.01, // ~10MB for configs/logs
|
||||
@ -669,8 +693,6 @@ impl SystemdCollector {
|
||||
|
||||
Some(estimated_gb)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@ -750,8 +772,11 @@ impl Collector for SystemdCollector {
|
||||
}
|
||||
|
||||
let collection_time = start_time.elapsed();
|
||||
debug!("Systemd collection completed in {:?} with {} individual service metrics",
|
||||
collection_time, metrics.len());
|
||||
debug!(
|
||||
"Systemd collection completed in {:?} with {} individual service metrics",
|
||||
collection_time,
|
||||
metrics.len()
|
||||
);
|
||||
|
||||
Ok(metrics)
|
||||
}
|
||||
@ -828,7 +853,7 @@ impl SystemdCollector {
|
||||
Ok(s) => s,
|
||||
Err(_) => return metrics,
|
||||
};
|
||||
|
||||
|
||||
for line in output_str.lines() {
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
@ -863,11 +888,11 @@ impl SystemdCollector {
|
||||
|
||||
/// Check site latency using HTTP GET requests
|
||||
fn check_site_latency(&self, url: &str) -> Result<f32, Box<dyn std::error::Error>> {
|
||||
use std::time::Instant;
|
||||
use std::time::Duration;
|
||||
|
||||
use std::time::Instant;
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
|
||||
// Create HTTP client with timeouts (similar to legacy implementation)
|
||||
let client = reqwest::blocking::Client::builder()
|
||||
.timeout(Duration::from_secs(5))
|
||||
@ -878,19 +903,24 @@ impl SystemdCollector {
|
||||
// Make GET request and measure latency
|
||||
let response = client.get(url).send()?;
|
||||
let latency = start.elapsed().as_millis() as f32;
|
||||
|
||||
|
||||
// Check if response is successful (2xx or 3xx status codes)
|
||||
if response.status().is_success() || response.status().is_redirection() {
|
||||
Ok(latency)
|
||||
} else {
|
||||
Err(format!("HTTP request failed for {} with status: {}", url, response.status()).into())
|
||||
Err(format!(
|
||||
"HTTP request failed for {} with status: {}",
|
||||
url,
|
||||
response.status()
|
||||
)
|
||||
.into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Discover nginx sites from configuration files (like the old working implementation)
|
||||
fn discover_nginx_sites(&self) -> Vec<(String, String)> {
|
||||
use tracing::debug;
|
||||
|
||||
|
||||
// Use the same approach as the old working agent: get nginx config from systemd
|
||||
let config_content = match self.get_nginx_config_from_systemd() {
|
||||
Some(content) => content,
|
||||
@ -905,28 +935,28 @@ impl SystemdCollector {
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// Parse the config content to extract sites
|
||||
self.parse_nginx_config_for_sites(&config_content)
|
||||
}
|
||||
|
||||
|
||||
/// Get nginx config from systemd service definition (NixOS compatible)
|
||||
fn get_nginx_config_from_systemd(&self) -> Option<String> {
|
||||
use tracing::debug;
|
||||
|
||||
|
||||
let output = std::process::Command::new("systemctl")
|
||||
.args(["show", "nginx", "--property=ExecStart", "--no-pager"])
|
||||
.output()
|
||||
.ok()?;
|
||||
|
||||
|
||||
if !output.status.success() {
|
||||
debug!("Failed to get nginx ExecStart from systemd");
|
||||
return None;
|
||||
}
|
||||
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
debug!("systemctl show nginx output: {}", stdout);
|
||||
|
||||
|
||||
// Parse ExecStart to extract -c config path
|
||||
for line in stdout.lines() {
|
||||
if line.starts_with("ExecStart=") {
|
||||
@ -941,25 +971,25 @@ impl SystemdCollector {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
|
||||
/// Extract config path from ExecStart line
|
||||
fn extract_config_path_from_exec_start(&self, exec_start: &str) -> Option<String> {
|
||||
use tracing::debug;
|
||||
|
||||
|
||||
// Remove ExecStart= prefix
|
||||
let exec_part = exec_start.strip_prefix("ExecStart=")?;
|
||||
debug!("Parsing exec part: {}", exec_part);
|
||||
|
||||
|
||||
// Handle NixOS format: ExecStart={ path=...; argv[]=...nginx -c /config; ... }
|
||||
if exec_part.contains("argv[]=") {
|
||||
// Extract the part after argv[]=
|
||||
let argv_start = exec_part.find("argv[]=")?;
|
||||
let argv_part = &exec_part[argv_start + 7..]; // Skip "argv[]="
|
||||
debug!("Found NixOS argv part: {}", argv_part);
|
||||
|
||||
|
||||
// Look for -c flag followed by config path
|
||||
if let Some(c_pos) = argv_part.find(" -c ") {
|
||||
let after_c = &argv_part[c_pos + 4..];
|
||||
@ -976,36 +1006,36 @@ impl SystemdCollector {
|
||||
return Some(config_path.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
|
||||
/// Fallback: get nginx config via nginx -T command
|
||||
fn get_nginx_config_via_command(&self) -> Option<String> {
|
||||
use tracing::debug;
|
||||
|
||||
|
||||
let output = std::process::Command::new("nginx")
|
||||
.args(["-T"])
|
||||
.output()
|
||||
.ok()?;
|
||||
|
||||
|
||||
if !output.status.success() {
|
||||
debug!("nginx -T failed");
|
||||
return None;
|
||||
}
|
||||
|
||||
|
||||
Some(String::from_utf8_lossy(&output.stdout).to_string())
|
||||
}
|
||||
|
||||
|
||||
/// Parse nginx config content to extract server names and build site list
|
||||
fn parse_nginx_config_for_sites(&self, config_content: &str) -> Vec<(String, String)> {
|
||||
use tracing::debug;
|
||||
let mut sites = Vec::new();
|
||||
let lines: Vec<&str> = config_content.lines().collect();
|
||||
let mut i = 0;
|
||||
|
||||
|
||||
debug!("Parsing nginx config with {} lines", lines.len());
|
||||
|
||||
|
||||
while i < lines.len() {
|
||||
let line = lines[i].trim();
|
||||
if line.starts_with("server") && line.contains("{") {
|
||||
@ -1019,11 +1049,11 @@ impl SystemdCollector {
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
|
||||
|
||||
debug!("Discovered {} nginx sites total", sites.len());
|
||||
sites
|
||||
}
|
||||
|
||||
|
||||
/// Parse a server block to extract the primary server_name
|
||||
fn parse_server_block(&self, lines: &[&str], start_index: &mut usize) -> Option<String> {
|
||||
use tracing::debug;
|
||||
@ -1031,38 +1061,42 @@ impl SystemdCollector {
|
||||
let mut has_redirect = false;
|
||||
let mut i = *start_index + 1;
|
||||
let mut brace_count = 1;
|
||||
|
||||
|
||||
// Parse until we close the server block
|
||||
while i < lines.len() && brace_count > 0 {
|
||||
let trimmed = lines[i].trim();
|
||||
|
||||
|
||||
// Track braces
|
||||
brace_count += trimmed.matches('{').count();
|
||||
brace_count -= trimmed.matches('}').count();
|
||||
|
||||
|
||||
// Extract server_name
|
||||
if trimmed.starts_with("server_name") {
|
||||
if let Some(names_part) = trimmed.strip_prefix("server_name") {
|
||||
let names_clean = names_part.trim().trim_end_matches(';');
|
||||
for name in names_clean.split_whitespace() {
|
||||
if name != "_" && !name.is_empty() && name.contains('.') && !name.starts_with('$') {
|
||||
if name != "_"
|
||||
&& !name.is_empty()
|
||||
&& name.contains('.')
|
||||
&& !name.starts_with('$')
|
||||
{
|
||||
server_names.push(name.to_string());
|
||||
debug!("Found server_name in block: {}", name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Check for redirects (skip redirect-only servers)
|
||||
if trimmed.contains("return") && (trimmed.contains("301") || trimmed.contains("302")) {
|
||||
has_redirect = true;
|
||||
}
|
||||
|
||||
|
||||
i += 1;
|
||||
}
|
||||
|
||||
|
||||
*start_index = i - 1;
|
||||
|
||||
|
||||
// Only return hostnames that are not redirects and have actual content
|
||||
if !server_names.is_empty() && !has_redirect {
|
||||
Some(server_names[0].clone())
|
||||
@ -1070,4 +1104,4 @@ impl SystemdCollector {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user