max / makenotwork

2.6 KB · 86 lines History Blame Raw

1	//! systemd daemon-health alert/recovery messages. Domain half of the Alerter
2	//! split; shared dispatch/cooldown plumbing lives in the parent module.
3
4	use tracing::instrument;
5
6	use super::{AlertCategory, AlertMeta, Alerter};
7
8	impl Alerter {
9	/// Fire when a host's watched daemons transition from all-healthy into a
10	/// degraded (crash-loop / host-wide failed unit) or down (a watched unit
11	/// inactive or unloaded) state. Carries the issue lines that fired.
12	#[instrument(skip_all)]
13	pub async fn send_systemd_alert(
14	&self,
15	target: &str,
16	label: &str,
17	status: &str,
18	issues: &[String],
19	) {
20	let alert_key = format!("systemd:{target}");
21	let subject = format!("[PoM] {label}: daemons {status}");
22	let body = format!(
23	"Target: {label} ({target})\n\
24	Status: {status}\n\
25	Issues:\n{}\n\
26	Instance: {}\n\
27	Time: {}\n\n\
28	- PoM",
29	issues
30	.iter()
31	.map(\|i\| format!(" - {i}"))
32	.collect::<Vec<_>>()
33	.join("\n"),
34	self.instance_name,
35	chrono::Utc::now().to_rfc3339(),
36	);
37
38	let priority = if status == "down" { "critical" } else { "high" };
39	self.fire_failure(
40	&subject,
41	&body,
42	priority,
43	"pom-systemd",
44	Some(target),
45	AlertMeta {
46	key: &alert_key,
47	category: AlertCategory::SystemdFailure,
48	from: None,
49	to: Some(status),
50	error: None,
51	},
52	)
53	.await;
54	}
55
56	/// Fire on recovery, every watched daemon healthy again and no host-wide
57	/// failed unit.
58	#[instrument(skip_all)]
59	pub async fn send_systemd_recovery(&self, target: &str, label: &str) {
60	let alert_key = format!("systemd:{target}");
61	let subject = format!("[PoM] {label}: daemons recovered");
62	let body = format!(
63	"Target: {label} ({target})\n\
64	All watched daemons are healthy.\n\
65	Instance: {}\n\
66	Time: {}\n\n\
67	- PoM",
68	self.instance_name,
69	chrono::Utc::now().to_rfc3339(),
70	);
71
72	self.fire_recovery(
73	&subject,
74	&body,
75	AlertMeta {
76	key: &alert_key,
77	category: AlertCategory::SystemdRecovery,
78	from: None,
79	to: Some("operational"),
80	error: None,
81	},
82	)
83	.await;
84	}
85	}
86