Skip to main content

max / makenotwork

5.2 KB · 130 lines History Blame Raw
1 //! N-of-M status-transition debounce, shared by the health and ssh-banner loops.
2 //!
3 //! The probe loops insert a health snapshot every tick (the raw history), but a
4 //! *transition*, the thing that pages an operator and opens an incident, must
5 //! not fire on the first differing check: one transient timeout would flip
6 //! `Operational -> Unreachable`, email an alert, and open an incident that
7 //! resolves on the next probe (fuzz-2026-07-06: alert fatigue + incident noise).
8 //!
9 //! [`TransitionGate`] only reports a transition once a new status has been
10 //! observed for `confirmations` consecutive checks, and it is **seeded from the
11 //! last stored status on startup**, so a restart of a persistently-degraded
12 //! target does not re-fire its alert.
13
14 use pom::types::HealthStatus;
15
16 /// Tracks the confirmed status and a pending candidate change.
17 #[derive(Debug, Clone)]
18 pub(crate) struct TransitionGate {
19 /// The status we have committed to (alerted/incident-tracked on).
20 confirmed: Option<HealthStatus>,
21 /// A differing status observed but not yet confirmed: (candidate, run length).
22 pending: Option<(HealthStatus, u32)>,
23 /// Consecutive matching observations required to confirm a change.
24 confirmations: u32,
25 }
26
27 impl TransitionGate {
28 /// Seed from the last stored status (`None` if the target has no history).
29 /// `confirmations` is clamped to at least 1 (1 = alert on first difference).
30 pub(crate) fn seeded(confirmed: Option<HealthStatus>, confirmations: u32) -> Self {
31 Self {
32 confirmed,
33 pending: None,
34 confirmations: confirmations.max(1),
35 }
36 }
37
38 /// Feed one observed status. Returns `Some((from, to))` only when a change is
39 /// *confirmed*: observed for `confirmations` consecutive checks. Returns
40 /// `None` while a change is still pending, when the status is unchanged, or on
41 /// the very first observation of a target with no prior confirmed status.
42 pub(crate) fn observe(
43 &mut self,
44 observed: HealthStatus,
45 ) -> Option<(HealthStatus, HealthStatus)> {
46 match self.confirmed {
47 // No prior status (fresh target with no history): adopt silently; the
48 // old code likewise needs a `previous` row before it alerts.
49 None => {
50 self.confirmed = Some(observed);
51 self.pending = None;
52 None
53 }
54 // Back to (or still at) the confirmed status: any pending change was a
55 // blip; drop it.
56 Some(cur) if observed == cur => {
57 self.pending = None;
58 None
59 }
60 // Differs from confirmed: accumulate consecutive agreement.
61 Some(cur) => {
62 let count = match self.pending {
63 Some((p, c)) if p == observed => c + 1,
64 _ => 1,
65 };
66 if count >= self.confirmations {
67 self.pending = None;
68 self.confirmed = Some(observed);
69 Some((cur, observed))
70 } else {
71 self.pending = Some((observed, count));
72 None
73 }
74 }
75 }
76 }
77 }
78
79 #[cfg(test)]
80 mod tests {
81 use super::*;
82 use pom::types::HealthStatus::*;
83
84 #[test]
85 fn confirmations_1_fires_on_first_difference() {
86 let mut g = TransitionGate::seeded(Some(Operational), 1);
87 assert_eq!(g.observe(Error), Some((Operational, Error)));
88 assert_eq!(g.observe(Error), None); // now confirmed
89 }
90
91 #[test]
92 fn single_blip_is_absorbed_at_confirmations_2() {
93 let mut g = TransitionGate::seeded(Some(Operational), 2);
94 // One differing check → pending, no transition.
95 assert_eq!(g.observe(Error), None);
96 // Back to operational → the blip is dropped.
97 assert_eq!(g.observe(Operational), None);
98 // A later genuine outage still needs two in a row.
99 assert_eq!(g.observe(Error), None);
100 assert_eq!(g.observe(Error), Some((Operational, Error)));
101 }
102
103 #[test]
104 fn flapping_candidate_resets_the_run() {
105 let mut g = TransitionGate::seeded(Some(Operational), 3);
106 assert_eq!(g.observe(Error), None); // Error x1
107 assert_eq!(g.observe(Unreachable), None); // candidate changed → Unreachable x1
108 assert_eq!(g.observe(Unreachable), None); // Unreachable x2
109 assert_eq!(g.observe(Unreachable), Some((Operational, Unreachable))); // x3 confirms
110 }
111
112 #[test]
113 fn seeded_status_does_not_refire_on_restart() {
114 // Restart: last stored status was Error. Observing Error again must NOT
115 // produce a transition (that was the re-fire-on-restart bug).
116 let mut g = TransitionGate::seeded(Some(Error), 2);
117 assert_eq!(g.observe(Error), None);
118 assert_eq!(g.observe(Error), None);
119 }
120
121 #[test]
122 fn first_ever_observation_is_silent() {
123 let mut g = TransitionGate::seeded(None, 2);
124 assert_eq!(g.observe(Operational), None);
125 // Now Operational is confirmed; a change debounces normally.
126 assert_eq!(g.observe(Error), None);
127 assert_eq!(g.observe(Error), Some((Operational, Error)));
128 }
129 }
130