[WIP][nexus] initial diagnosis engine prototype (PSU insert/remove)

Cargo.toml

@@ -87,6 +87,7 @@ members = [
     "nexus/db-schema",
     "nexus/defaults",
     "nexus/external-api",
+    "nexus/fm",
     "nexus/internal-api",
     "nexus/inventory",
     "nexus/lockstep-api",

dev-tools/omdb/src/bin/omdb/db/sitrep.rs

@@ -238,7 +238,7 @@ async fn cmd_db_sitrep_show(
         }
     };
 
-    let fm::Sitrep { metadata } = sitrep;
+    let fm::Sitrep { metadata, cases } = sitrep;
     let fm::SitrepMetadata {
         id,
         creator_id,

ereport/types/src/lib.rs

@@ -102,7 +102,9 @@ impl TryFrom<i64> for Ena {
 }
 
 /// Unique identifier for an ereport.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(
+    Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Ord,
+)]
 pub struct EreportId {
     pub restart_id: EreporterRestartUuid,
     pub ena: Ena,

nexus/db-model/src/alert_class.rs

@@ -30,6 +30,8 @@ impl_enum_type!(
     TestFooBaz => b"test.foo.baz"
     TestQuuxBar => b"test.quux.bar"
     TestQuuxBarBaz => b"test.quux.bar.baz"
+    PsuInserted => b"hw.insert.power.power_shelf.psu"
+    PsuRemoved => b"hw.remove.power.power_shelf.psu"
 );
 
 impl AlertClass {
@@ -44,6 +46,8 @@ impl AlertClass {
             Self::TestFooBaz => "test.foo.baz",
             Self::TestQuuxBar => "test.quux.bar",
             Self::TestQuuxBarBaz => "test.quux.bar.baz",
+            Self::PsuInserted => "hw.insert.power.power_shelf.psu",
+            Self::PsuRemoved => "hw.remove.power.power_shelf.psu",
         }
     }
 
@@ -76,6 +80,12 @@ impl AlertClass {
             | Self::TestQuuxBarBaz => {
                 "This is a test of the emergency alert system"
             }
+            Self::PsuInserted => {
+                "A power supply unit (PSU) has been inserted into the power shelf"
+            }
+            Self::PsuRemoved => {
+                "A power supply unit (PSU) has been removed from the power shelf"
+            }
         }
     }
 
@@ -84,6 +94,16 @@ impl AlertClass {
         <Self as strum::VariantArray>::VARIANTS;
 }
 
+impl From<nexus_types::fm::AlertClass> for AlertClass {
+    fn from(input: nexus_types::fm::AlertClass) -> Self {
+        use nexus_types::fm::AlertClass as In;
+        match input {
+            In::PsuRemoved => Self::PsuRemoved,
+            In::PsuInserted => Self::PsuInserted,
+        }
+    }
+}
+
 impl fmt::Display for AlertClass {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "{}", self.as_str())

nexus/db-queries/src/db/datastore/fm.rs

@@ -142,7 +142,11 @@ impl DataStore {
         // TODO(eliza): this is where we would read all the other sitrep data,
         // if there was any.
 
-        Ok(Sitrep { metadata })
+        Ok(Sitrep {
+            metadata,
+            // TODO(eliza) read these
+            cases: Default::default(),
+        })
     }
 
     /// Insert the provided [`Sitrep`] into the database, and attempt to mark it
@@ -755,6 +759,7 @@ mod tests {
                 time_created: Utc::now(),
                 parent_sitrep_id: None,
             },
+            cases: Default::default(),
         };
 
         datastore.fm_sitrep_insert(&opctx, &sitrep).await.unwrap();
@@ -801,6 +806,7 @@ mod tests {
                 time_created: Utc::now(),
                 parent_sitrep_id: None,
             },
+            cases: Default::default(),
         };
         datastore.fm_sitrep_insert(&opctx, &sitrep1).await.unwrap();
 
@@ -814,6 +820,7 @@ mod tests {
                 time_created: Utc::now(),
                 parent_sitrep_id: Some(sitrep1.id()),
             },
+            cases: Default::default(),
         };
         datastore.fm_sitrep_insert(&opctx, &sitrep2).await.expect(
             "inserting a sitrep whose parent is current should succeed",
@@ -854,6 +861,7 @@ mod tests {
                 time_created: Utc::now(),
                 parent_sitrep_id: None,
             },
+            cases: Default::default(),
         };
         datastore.fm_sitrep_insert(&opctx, &sitrep1).await.unwrap();
 
@@ -868,6 +876,7 @@ mod tests {
                 time_created: Utc::now(),
                 parent_sitrep_id: Some(nonexistent_id),
             },
+            cases: Default::default(),
         };
 
         let result = datastore.fm_sitrep_insert(&opctx, &sitrep2).await;
@@ -902,6 +911,7 @@ mod tests {
                 time_created: Utc::now(),
                 parent_sitrep_id: None,
             },
+            cases: Default::default(),
         };
         datastore.fm_sitrep_insert(&opctx, &sitrep1).await.unwrap();
 
@@ -915,6 +925,7 @@ mod tests {
                 time_created: Utc::now(),
                 parent_sitrep_id: Some(sitrep1.id()),
             },
+            cases: Default::default(),
         };
         datastore.fm_sitrep_insert(&opctx, &sitrep2).await.unwrap();
 
@@ -929,6 +940,7 @@ mod tests {
                 time_created: Utc::now(),
                 parent_sitrep_id: Some(sitrep1.id()),
             },
+            cases: Default::default(),
         };
         let result = datastore.fm_sitrep_insert(&opctx, &sitrep3).await;
 
@@ -969,6 +981,7 @@ mod tests {
                 time_created: Utc::now(),
                 parent_sitrep_id: None,
             },
+            cases: Default::default(),
         };
         datastore
             .fm_sitrep_insert(&opctx, &sitrep1)
@@ -1009,6 +1022,7 @@ mod tests {
                 time_created: Utc::now(),
                 parent_sitrep_id: Some(sitrep1.metadata.id),
             },
+            cases: Default::default(),
         };
         datastore
             .fm_sitrep_insert(&opctx, &sitrep2)
@@ -1072,6 +1086,7 @@ mod tests {
                 time_created: Utc::now(),
                 parent_sitrep_id,
             },
+            cases: Default::default(),
         };
         match datastore.fm_sitrep_insert(&opctx, &sitrep).await {
             Ok(_) => {

nexus/fm/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "nexus-fm"
+version = "0.1.0"
+edition = "2021"
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow.workspace = true
+chrono.workspace = true
+iddqd.workspace = true
+nexus-types.workspace = true
+omicron-uuid-kinds.workspace = true
+schemars.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+slog.workspace = true
+
+omicron-workspace-hack.workspace = true

nexus/fm/src/alert.rs

@@ -0,0 +1,22 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Alert messages.
+
+use nexus_types::fm::AlertClass;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+pub mod power_shelf;
+
+pub trait Alert: Serialize + JsonSchema + std::fmt::Debug {
+    const CLASS: AlertClass;
+}
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema)]
+pub struct VpdIdentity {
+    pub part_number: Option<String>,
+    pub revision: Option<String>,
+    pub serial_number: Option<String>,
+}

nexus/fm/src/alert/power_shelf.rs

@@ -0,0 +1,52 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Power shelf alerts.
+
+use super::{Alert, VpdIdentity};
+use nexus_types::fm::AlertClass;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema)]
+#[serde(tag = "version", rename_all = "snake_case")]
+pub enum PsuInserted {
+    V0 {
+        #[serde(flatten)]
+        psc_psu: PscPsu,
+    },
+}
+
+impl Alert for PsuInserted {
+    const CLASS: AlertClass = AlertClass::PsuInserted;
+}
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema)]
+#[serde(tag = "version", rename_all = "snake_case")]
+pub enum PsuRemoved {
+    V0 {
+        #[serde(flatten)]
+        psc_psu: PscPsu,
+    },
+}
+
+impl Alert for PsuRemoved {
+    const CLASS: AlertClass = AlertClass::PsuInserted;
+}
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema)]
+pub struct PscPsu {
+    pub psc_id: VpdIdentity,
+    pub psc_slot: u16,
+    pub psu_id: PsuIdentity,
+    pub psu_slot: u16,
+}
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema)]
+pub struct PsuIdentity {
+    pub manufacturer: Option<String>,
+    pub part_number: Option<String>,
+    pub firmware_revision: Option<String>,
+    pub serial_number: Option<String>,
+}

nexus/fm/src/de.rs

@@ -0,0 +1,7 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Diagnosis engines
+
+pub mod power_shelf;

nexus/fm/src/de/power_shelf.rs

@@ -0,0 +1,76 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Power shelf diagnosis
+
+use crate::SitrepBuilder;
+use crate::alert
+use nexus_types::fm::AlertRequest;
+use nexus_types::fm::DiagnosisEngine;
+use nexus_types::fm::Ereport;
+use nexus_types::fm::ereport;
+use nexus_types::inventory::SpType;
+use std::sync::Arc;
+
+pub fn diagnose(
+    sitrep: &mut SitrepBuilder<'_>,
+    new_ereports: &[Arc<Ereport>],
+) -> anyhow::Result<()> {
+    for ereport in new_ereports {
+        // Skip non-power shelf reports
+        let ereport::Reporter::Sp { sp_type: SpType::Power, slot, } = ereport.reporter else {
+            continue;
+        };
+
+        // TODO: check for existing cases tracked for this power shelf and see
+        // if the ereport is related to them...
+
+        match ereport.data.class.as_deref() {
+            // PSU inserted
+            Some("hw.insert.psu") => {
+                let mut case = sitrep.open_case(DiagnosisEngine::PowerShelf)?;
+                case.add_ereport(ereport);
+                case.comment = "PSU inserted".to_string();
+                let psu_id = match ereport.get("fruid") {
+                    Some(serde_json::Value::Object(fruid)) => {
+                        todo!()
+                    },
+                    None => {
+                        todo!()
+                    }
+                };
+                case.request_alert(alert::power_shelf::PsuInserted::V0 {
+                    psc_psu: alert::power_shelf::PscPsu {
+                        psc_id: alert::VpdIdentity {
+                            serial_number: ereport.serial_number.clone(),
+                            revision: ereport.report.get("baseboard_rev").map(ToString::to_string),
+                            part_number: ereport.part_number.clone(),
+                        },
+                        psc_slot: slot,
+                        psu_id,
+                        psu_slot: ereport.report.get("slot").map(|s| todo!()),
+                    }
+                })
+            }
+            Some("hw.remove.psu") => {}
+            Some(unknown) => {
+                slog::warn!(
+                    &sitrep.log,
+                    "ignoring unhandled PSC ereport class";
+                    "ereport_class" => %unknown,
+                    "ereport" => %ereport.id,
+                );
+            }
+            None => {
+                slog::warn!(
+                    &sitrep.log,
+                    "ignoring PSC ereport with no class";
+                    "ereport" => %ereport.id,
+                );
+            }
+        }
+    }
+
+    Ok(())
+}