[nexus] Refuse to boot until Nexus finds a matching DB schema (#3531)

- Nexus now reads the `db_metadata` table, ensuring that the schema
"in-the-DB" matches the schema "in Nexus". At the moment, we're being
picky, and we require an exact match.
- Adds tests validating that we refuse to boot if the schema does not
match
- Add tests validating that we **eventually** boot Nexus if the schema
is corrected

Author: smklein

Cargo.lock

@@ -2510,7 +2510,7 @@ CREATE TABLE omicron.public.switch_port_settings_address_config (
  */
 
 CREATE TABLE omicron.public.db_metadata (
-    name  STRING(63) NOT NULL,
+    name  STRING(63) NOT NULL PRIMARY KEY,
     value STRING(1023) NOT NULL
 );
 

common/src/sql/dbinit.sql

@@ -60,6 +60,7 @@ reqwest = { workspace = true, features = ["json"] }
 ring.workspace = true
 samael.workspace = true
 schemars = { workspace = true, features = ["chrono", "uuid1"] }
+semver.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 serde_urlencoded.workspace = true

nexus/Cargo.toml

@@ -0,0 +1,27 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use crate::schema::db_metadata;
+
+/// Internal database metadata
+#[derive(Queryable, Insertable, Debug, Clone, Selectable)]
+#[diesel(table_name = db_metadata)]
+pub struct DbMetadata {
+    name: String,
+    value: String,
+}
+
+impl DbMetadata {
+    pub fn new(name: String, value: String) -> Self {
+        Self { name, value }
+    }
+
+    pub fn name(&self) -> &str {
+        &self.name
+    }
+
+    pub fn value(&self) -> &str {
+        &self.value
+    }
+}

nexus/db-model/src/db_metadata.rs

@@ -18,6 +18,7 @@ mod collection;
 mod console_session;
 mod dataset;
 mod dataset_kind;
+mod db_metadata;
 mod device_auth;
 mod digest;
 mod disk;
@@ -104,6 +105,7 @@ pub use collection::*;
 pub use console_session::*;
 pub use dataset::*;
 pub use dataset_kind::*;
+pub use db_metadata::*;
 pub use device_auth::*;
 pub use digest::*;
 pub use disk::*;

nexus/db-model/src/lib.rs

@@ -6,6 +6,8 @@
 //!
 //! NOTE: Should be kept up-to-date with dbinit.sql.
 
+use omicron_common::api::external::SemverVersion;
+
 table! {
     disk (id) {
         id -> Uuid,
@@ -1109,6 +1111,17 @@ table! {
     }
 }
 
+table! {
+    db_metadata (name) {
+        name -> Text,
+        value -> Text,
+    }
+}
+
+/// The version of the database schema this particular version of Nexus was
+/// built against.
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(1, 0, 0);
+
 allow_tables_to_appear_in_same_query!(
     system_update,
     component_update,

nexus/db-model/src/schema.rs

@@ -0,0 +1,36 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! [`DataStore`] methods on Database Metadata.
+
+use super::DataStore;
+use crate::db;
+use crate::db::error::public_error_from_diesel_pool;
+use crate::db::error::ErrorHandler;
+use async_bb8_diesel::AsyncRunQueryDsl;
+use diesel::prelude::*;
+use omicron_common::api::external::Error;
+use omicron_common::api::external::SemverVersion;
+use std::str::FromStr;
+
+impl DataStore {
+    pub async fn database_schema_version(
+        &self,
+    ) -> Result<SemverVersion, Error> {
+        use db::schema::db_metadata::dsl;
+
+        let version: String = dsl::db_metadata
+            .filter(dsl::name.eq("schema_version"))
+            .select(dsl::value)
+            .get_result_async(self.pool())
+            .await
+            .map_err(|e| {
+                public_error_from_diesel_pool(e, ErrorHandler::Server)
+            })?;
+
+        SemverVersion::from_str(&version).map_err(|e| {
+            Error::internal_error(&format!("Invalid schema version: {e}"))
+        })
+    }
+}

nexus/db-queries/src/db/datastore/db_metadata.rs

@@ -37,6 +37,11 @@ use omicron_common::api::external::Error;
 use omicron_common::api::external::IdentityMetadataCreateParams;
 use omicron_common::api::external::LookupType;
 use omicron_common::api::external::ResourceType;
+use omicron_common::api::external::SemverVersion;
+use omicron_common::backoff::{
+    retry_notify, retry_policy_internal_service, BackoffError,
+};
+use slog::Logger;
 use std::net::Ipv6Addr;
 use std::sync::Arc;
 use uuid::Uuid;
@@ -45,6 +50,7 @@ mod address_lot;
 mod certificate;
 mod console_session;
 mod dataset;
+mod db_metadata;
 mod device_auth;
 mod disk;
 mod dns;
@@ -131,12 +137,40 @@ pub struct DataStore {
 // to compilation times; changing a query only requires incremental
 // recompilation of that query's module instead of all queries on `DataStore`.
 impl DataStore {
-    pub fn new(pool: Arc<Pool>) -> Self {
-        DataStore {
+    /// Constructs a new Datastore object.
+    ///
+    /// Only returns if the database schema is compatible with Nexus's known
+    /// schema version.
+    pub async fn new(log: &Logger, pool: Arc<Pool>) -> Result<Self, String> {
+        let datastore = DataStore {
             pool,
             virtual_provisioning_collection_producer:
                 crate::provisioning::Producer::new(),
-        }
+        };
+
+        // Keep looping until we find that the schema matches our expectation.
+        const EXPECTED_VERSION: SemverVersion = SemverVersion::new(1, 0, 0);
+        retry_notify(
+            retry_policy_internal_service(),
+            || async {
+                match datastore.database_schema_version().await {
+                    Ok(version) => {
+                        if version == nexus_db_model::schema::SCHEMA_VERSION {
+                            return Ok(());
+                        }
+                        let observed = version.0;
+                        warn!(log, "Incompatible database schema: Saw {observed}, expected {EXPECTED_VERSION}");
+                    }
+                    Err(e) => {
+                        warn!(log, "Cannot read database schema version: {e}");
+                    }
+                };
+                return Err(BackoffError::transient(()));
+            },
+            |_, _| {},
+        ).await.map_err(|_| "Failed to read valid DB schema".to_string())?;
+
+        Ok(datastore)
     }
 
     pub fn register_producers(&self, registry: &ProducerRegistry) {
@@ -248,7 +282,7 @@ pub async fn datastore_test(
 
     let cfg = db::Config { url: db.pg_config().clone() };
     let pool = Arc::new(db::Pool::new(&logctx.log, &cfg));
-    let datastore = Arc::new(DataStore::new(pool));
+    let datastore = Arc::new(DataStore::new(&logctx.log, pool).await.unwrap());
 
     // Create an OpContext with the credentials of "db-init" just for the
     // purpose of loading the built-in users, roles, and assignments.
@@ -907,7 +941,8 @@ mod test {
         let mut db = test_setup_database(&logctx.log).await;
         let cfg = db::Config { url: db.pg_config().clone() };
         let pool = db::Pool::new(&logctx.log, &cfg);
-        let datastore = DataStore::new(Arc::new(pool));
+        let datastore =
+            DataStore::new(&logctx.log, Arc::new(pool)).await.unwrap();
 
         let explanation = DataStore::get_allocated_regions_query(Uuid::nil())
             .explain_async(datastore.pool())
@@ -956,7 +991,8 @@ mod test {
         let mut db = test_setup_database(&logctx.log).await;
         let cfg = db::Config { url: db.pg_config().clone() };
         let pool = Arc::new(db::Pool::new(&logctx.log, &cfg));
-        let datastore = Arc::new(DataStore::new(Arc::clone(&pool)));
+        let datastore =
+            Arc::new(DataStore::new(&logctx.log, pool).await.unwrap());
         let opctx =
             OpContext::for_tests(logctx.log.new(o!()), datastore.clone());
 

nexus/db-queries/src/db/datastore/mod.rs

@@ -851,8 +851,11 @@ mod tests {
             crate::db::datastore::datastore_test(&logctx, &db).await;
             let cfg = crate::db::Config { url: db.pg_config().clone() };
             let pool = Arc::new(crate::db::Pool::new(&logctx.log, &cfg));
-            let db_datastore =
-                Arc::new(crate::db::DataStore::new(Arc::clone(&pool)));
+            let db_datastore = Arc::new(
+                crate::db::DataStore::new(&logctx.log, Arc::clone(&pool))
+                    .await
+                    .unwrap(),
+            );
             let opctx =
                 OpContext::for_tests(log.new(o!()), db_datastore.clone());
             Self { logctx, opctx, db, db_datastore }

nexus/db-queries/src/db/queries/external_ip.rs

@@ -444,8 +444,9 @@ mod test {
         let mut db = test_setup_database(&log).await;
         let cfg = crate::db::Config { url: db.pg_config().clone() };
         let pool = Arc::new(crate::db::Pool::new(&logctx.log, &cfg));
-        let db_datastore =
-            Arc::new(crate::db::DataStore::new(Arc::clone(&pool)));
+        let db_datastore = Arc::new(
+            crate::db::DataStore::new(&log, Arc::clone(&pool)).await.unwrap(),
+        );
 
         // We should be able to insert anything into an empty table.
         assert!(