diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index 8503f789816..1f610ea98e4 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -51,3 +51,10 @@ mode = "stderr-terminal" # Configuration for interacting with the timeseries database [timeseries_db] address = "[::1]:8123" + +# Tunable configuration parameters, for testing or experimentation +[tunables] + +# The maximum allowed prefix (thus smallest size) for a VPC Subnet's +# IPv4 subnetwork. This size allows for ~60 hosts. +max_vpc_ipv4_subnet_prefix = 26 diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 71eb3dfdca8..ce20065fa1f 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -84,6 +84,9 @@ pub struct Nexus { /// Contents of the trusted root role for the TUF repository. updates_config: Option, + /// The tunable parameters from a configuration file + tunables: config::Tunables, + /// Operational context used for Instance allocation opctx_alloc: OpContext, @@ -151,6 +154,7 @@ impl Nexus { populate_status, timeseries_client, updates_config: config.updates.clone(), + tunables: config.tunables.clone(), opctx_alloc: OpContext::for_background( log.new(o!("component" => "InstanceAllocator")), Arc::clone(&authz), @@ -191,6 +195,11 @@ impl Nexus { nexus } + /// Return the tunable configuration parameters, e.g. for use in tests. + pub fn tunables(&self) -> &config::Tunables { + &self.tunables + } + pub async fn wait_for_populate(&self) -> Result<(), anyhow::Error> { let mut my_rx = self.populate_status.clone(); loop { diff --git a/nexus/src/app/vpc_subnet.rs b/nexus/src/app/vpc_subnet.rs index f75965cb426..c996ae485f7 100644 --- a/nexus/src/app/vpc_subnet.rs +++ b/nexus/src/app/vpc_subnet.rs @@ -49,7 +49,8 @@ impl super::Nexus { )); } if params.ipv4_block.prefix() < defaults::MIN_VPC_IPV4_SUBNET_PREFIX - || params.ipv4_block.prefix() > defaults::MAX_VPC_IPV4_SUBNET_PREFIX + || params.ipv4_block.prefix() + > self.tunables.max_vpc_ipv4_subnet_prefix { return Err(external::Error::invalid_request(&format!( concat!( @@ -57,7 +58,7 @@ impl super::Nexus { "length between {} and {}, inclusive" ), defaults::MIN_VPC_IPV4_SUBNET_PREFIX, - defaults::MAX_VPC_IPV4_SUBNET_PREFIX + self.tunables.max_vpc_ipv4_subnet_prefix, ))); } diff --git a/nexus/src/config.rs b/nexus/src/config.rs index 64501288243..11b2c8d861e 100644 --- a/nexus/src/config.rs +++ b/nexus/src/config.rs @@ -51,6 +51,77 @@ pub struct TimeseriesDbConfig { pub address: SocketAddr, } +// A deserializable type that does no validation on the tunable parameters. +#[derive(Clone, Debug, Deserialize, PartialEq)] +struct UnvalidatedTunables { + max_vpc_ipv4_subnet_prefix: u8, +} + +/// Tunable configuration parameters, intended for use in test environments or +/// other situations in which experimentation / tuning is valuable. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(try_from = "UnvalidatedTunables")] +pub struct Tunables { + /// The maximum prefix size supported for VPC Subnet IPv4 subnetworks. + /// + /// Note that this is the maximum _prefix_ size, which sets the minimum size + /// of the subnet. + pub max_vpc_ipv4_subnet_prefix: u8, +} + +// Convert from the unvalidated tunables, verifying each parameter as needed. +impl TryFrom for Tunables { + type Error = InvalidTunable; + + fn try_from(unvalidated: UnvalidatedTunables) -> Result { + Tunables::validate_ipv4_prefix(unvalidated.max_vpc_ipv4_subnet_prefix)?; + Ok(Tunables { + max_vpc_ipv4_subnet_prefix: unvalidated.max_vpc_ipv4_subnet_prefix, + }) + } +} + +impl Tunables { + fn validate_ipv4_prefix(prefix: u8) -> Result<(), InvalidTunable> { + let absolute_max: u8 = 32_u8.checked_sub( + // Always need space for the reserved Oxide addresses, including the + // broadcast address at the end of the subnet. + ((crate::defaults::NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) as f32) + .log2() // Subnet size to bit prefix. + .ceil() // Round up to a whole number of bits. + as u8 + ).expect("Invalid absolute maximum IPv4 subnet prefix"); + if prefix >= crate::defaults::MIN_VPC_IPV4_SUBNET_PREFIX + && prefix <= absolute_max + { + Ok(()) + } else { + Err(InvalidTunable { + tunable: String::from("max_vpc_ipv4_subnet_prefix"), + message: format!( + "IPv4 subnet prefix must be in the range [0, {}], found: {}", + absolute_max, + prefix, + ), + }) + } + } +} + +/// The maximum prefix size by default. +/// +/// There are 6 Oxide reserved IP addresses, 5 at the beginning for DNS and the +/// like, and the broadcast address at the end of the subnet. This size provides +/// room for 2 ** 6 - 6 = 58 IP addresses, which seems like a reasonable size +/// for the smallest subnet that's still useful in many contexts. +pub const MAX_VPC_IPV4_SUBNET_PREFIX: u8 = 26; + +impl Default for Tunables { + fn default() -> Self { + Tunables { max_vpc_ipv4_subnet_prefix: MAX_VPC_IPV4_SUBNET_PREFIX } + } +} + /// Configuration for a nexus server #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Config { @@ -74,8 +145,25 @@ pub struct Config { /// unconfigured. #[serde(default)] pub updates: Option, + /// Tunable configuration for testing and experimentation + #[serde(default)] + pub tunables: Tunables, +} + +#[derive(Debug)] +pub struct InvalidTunable { + tunable: String, + message: String, +} + +impl std::fmt::Display for InvalidTunable { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "invalid \"{}\": \"{}\"", self.tunable, self.message) + } } +impl std::error::Error for InvalidTunable {} + #[derive(Debug)] pub struct LoadError { path: PathBuf, @@ -85,6 +173,7 @@ pub struct LoadError { pub enum LoadErrorKind { Io(std::io::Error), Parse(toml::de::Error), + InvalidTunable(InvalidTunable), } impl From<(PathBuf, std::io::Error)> for LoadError { @@ -110,6 +199,14 @@ impl fmt::Display for LoadError { LoadErrorKind::Parse(e) => { write!(f, "parse \"{}\": {}", self.path.display(), e) } + LoadErrorKind::InvalidTunable(inner) => { + write!( + f, + "invalid tunable \"{}\": {}", + self.path.display(), + inner, + ) + } } } } @@ -175,6 +272,7 @@ impl Config { #[cfg(test)] mod test { + use super::Tunables; use super::{ AuthnConfig, Config, ConsoleConfig, LoadError, LoadErrorKind, SchemeName, TimeseriesDbConfig, UpdatesConfig, @@ -301,6 +399,8 @@ mod test { [updates] trusted_root = "/path/to/root.json" default_base_url = "http://example.invalid/" + [tunables] + max_vpc_ipv4_subnet_prefix = 27 "##, ) .unwrap(); @@ -345,6 +445,7 @@ mod test { trusted_root: PathBuf::from("/path/to/root.json"), default_base_url: "http://example.invalid/".into(), }), + tunables: Tunables { max_vpc_ipv4_subnet_prefix: 27 }, } ); @@ -427,4 +528,52 @@ mod test { ); } } + + #[test] + fn test_invalid_ipv4_prefix_tunable() { + let error = read_config( + "invalid_ipv4_prefix_tunable", + r##" + id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" + [console] + static_dir = "tests/static" + cache_control_max_age_minutes = 10 + session_idle_timeout_minutes = 60 + session_absolute_timeout_minutes = 480 + [authn] + schemes_external = [] + [dropshot_external] + bind_address = "10.1.2.3:4567" + request_body_max_bytes = 1024 + [dropshot_internal] + bind_address = "10.1.2.3:4568" + request_body_max_bytes = 1024 + [database] + url = "postgresql://127.0.0.1?sslmode=disable" + [log] + mode = "file" + level = "debug" + path = "/nonexistent/path" + if_exists = "fail" + [timeseries_db] + address = "[::1]:8123" + [updates] + trusted_root = "/path/to/root.json" + default_base_url = "http://example.invalid/" + [tunables] + max_vpc_ipv4_subnet_prefix = 100 + "##, + ) + .expect_err("Expected failure"); + if let LoadErrorKind::Parse(error) = &error.kind { + assert!(error.to_string().starts_with( + r#"invalid "max_vpc_ipv4_subnet_prefix": "IPv4 subnet prefix must"#, + )); + } else { + panic!( + "Got an unexpected error, expected Parse but got {:?}", + error + ); + } + } } diff --git a/nexus/src/defaults.rs b/nexus/src/defaults.rs index d7c22abfc04..15769c86611 100644 --- a/nexus/src/defaults.rs +++ b/nexus/src/defaults.rs @@ -18,11 +18,6 @@ use std::net::Ipv6Addr; /// NOTE: This is the minimum _prefix_, which sets the maximum subnet size. pub const MIN_VPC_IPV4_SUBNET_PREFIX: u8 = 8; -/// Maximum prefix size supported in IPv4 VPC Subnets. -/// -/// NOTE: This is the maximum _prefix_, which sets the minimum subnet size. -pub const MAX_VPC_IPV4_SUBNET_PREFIX: u8 = 26; - /// The number of reserved addresses at the beginning of a subnet range. pub const NUM_INITIAL_RESERVED_IP_ADDRESSES: usize = 5; diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index 5ae440e6620..9b8f1f42731 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -54,3 +54,8 @@ path = "UNUSED" # is listening. [timeseries_db] address = "[::1]:0" + +# Tunable parameters used during tests +[tunables] +# Allow small subnets, so we can test IP address exhaustion easily / quickly +max_vpc_ipv4_subnet_prefix = 29 diff --git a/nexus/tests/integration_tests/subnet_allocation.rs b/nexus/tests/integration_tests/subnet_allocation.rs index 4b7a83e8f8b..0b45389968a 100644 --- a/nexus/tests/integration_tests/subnet_allocation.rs +++ b/nexus/tests/integration_tests/subnet_allocation.rs @@ -9,6 +9,7 @@ use dropshot::test_util::ClientTestContext; use dropshot::HttpErrorResponseBody; use http::method::Method; use http::StatusCode; +use ipnetwork::Ipv4Network; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; @@ -23,6 +24,7 @@ use omicron_common::api::external::{ }; use omicron_nexus::defaults::NUM_INITIAL_RESERVED_IP_ADDRESSES; use omicron_nexus::external_api::params; +use std::net::Ipv4Addr; async fn create_instance_expect_failure( client: &ClientTestContext, @@ -87,12 +89,16 @@ async fn test_subnet_allocation(cptestctx: &ControlPlaneTestContext) { // Create a new, small VPC Subnet, so we don't need to issue many requests // to test address exhaustion. + let subnet_size = + cptestctx.server.apictx.nexus.tunables().max_vpc_ipv4_subnet_prefix; let url_subnets = format!( "/organizations/{}/projects/{}/vpcs/default/subnets", organization_name, project_name ); let subnet_name = "small"; - let subnet = "192.168.42.0/26".parse().unwrap(); + let network_address = Ipv4Addr::new(192, 168, 42, 0); + let subnet = Ipv4Network::new(network_address, subnet_size) + .expect("Invalid IPv4 network"); let subnet_create = params::VpcSubnetCreate { identity: IdentityMetadataCreateParams { name: subnet_name.parse().unwrap(), @@ -122,9 +128,8 @@ async fn test_subnet_allocation(cptestctx: &ControlPlaneTestContext) { }, ]); - // Create enough instances to fill the subnet. There should be 58 instances - // total created here. The subnet is a /26, and there are 6 reserved - // addresses. So 2 ** (32 - 26) - 6 = 2 ** 6 - 6 = 64 - 6 = 58. + // Create enough instances to fill the subnet. There are subnet.size() total + // addresses, 6 of which are reserved. let n_final_reserved_addresses = 1; let n_reserved_addresses = NUM_INITIAL_RESERVED_IP_ADDRESSES + n_final_reserved_addresses;