Slightly change IPv6 prefix used and update README with address mappings.

Author: luqmana

README.adoc

@@ -437,6 +437,33 @@ executed:
 $ sudo ./target/debug/omicron-package uninstall
 ----
 
+=== Test Environment
+
+When we deploy, we're effectively creating a number of different zones
+for all the components that make up Omicron (Nexus, Clickhouse, Crucible, etc).
+Since all these services run in different zones they cannot communicate with
+each other (and Sled Agent in the global zone) via `localhost`. In practise,
+we'll assign addresses as per RFD 63 as well as incorporating DNS based
+service discovery.
+
+For the purposes of local development today, we specify some hardcoded IPv6
+unique local addresses in `fd00:1de::/16`:
+
+[options="header"]
+|===================================================================================================
+| Service                    | Endpoint                                                             
+| Sled Agent: Bootstrap      | `[::]:12346`                                                         
+| Sled Agent: Dropshot API   | `[fd00:1de::]:12345`                                                 
+| Cockroach DB               | `[fd00:1de::5]:32221`                                                
+| Oximeter                   | `[fd00:1de::6]:12223`                                                
+| Nexus: External API        | `[fd00:1de::7]:12220`                                                
+| Nexus: Internal API        | `[fd00:1de::7]:12221`                                                
+| Clickhouse                 | `[fd00:1de::8]:8123`                                                 
+| Crucible Downstairs        | `[fd00:1de::9]:32345`, `[fd00:1de::10]:32345`, `[fd00:1de::11]:32345`
+|===================================================================================================
+
+Note that Sled Agent runs in the global zone and is the one responsible for bringing up all the other
+other services and allocating them with vNICs and IPv6 addresses.
 
 == Configuration reference
 

sled-agent/src/illumos/zone.rs

@@ -428,10 +428,7 @@ impl Zones {
         Self::ensure_address(
             None,
             &gz_link_local_addrobj.on_same_interface("v6route")?,
-            AddressRequest::new_static(
-                "fd00:1234::".parse().unwrap(),
-                Some(16),
-            ),
+            AddressRequest::new_static("fd00:1de::".parse().unwrap(), Some(16)),
         )?;
         Ok(())
     }

smf/nexus/config.toml

@@ -18,15 +18,15 @@ schemes_external = ["spoof", "session_cookie"]
 
 [database]
 # URL for connecting to the database
-url = "postgresql://root@[fd00:1234::5]:32221/omicron?sslmode=disable"
+url = "postgresql://root@[fd00:1de::5]:32221/omicron?sslmode=disable"
 
 [dropshot_external]
 # IP address and TCP port on which to listen for the external API
-bind_address = "192.168.1.123:12220"
+bind_address = "[fd00:1de::7]:12220"
 
 [dropshot_internal]
 # IP address and TCP port on which to listen for the internal API
-bind_address = "[fd00:1234::7]:12221"
+bind_address = "[fd00:1de::7]:12221"
 
 [log]
 # Show log messages of this level and more severe
@@ -42,4 +42,4 @@ mode = "stderr-terminal"
 
 # Configuration for interacting with the timeseries database
 [timeseries_db]
-address = "[fd00:1234::8]:8123"
+address = "[fd00:1de::8]:8123"

smf/oximeter/config.toml

@@ -2,10 +2,10 @@
 
 id = "1da65e5b-210c-4859-a7d7-200c1e659972"
 # Internal address of nexus
-nexus_address = "[fd00:1234::7]:12221"
+nexus_address = "[fd00:1de::7]:12221"
 
 [db]
-address = "[fd00:1234::8]:8123"
+address = "[fd00:1de::8]:8123"
 batch_size = 1000
 batch_interval = 5 # In seconds
 
@@ -14,4 +14,4 @@ level = "debug"
 mode = "stderr-terminal"
 
 [dropshot]
-bind_address = "[fd00:1234::6]:12223"
+bind_address = "[fd00:1de::6]:12223"

smf/sled-agent/config-rss.toml

@@ -1,40 +1,40 @@
 # RSS (Rack Setup Service) "stand-in" configuration.
 
 [[request]]
-sled_address = "192.168.1.201:12345"
+sled_address = "[fd00:1de::]:12345"
 
 [[request.partition]]
 zpool_uuid = "d462a7f7-b628-40fe-80ff-4e4189e2d62b"
-address = "[fd00:1234::9]:32345"
+address = "[fd00:1de::9]:32345"
 partition_kind.type = "crucible"
 
 [[request.partition]]
 zpool_uuid = "e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
-address = "[fd00:1234::10]:32345"
+address = "[fd00:1de::10]:32345"
 partition_kind.type = "crucible"
 
 [[request.partition]]
 zpool_uuid = "f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
-address = "[fd00:1234::11]:32345"
+address = "[fd00:1de::11]:32345"
 partition_kind.type = "crucible"
 
 [[request.partition]]
 zpool_uuid = "d462a7f7-b628-40fe-80ff-4e4189e2d62b"
-address = "[fd00:1234::5]:32221"
+address = "[fd00:1de::5]:32221"
 partition_kind.type = "cockroach_db"
 partition_kind.all_addresses = [
-  "[fd00:1234::5]:32221",
+  "[fd00:1de::5]:32221",
 ]
 
 [[request.partition]]
 zpool_uuid = "d462a7f7-b628-40fe-80ff-4e4189e2d62b"
-address = "[fd00:1234::8]:8123"
+address = "[fd00:1de::8]:8123"
 partition_kind.type = "clickhouse"
 
 [[request.service]]
 name = "nexus"
-addresses = [ "[fd00:1234::7]:12221", "192.168.1.123:12220" ]
+addresses = [ "[fd00:1de::7]:12220", "[fd00:1de::7]:12221" ]
 
 [[request.service]]
 name = "oximeter"
-addresses = [ "[fd00:1234::6]:12223" ]
+addresses = [ "[fd00:1de::6]:12223" ]

smf/sled-agent/config.toml

@@ -4,7 +4,7 @@ id = "fb0f7546-4d46-40ca-9d56-cbb810684ca7"
 
 bootstrap_address = "[::]:12346"
 # Internal address of Nexus
-nexus_address = "[fd00:1234::7]:12221"
+nexus_address = "[fd00:1de::7]:12221"
 
 # A file-backed zpool can be manually created with the following:
 # $ truncate -s 10GB testpool.vdev
@@ -25,16 +25,8 @@ zpools = [
 # With the usage of non-global zones, we no longer can use localhost addresses,
 # as Nexus (within a Zone) is effectively "on a different machine" from the
 # sled agent.
-#
-# This address may be allocated with the following command:
-#
-# pfexec ipadm create-addr -t -T static -a 192.168.1.201 $(dladm show-phys -p -o LINK)/sled4
-# |      |                                               |
-# Please |                                               |
-#        Create a temporary static address               |
-#                                                        On whatever physical datalink I have
 [dropshot]
-bind_address = "192.168.1.201:12345"
+bind_address = "[fd00:1de::]:12345"
 
 [log]
 level = "info"