Migrate guest time-related data

Author: jordanhendricks

Cargo.lock

@@ -118,6 +118,7 @@ softnpu-lib = { git = "https://github.com/oxidecomputer/softnpu", branch = "main
 syn = "1.0"
 tempfile = "3.2"
 thiserror = "1.0"
+time = { path = "crates/time" }
 tokio = "1"
 tokio-tungstenite = "0.17"
 tokio-util = "0.7"

Cargo.toml

@@ -64,15 +64,15 @@ pub fn build_instance(
     name: &str,
     spec: &InstanceSpec,
     use_reservoir: bool,
-    _log: slog::Logger,
+    log: slog::Logger,
 ) -> Result<Instance> {
     let (lowmem, highmem) = get_spec_guest_ram_limits(spec);
     let create_opts = propolis::vmm::CreateOpts {
         force: true,
         use_reservoir,
         track_dirty: true,
     };
-    let mut builder = Builder::new(name, create_opts)?
+    let mut builder = Builder::new(name, log.clone(), create_opts)?
         .max_cpus(spec.devices.board.cpus)?
         .add_mem_region(0, lowmem, "lowmem")?
         .add_rom_region(0x1_0000_0000 - MAX_ROM_SIZE, MAX_ROM_SIZE, "bootrom")?

bin/propolis-server/src/lib/initializer.rs

@@ -313,6 +313,26 @@ impl<T: AsyncRead + AsyncWrite + Unpin + Send> DestinationProtocol<T> {
                 }
             }
         }
+        self.send_msg(codec::Message::Okay).await?;
+
+        // Update VMM data
+        let vmm_state: String = match self.read_msg().await? {
+            codec::Message::Serialized(encoded) => encoded,
+            msg => {
+                error!(self.log(), "arch state: unexpected message: {msg:?}");
+                return Err(MigrateError::UnexpectedMessage);
+            }
+        };
+        info!(self.log(), "VMM State: {:?}", vmm_state);
+        {
+            let instance_guard = self.vm_controller.instance().lock();
+            let vmm_hdl = &instance_guard.machine().hdl;
+            let mut deserializer = ron::Deserializer::from_str(&vmm_state)
+                .map_err(codec::ProtocolError::from)?;
+            let deserializer =
+                &mut <dyn erased_serde::Deserializer>::erase(&mut deserializer);
+            vmm_hdl.import(deserializer)?;
+        }
 
         self.send_msg(codec::Message::Okay).await
     }

bin/propolis-server/src/lib/migrate/destination.rs

@@ -60,6 +60,18 @@ struct SourceProtocol<T: AsyncRead + AsyncWrite + Unpin + Send> {
 
     /// Transport to the destination Instance.
     conn: WebSocketStream<T>,
+
+    // TODO: A hacky way for now to hang onto the VMM timing data between
+    // migration phases.
+    //
+    // We want to read the VMM timing data as soon as we can after we pause the
+    // source, and make adjustments on the destination as close as we can to
+    // the end of the migration.
+    //
+    // This lets us hang on to the data between export in the pause phase,
+    // the send the data in the device_state phase, after the bulk of the
+    // migration time has passed.
+    vmm_data: Option<propolis::vmm::migrate::BhyveVmV1>,
 }
 
 impl<T: AsyncRead + AsyncWrite + Unpin + Send> SourceProtocol<T> {
@@ -69,7 +81,7 @@ impl<T: AsyncRead + AsyncWrite + Unpin + Send> SourceProtocol<T> {
         response_rx: tokio::sync::mpsc::Receiver<MigrateSourceResponse>,
         conn: WebSocketStream<T>,
     ) -> Self {
-        Self { vm_controller, command_tx, response_rx, conn }
+        Self { vm_controller, command_tx, response_rx, conn, vmm_data: None }
     }
 
     fn log(&self) -> &slog::Logger {
@@ -233,16 +245,28 @@ impl<T: AsyncRead + AsyncWrite + Unpin + Send> SourceProtocol<T> {
         info!(self.log(), "Pausing devices");
         self.command_tx.send(MigrateSourceCommand::Pause).await.unwrap();
         let resp = self.response_rx.recv().await.unwrap();
-        match resp {
-            MigrateSourceResponse::Pause(Ok(())) => Ok(()),
-            _ => {
-                info!(
-                    self.log(),
-                    "Unexpected pause response from state worker: {:?}", resp
-                );
-                Err(MigrateError::SourcePause)
-            }
+        if !matches!(resp, MigrateSourceResponse::Pause(Ok(()))) {
+            info!(
+                self.log(),
+                "Unexpected pause response from state worker: {:?}", resp
+            );
+            return Err(MigrateError::SourcePause);
         }
+
+        // Fetch the timing data values after we pause, but before lengthier
+        // migration steps, so we can correctly account for migration time in
+        // our adjustments.
+        self.timing_data_snapshot().await
+    }
+
+    async fn timing_data_snapshot(&mut self) -> Result<(), MigrateError> {
+        let instance_guard = self.vm_controller.instance().lock();
+        let vmm_hdl = &instance_guard.machine().hdl;
+        let raw = vmm_hdl.export_vm()?;
+        self.vmm_data = Some(raw);
+        info!(self.log(), "VMM State: {:#?}", self.vmm_data);
+
+        Ok(())
     }
 
     async fn device_state(&mut self) -> Result<(), MigrateError> {
@@ -286,6 +310,14 @@ impl<T: AsyncRead + AsyncWrite + Unpin + Send> SourceProtocol<T> {
         .await?;
 
         self.send_msg(codec::Message::Okay).await?;
+        self.read_ok().await?;
+
+        // Migrate VMM-wide data
+        let vmm_data = self.vmm_data.as_mut().unwrap();
+        let vmm_state = ron::ser::to_string(&vmm_data)
+            .map_err(codec::ProtocolError::from)?;
+        info!(self.log(), "VMM State: {:#?}", vmm_state);
+        self.send_msg(codec::Message::Serialized(vmm_state)).await?;
         self.read_ok().await
     }
 

bin/propolis-server/src/lib/migrate/source.rs

@@ -600,12 +600,14 @@ impl std::ops::DerefMut for InnerGuard<'_> {
 
 fn build_instance(
     name: &str,
+    log: slog::Logger,
     max_cpu: u8,
     lowmem: usize,
     highmem: usize,
 ) -> Result<propolis::Instance> {
     let mut builder = Builder::new(
         name,
+        log,
         propolis::vmm::CreateOpts { force: true, ..Default::default() },
     )?
     .max_cpus(max_cpu)?
@@ -687,7 +689,7 @@ fn setup_instance(
 
     slog::info!(log, "Creating VM with {} vCPUs, {} lowmem, {} highmem",
         cpus, lowmem, highmem;);
-    let pinst = build_instance(vm_name, cpus, lowmem, highmem)
+    let pinst = build_instance(vm_name, log.clone(), cpus, lowmem, highmem)
         .context("Failed to create VM Instance")?;
     let inst = Instance::new(pinst, config.clone(), from_restore, log.clone());
     slog::info!(log, "VM created"; "name" => vm_name);

bin/propolis-standalone/src/main.rs

@@ -230,6 +230,9 @@ unsafe fn ioctl(
 #[repr(u32)]
 #[derive(IntoPrimitive)]
 pub enum ApiVersion {
+    /// Adds support for modifying timing-related data of a guest.
+    V10 = 10,
+
     /// Revamps ioctls for administrating the VMM memory reservoir and adds
     /// kstat for tracking its capacity and utilization.
     V9 = 9,
@@ -250,7 +253,7 @@ pub enum ApiVersion {
 }
 impl ApiVersion {
     pub const fn current() -> Self {
-        Self::V9
+        Self::V10
     }
 }
 

crates/bhyve-api/src/lib.rs

@@ -13,4 +13,4 @@ pub const VM_MAXCPU: usize = 32;
 /// This is the VMM interface version which bhyve_api expects to operate
 /// against.  All constants and structs defined by the crate are done so in
 /// terms of that specific version.
-pub const VMM_CURRENT_INTERFACE_VERSION: u32 = 9;
+pub const VMM_CURRENT_INTERFACE_VERSION: u32 = 10;

crates/bhyve-api/sys/src/lib.rs

@@ -21,6 +21,7 @@ pub const VDC_ATPIC: u16 = 9;
 pub const VDC_HPET: u16 = 10;
 pub const VDC_PM_TIMER: u16 = 11;
 pub const VDC_RTC: u16 = 12;
+pub const VDC_VMM_TIMING: u16 = 13;
 
 #[repr(C)]
 #[derive(Copy, Clone, Default, Serialize, Deserialize)]
@@ -189,7 +190,19 @@ impl Default for vdi_rtc_v1 {
     }
 }
 
+#[repr(C)]
+#[derive(Copy, Clone, Default, Serialize, Deserialize)]
+pub struct vdi_timing_info_v1 {
+    pub vt_guest_freq: u64,
+    pub vt_guest_tsc: u64,
+    pub vt_boot_hrtime: i64,
+    pub vt_hrtime: i64,
+    pub vt_hres_sec: u64,
+    pub vt_hres_ns: u64,
+}
+
 // VDC_VMM_ARCH v1 data identifiers
+// TODO: fix this up when illumos 15143 lands
 
 /// Offset of guest TSC from system at time of boot
 pub const VAI_TSC_BOOT_OFFSET: u32 = 1;

crates/bhyve-api/sys/src/vmm_data.rs

@@ -0,0 +1,9 @@
+[package]
+name = "time"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+libc.workspace = true