RDMA no longer needs torch

Pull Request resolved: #2045

This replaces the internal C++ APIs to torch with an interace for injecting the scanned segments. We then use a pyo3 function that indirectly calls the python torch.cuda.memory._snapshot function to get the same information.

So we no longer need rdma to directly link torch.
ghstack-source-id: 327462736
@exported-using-ghexport

Differential Revision: [D88338646](https://our.internmc.facebook.com/intern/diff/D88338646/)

Author: zdevito

.github/workflows/test-gpu-rust.yml

@@ -68,6 +68,7 @@ jobs:
           --exclude monarch_tensor_worker \
           --exclude torch-sys \
           --exclude torch-sys-cuda
+          --exclude monarch_rdma
         # Copy the test results to the expected location
         # TODO: error in pytest-results-action, TypeError: results.testsuites.testsuite.testcase is not iterable
         # Don't try to parse these results for now.

Cargo.toml

@@ -2,7 +2,6 @@
 resolver = "2"
 members = [
     "build_utils",
-    "cuda-sys",
     "erased_lifetime",
     "hyper",
     "hyperactor",

cuda-sys/Cargo.toml

@@ -13,7 +13,6 @@ edition = "2024"
 [dependencies]
 anyhow = "1.0.98"
 async-trait = "0.1.86"
-cuda-sys = { path = "../cuda-sys" }
 futures = { version = "0.3.31", features = ["async-await", "compat"] }
 hyperactor = { version = "0.0.0", path = "../hyperactor" }
 rand = { version = "0.8", features = ["small_rng"] }

cuda-sys/build.rs

@@ -407,7 +407,7 @@ impl Handler<InitializeBuffer> for CudaRdmaActor {
                 self.device_id as i32
             ));
             cu_check!(rdmaxcel_sys::rdmaxcel_cuCtxSetCurrent(context));
-            cuda_sys::cudaDeviceSynchronize();
+            rdmaxcel_sys::rdmaxcel_cuCtxSynchronize();
             cu_check!(rdmaxcel_sys::rdmaxcel_cuMemcpyHtoD_v2(
                 self.cu_ptr as u64,
                 self.cpu_buffer.as_ptr() as *const std::ffi::c_void,
@@ -554,7 +554,7 @@ impl Handler<VerifyBuffer> for CudaRdmaActor {
                 self.device_id as i32
             ));
             cu_check!(rdmaxcel_sys::rdmaxcel_cuCtxSetCurrent(context));
-            cuda_sys::cudaDeviceSynchronize();
+            rdmaxcel_sys::rdmaxcel_cuCtxSynchronize();
             cu_check!(rdmaxcel_sys::rdmaxcel_cuMemcpyDtoH_v2(
                 self.cpu_buffer.as_mut_ptr() as *mut std::ffi::c_void,
                 self.cu_ptr as rdmaxcel_sys::CUdeviceptr,