btl: add a new btl for the UCT layer in OpenUCX

This commit adds a new btl for one-sided and two-sided. This btl
uses the uct layer in OpenUCX. This btl makes use of multiple uct
contexts and per-thread device pinning to provide good performance
when using threads and osc/rdma. This btl has been tested extensively
with osc/rdma and passes all MTT tests on aries and IB hardware.

For now this new component disables itself but can be enabled by
setting the btl_ucx_transports MCA variable with a comma-delimited
list of supported memory domains/transport layers. For example:
--mca btl_uct_memory_domains ib/mlx5_0. The specific transports used
can be selected using --mca btl_uct_transports. The default is to use
any available transport.

Signed-off-by: Nathan Hjelm <[email protected]>

Author: hjelmn

config/ompi_check_ucx.m4

@@ -51,6 +51,15 @@ AC_DEFUN([OMPI_CHECK_UCX],[
                                    [],
                                    [ompi_check_ucx_happy="yes"],
                                    [ompi_check_ucx_happy="no"])
+                         OPAL_CHECK_PACKAGE([ompi_check_ucx],
+                                   [uct/api/uct.h],
+                                   [uct],
+                                   [uct_md_query],
+                                   [],
+                                   [],
+                                   [],
+                                   [ompi_check_ucx_happy="yes"],
+                                   [ompi_check_ucx_happy="no"])
                          AS_IF([test "$ompi_check_ucx_happy" = yes],
                                [AC_MSG_CHECKING(for UCX version compatibility)
                                 AC_REQUIRE_CPP
@@ -83,6 +92,18 @@ AC_DEFUN([OMPI_CHECK_UCX],[
                                             [$ompi_check_ucx_libdir],
                                             [ompi_check_ucx_happy="yes"],
                                             [ompi_check_ucx_happy="no"])
+
+
+                         OPAL_CHECK_PACKAGE([ompi_check_ucx],
+                                            [uct/api/uct.h],
+                                            [uct],
+                                            [uct_md_query],
+                                            [],
+                                            [$ompi_check_ucx_dir],
+                                            [$ompi_check_ucx_libdir],
+                                            [ompi_check_ucx_happy="yes"],
+                                            [ompi_check_ucx_happy="no"])
+
                          CPPFLAGS="$ompi_check_ucx_$1_save_CPPFLAGS"
                          LDFLAGS="$ompi_check_ucx_$1_save_LDFLAGS"
                          LIBS="$ompi_check_ucx_$1_save_LIBS"

opal/mca/btl/uct/Makefile.am

@@ -0,0 +1,68 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights
+#                         reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_CPPFLAGS = $(btl_uct_CPPFLAGS)
+
+amca_paramdir = $(AMCA_PARAM_SETS_DIR)
+
+sources = \
+    btl_uct.h \
+    btl_uct_module.c \
+    btl_uct_component.c \
+    btl_uct_rdma.h \
+    btl_uct_rdma.c \
+    btl_uct_endpoint.h \
+    btl_uct_endpoint.c \
+    btl_uct_amo.c \
+    btl_uct_am.h \
+    btl_uct_am.c \
+    btl_uct_frag.h \
+    btl_uct_frag.c \
+    btl_uct_tl.c \
+    btl_uct_types.h \
+    btl_uct_device_context.h
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_opal_btl_uct_DSO
+lib =
+lib_sources =
+component = mca_btl_uct.la
+component_sources = $(sources)
+else
+lib = libmca_btl_uct.la
+lib_sources = $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(opallibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_btl_uct_la_SOURCES = $(component_sources)
+mca_btl_uct_la_LDFLAGS = -module -avoid-version $(btl_uct_LDFLAGS)
+mca_btl_uct_la_LIBADD = $(top_builddir)/opal/lib@[email protected] $(btl_uct_LIBS)
+
+noinst_LTLIBRARIES = $(lib)
+libmca_btl_uct_la_SOURCES = $(lib_sources)
+libmca_btl_uct_la_LDFLAGS = -module -avoid-version $(btl_uct_LDFLAGS)