Added an initial implementation of partitioned communication.

ompi/include/mpi.h.in

@@ -12,7 +12,7 @@
  * Copyright (c) 2007-2020 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2008-2009 Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2009-2012 Oak Rigde National Laboratory.  All rights reserved.
- * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2011-2020 Sandia National Laboratories. All rights reserved.
  * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2011-2013 INRIA.  All rights reserved.
@@ -1618,6 +1618,17 @@ OMPI_DECLSPEC  int MPI_Isend(const void *buf, int count, MPI_Datatype datatype,
                              int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest,
                               int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Precv_init(const void* buf, int partitions, MPI_Count count,
+                                  MPI_Datatype datatype, int source, int tag, MPI_Comm comm,
+                                  MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Psend_init(const void* buf, int partitions, MPI_Count count,
+                                  MPI_Datatype datatype, int dest, int tag, MPI_Comm comm,
+                                  MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Pready(int partitions, MPI_Request request);
+OMPI_DECLSPEC  int MPI_Pready_range(int partition_low, int partition_high,
+                                    MPI_Request request);
+OMPI_DECLSPEC  int MPI_Pready_list(int length, int partition_list[], MPI_Request request);
+OMPI_DECLSPEC  int MPI_Parrived(MPI_Request request, MPI_Count partition, int *flag);
 OMPI_DECLSPEC  int MPI_Is_thread_main(int *flag);
 OMPI_DECLSPEC  int MPI_Lookup_name(const char *service_name, MPI_Info info, char *port_name);
 OMPI_DECLSPEC  MPI_Fint MPI_Message_c2f(MPI_Message message);
@@ -2279,6 +2290,17 @@ OMPI_DECLSPEC  int PMPI_Isend(const void *buf, int count, MPI_Datatype datatype,
                               int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest,
                                int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Precv_init(const void* buf, int partitions, MPI_Count count,
+                                  MPI_Datatype datatype, int source, int tag, MPI_Comm comm,
+                                  MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Psend_init(const void* buf, int partitions, MPI_Count count,
+                                  MPI_Datatype datatype, int dest, int tag, MPI_Comm comm,
+                                  MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Pready(int partitions, MPI_Request request);
+OMPI_DECLSPEC  int PMPI_Pready_range(int partition_low, int partition_high,
+                                    MPI_Request request);
+OMPI_DECLSPEC  int PMPI_Pready_list(int length, int partition_list[], MPI_Request request);
+OMPI_DECLSPEC  int PMPI_Parrived(MPI_Request request, MPI_Count partition, int *flag);
 OMPI_DECLSPEC  int PMPI_Is_thread_main(int *flag);
 OMPI_DECLSPEC  int PMPI_Lookup_name(const char *service_name, MPI_Info info, char *port_name);
 OMPI_DECLSPEC  MPI_Fint PMPI_Message_c2f(MPI_Message message);

ompi/mca/part/Makefile.am

@@ -0,0 +1,39 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# main library setup
+noinst_LTLIBRARIES = libmca_part.la
+libmca_part_la_SOURCES =
+
+# local files
+headers = part.h 
+libmca_part_la_SOURCES += $(headers) $(nodist_headers)
+
+# Conditionally install the header files
+if WANT_INSTALL_HEADERS
+ompidir = $(ompiincludedir)/$(subdir)
+nobase_ompi_HEADERS = $(headers)
+nobase_nodist_ompi_HEADERS = $(nodist_headers)
+endif
+
+include base/Makefile.am
+
+distclean-local:
+	rm -f base/static-components.h

ompi/mca/part/base/Makefile.am

@@ -0,0 +1,21 @@
+#
+# Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers += \
+        base/base.h \
+        base/part_base_prequest.h \
+        base/part_base_precvreq.h \
+        base/part_base_psendreq.h 
+
+libmca_part_la_SOURCES += \
+        base/part_base_frame.c \
+        base/part_base_precvreq.c \
+        base/part_base_prequest.c \
+        base/part_base_select.c \
+        base/part_base_psendreq.c 

ompi/mca/part/base/base.h

@@ -0,0 +1,71 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_PART_BASE_H
+#define MCA_PART_BASE_H
+
+#include "ompi_config.h"
+
+#include "ompi/mca/mca.h"
+#include "opal/mca/base/mca_base_framework.h"
+#include "opal/class/opal_list.h"
+#include "opal/class/opal_pointer_array.h"
+
+#include "ompi/mca/part/part.h"
+
+/*
+ * Global functions for PART
+ */
+
+BEGIN_C_DECLS
+
+/*
+ * This is the base priority for a PART wrapper component
+ * If there exists more than one then it is undefined
+ * which one is picked.
+ */
+#define PART_SELECT_WRAPPER_PRIORITY -128
+
+/*
+ * MCA framework
+ */
+OMPI_DECLSPEC extern mca_base_framework_t ompi_part_base_framework;
+
+/*
+ * Select an available component.
+ */
+OMPI_DECLSPEC  int mca_part_base_select(bool enable_progress_threads,
+                                        bool enable_mpi_threads);
+
+OMPI_DECLSPEC  int mca_part_base_progress(void);
+
+OMPI_DECLSPEC int mca_part_base_finalize(void);
+
+/*
+ * Globals
+ */
+OMPI_DECLSPEC extern mca_part_base_component_t mca_part_base_selected_component;
+OMPI_DECLSPEC extern mca_part_base_module_t mca_part;
+OMPI_DECLSPEC extern opal_pointer_array_t mca_part_base_part;
+
+END_C_DECLS
+
+#endif /* MCA_PART_BASE_H */

ompi/mca/part/base/part_base_frame.c

@@ -0,0 +1,153 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      IBM Corporation. All rights reserved.
+ * Copyright (c) 2020      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif  /* HAVE_UNIST_H */
+#include "ompi/mca/mca.h"
+#include "opal/util/output.h"
+#include "opal/mca/base/base.h"
+
+
+#include "ompi/constants.h"
+#include "ompi/mca/part/part.h"
+#include "ompi/mca/part/base/base.h"
+#include "ompi/mca/part/base/part_base_prequest.h"
+
+/*
+ * The following file was created by configure.  It contains extern
+ * statements and the definition of an array of pointers to each
+ * component's public mca_base_component_t struct.
+ */
+
+#include "ompi/mca/part/base/static-components.h"
+
+/*
+ * This function allows modules to not declare a progress function,
+ * by defaulting to doing nothing.
+ */
+int mca_part_base_progress(void)
+{
+    return OMPI_SUCCESS;
+}
+
+#define xstringify(part) #part
+#define stringify(part) xstringify(part)
+
+/*
+ * Global variables
+ */
+mca_part_base_module_t mca_part = {
+    mca_part_base_progress,   /* part_progress */
+    NULL,                     /* part_precv_init */
+    NULL,                     /* part_psend_init */
+    NULL,                     /* part_start */
+    NULL,                     /* part_pready */
+    NULL                      /* part_parrived */
+};
+
+mca_part_base_component_t mca_part_base_selected_component = {{0}};
+opal_pointer_array_t mca_part_base_part = {{0}};
+
+static int mca_part_base_register(mca_base_register_flag_t flags)
+{
+    return OMPI_SUCCESS;
+}
+
+int mca_part_base_finalize(void) {
+  if (NULL != mca_part_base_selected_component.partm_finalize) {
+    return mca_part_base_selected_component.partm_finalize();
+  }
+  return OMPI_SUCCESS;
+}
+
+
+static int mca_part_base_close(void)
+{
+    int i, j;
+
+    /* unregister the progress function */
+    if( NULL != mca_part.part_progress ) {
+        opal_progress_unregister(mca_part.part_progress);
+    }
+
+    /* reset the progress function to do nothing */
+    mca_part.part_progress = mca_part_base_progress;
+
+    /* Free all the strings in the array of components */
+    j = opal_pointer_array_get_size(&mca_part_base_part);
+    for (i = 0; i < j; ++i) {
+        char *str;
+        str = (char*) opal_pointer_array_get_item(&mca_part_base_part, i);
+        free(str);
+    }
+    OBJ_DESTRUCT(&mca_part_base_part);
+
+    OBJ_DESTRUCT(&mca_part_base_psend_requests);
+    OBJ_DESTRUCT(&mca_part_base_precv_requests);
+
+    /* Close all remaining available components */
+    return mca_base_framework_components_close(&ompi_part_base_framework, NULL);
+}
+
+/**
+ * Function for finding and opening either all MCA components, or the one
+ * that was specifically requested via a MCA parameter.
+ */
+static int mca_part_base_open(mca_base_open_flag_t flags)
+{
+    OBJ_CONSTRUCT(&mca_part_base_part, opal_pointer_array_t);
+
+
+    OBJ_CONSTRUCT(&mca_part_base_psend_requests, opal_free_list_t);
+    OBJ_CONSTRUCT(&mca_part_base_precv_requests, opal_free_list_t);
+    /* Open up all available components */
+
+    if (OPAL_SUCCESS !=
+        mca_base_framework_components_open(&ompi_part_base_framework, flags)) {
+        return OMPI_ERROR;
+    }
+
+    /* Set a sentinel in case we don't select any components (e.g.,
+       ompi_info) */
+
+    mca_part_base_selected_component.partm_finalize = NULL;
+
+    /* Currently this uses a default with no selection criteria as there is only 1 module. */
+    opal_pointer_array_add(&mca_part_base_part, strdup("rma"));
+
+    return OMPI_SUCCESS;
+}
+
+MCA_BASE_FRAMEWORK_DECLARE(ompi, part, "OMPI PART", mca_part_base_register,
+                           mca_part_base_open, mca_part_base_close,
+                           mca_part_base_static_components, 0);

ompi/mca/part/base/part_base_precvreq.c

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2020      Sandia National Laboratories. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/*%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%*/
+
+#include "ompi_config.h"
+#include "ompi/types.h"
+#include "ompi/mca/part/part.h"
+#include "ompi/mca/part/base/part_base_precvreq.h"
+
+
+static void mca_part_base_precv_request_construct(mca_part_base_precv_request_t*);
+static void mca_part_base_precv_request_destruct(mca_part_base_precv_request_t*);
+
+
+OBJ_CLASS_INSTANCE(
+    mca_part_base_precv_request_t,
+    mca_part_base_prequest_t,
+    mca_part_base_precv_request_construct,
+    mca_part_base_precv_request_destruct
+);
+
+
+static void mca_part_base_precv_request_construct(mca_part_base_precv_request_t* request)
+{
+    OBJ_CONSTRUCT(&request->req_base.req_convertor, opal_convertor_t);
+}
+
+
+static void mca_part_base_precv_request_destruct(mca_part_base_precv_request_t* request)
+{
+    /* For each request the convertor get cleaned after each message
+     * (in the base _FINI macro). Therefore, as the convertor is a static object
+     * we don't have to call OBJ_DESTRUCT here.
+     */
+}
+