v4.0.x: Backport: Ensure that --host / --hostfile nodes are always used in order provided

.gitignore

@@ -199,6 +199,8 @@ ompi/mca/rte/orte/mpirun.1
 
 ompi/mca/sharedfp/addproc/mca_sharedfp_addproc_control
 
+ompi/mca/topo/treematch/config.h
+
 ompi/mpi/c/profile/p*.c
 
 ompi/mpi/fortran/configure-fortran-output.h

opal/mca/compress/base/Makefile.am

@@ -3,6 +3,7 @@
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
 # Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2019      Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -14,7 +15,6 @@ headers += \
         base/base.h
 
 libmca_compress_la_SOURCES += \
-        base/compress_base_open.c \
-        base/compress_base_close.c \
+        base/compress_base_frame.c \
         base/compress_base_select.c \
         base/compress_base_fns.c

opal/mca/compress/base/base.h

@@ -3,6 +3,7 @@
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
  *
+ * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -27,6 +28,12 @@
 extern "C" {
 #endif
 
+typedef struct {
+    size_t compress_limit;
+} opal_compress_base_t;
+
+OPAL_DECLSPEC extern opal_compress_base_t opal_compress_base;
+
     /**
      * Initialize the COMPRESS MCA framework
      *

opal/mca/compress/base/compress_base_fns.c

@@ -4,6 +4,8 @@
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  *
+ * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,6 +36,7 @@
 #include "opal/util/os_dirpath.h"
 #include "opal/util/output.h"
 #include "opal/util/argv.h"
+#include "opal/util/printf.h"
 
 #include "opal/mca/compress/compress.h"
 #include "opal/mca/compress/base/base.h"
@@ -54,12 +57,12 @@ int opal_compress_base_tar_create(char ** target)
     pid_t child_pid = 0;
     int status = 0;
 
-    asprintf(&tar_target, "%s.tar", *target);
+    opal_asprintf(&tar_target, "%s.tar", *target);
 
     child_pid = fork();
     if( 0 == child_pid ) { /* Child */
         char *cmd;
-        asprintf(&cmd, "tar -cf %s %s", tar_target, *target);
+        opal_asprintf(&cmd, "tar -cf %s %s", tar_target, *target);
 
         argv = opal_argv_split(cmd, ' ');
         status = execvp(argv[0], argv);
@@ -101,7 +104,7 @@ int opal_compress_base_tar_extract(char ** target)
     child_pid = fork();
     if( 0 == child_pid ) { /* Child */
         char *cmd;
-        asprintf(&cmd, "tar -xf %s", *target);
+        opal_asprintf(&cmd, "tar -xf %s", *target);
 
         argv = opal_argv_split(cmd, ' ');
         status = execvp(argv[0], argv);

opal/mca/compress/base/compress_base_frame.c

@@ -0,0 +1,91 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2019      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include "opal/mca/base/base.h"
+#include "opal/mca/compress/base/base.h"
+
+#include "opal/mca/compress/base/static-components.h"
+
+/*
+ * Globals
+ */
+static bool compress_block(uint8_t *inbytes,
+                           size_t inlen,
+                           uint8_t **outbytes,
+                           size_t *olen)
+{
+    return false;
+}
+
+static bool decompress_block(uint8_t **outbytes, size_t olen,
+                             uint8_t *inbytes, size_t len)
+{
+    return false;
+}
+
+opal_compress_base_module_t opal_compress = {
+    NULL, /* init             */
+    NULL, /* finalize         */
+    NULL, /* compress         */
+    NULL, /* compress_nb      */
+    NULL, /* decompress       */
+    NULL,  /* decompress_nb    */
+    compress_block,
+    decompress_block
+};
+opal_compress_base_t opal_compress_base = {0};
+
+opal_compress_base_component_t opal_compress_base_selected_component = {{0}};
+
+static int opal_compress_base_register(mca_base_register_flag_t flags);
+
+MCA_BASE_FRAMEWORK_DECLARE(opal, compress, "COMPRESS MCA",
+                           opal_compress_base_register, opal_compress_base_open,
+                           opal_compress_base_close, mca_compress_base_static_components, 0);
+
+static int opal_compress_base_register(mca_base_register_flag_t flags)
+{
+    opal_compress_base.compress_limit = 4096;
+    (void) mca_base_var_register("opal", "compress", "base", "limit",
+                                 "Threshold beyond which data will be compressed",
+                                 MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_3,
+                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_compress_base.compress_limit);
+
+    return OPAL_SUCCESS;
+}
+
+/**
+ * Function for finding and opening either all MCA components,
+ * or the one that was specifically requested via a MCA parameter.
+ */
+int opal_compress_base_open(mca_base_open_flag_t flags)
+{
+    /* Open up all available components */
+    return mca_base_framework_components_open(&opal_compress_base_framework, flags);
+}
+
+int opal_compress_base_close(void)
+{
+    /* Call the component's finalize routine */
+    if( NULL != opal_compress.finalize ) {
+        opal_compress.finalize();
+    }
+
+    /* Close all available modules that are open */
+    return mca_base_framework_components_close (&opal_compress_base_framework, NULL);
+}

opal/mca/compress/base/compress_base_select.c

@@ -7,6 +7,7 @@
  *
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,26 +30,19 @@
 
 int opal_compress_base_select(void)
 {
-    int ret, exit_status = OPAL_SUCCESS;
+    int ret = OPAL_SUCCESS;
     opal_compress_base_component_t *best_component = NULL;
     opal_compress_base_module_t *best_module = NULL;
 
-    /* Compression currently only used with C/R */
-    if( !opal_cr_is_enabled ) {
-        opal_output_verbose(10, opal_compress_base_framework.framework_output,
-                            "compress:open: FT is not enabled, skipping!");
-        return OPAL_SUCCESS;
-    }
-
     /*
      * Select the best component
      */
     if( OPAL_SUCCESS != mca_base_select("compress", opal_compress_base_framework.framework_output,
                                         &opal_compress_base_framework.framework_components,
                                         (mca_base_module_t **) &best_module,
                                         (mca_base_component_t **) &best_component, NULL) ) {
-        /* This will only happen if no component was selected */
-        exit_status = OPAL_ERROR;
+        /* This will only happen if no component was selected,
+         * in which case we use the default one */
         goto cleanup;
     }
 
@@ -58,12 +52,11 @@ int opal_compress_base_select(void)
     /* Initialize the winner */
     if (NULL != best_module) {
         if (OPAL_SUCCESS != (ret = best_module->init()) ) {
-            exit_status = ret;
             goto cleanup;
         }
         opal_compress = *best_module;
     }
 
  cleanup:
-    return exit_status;
+    return ret;
 }