Update ORTE to support PMIx v3

This is a point-in-time update that includes support for several new PMIx features, mostly focused on debuggers and "instant on":

* initial prototype support for PMIx-based debuggers. For the moment, this is restricted to using the DVM. Supports direct launch of apps under debugger control, and indirect launch using prun as the intermediate launcher. Includes ability for debuggers to control the environment of both the launcher and the spawned app procs. Work continues on completing support for indirect launch

* IO forwarding for tools. Output of apps launched under tool control is directed to the tool and output there - includes support for XML formatting and output to files. Stdin can be forwarded from the tool to apps, but this hasn't been implemented in ORTE yet.

* Fabric integration for "instant on". Enable collection of network "blobs" to be delivered to network libraries on compute nodes prior to local proc spawn. Infrastructure is in place - implementation will come later.

* Harvesting and forwarding of envars. Enable network plugins to harvest envars and include them in the launch msg for setting the environment prior to local proc spawn. Currently, only OmniPath is supported. PMIx MCA params control which envars are included, and also allows envars to be excluded.

Signed-off-by: Ralph Castain <[email protected]>

Author: Ralph Castain

opal/dss/dss_compare.c

@@ -12,7 +12,7 @@
  * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -359,6 +359,8 @@ int opal_dss_compare_value(opal_value_t *value1, opal_value_t *value2, opal_data
         return opal_dss_compare_timeval(&value1->data.tv, &value2->data.tv, type);
     case OPAL_NAME:
         return opal_dss_compare_name(&value1->data.name, &value2->data.name, type);
+    case OPAL_ENVAR:
+        return opal_dss_compare_envar(&value1->data.envar, &value2->data.envar, type);
     default:
         opal_output(0, "COMPARE-OPAL-VALUE: UNSUPPORTED TYPE %d", (int)value1->type);
         return OPAL_EQUAL;
@@ -458,3 +460,47 @@ int opal_dss_compare_status(int *value1, int *value2, opal_data_type_t type)
     return OPAL_EQUAL;
 }
 
+int opal_dss_compare_envar(opal_envar_t *value1, opal_envar_t *value2, opal_data_type_t type)
+{
+    int rc;
+
+    if (NULL != value1->envar) {
+        if (NULL == value2->envar) {
+            return OPAL_VALUE1_GREATER;
+        }
+        rc = strcmp(value1->envar, value2->envar);
+        if (rc < 0) {
+            return OPAL_VALUE2_GREATER;
+        } else if (0 < rc) {
+            return OPAL_VALUE1_GREATER;
+        }
+    } else if (NULL != value2->envar) {
+        /* we know value1->envar had to be NULL */
+        return OPAL_VALUE2_GREATER;
+    }
+
+    /* if both are NULL or are equal, then check value */
+    if (NULL != value1->value) {
+        if (NULL == value2->value) {
+            return OPAL_VALUE1_GREATER;
+        }
+        rc = strcmp(value1->value, value2->value);
+        if (rc < 0) {
+            return OPAL_VALUE2_GREATER;
+        } else if (0 < rc) {
+            return OPAL_VALUE1_GREATER;
+        }
+    } else if (NULL != value2->value) {
+        /* we know value1->value had to be NULL */
+        return OPAL_VALUE2_GREATER;
+    }
+
+    /* finally, check separator */
+    if (value1->separator < value2->separator) {
+        return OPAL_VALUE2_GREATER;
+    }
+    if (value2->separator < value1->separator) {
+        return OPAL_VALUE1_GREATER;
+    }
+    return OPAL_EQUAL;
+}

opal/dss/dss_copy.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -338,6 +338,16 @@ int opal_dss_copy_value(opal_value_t **dest, opal_value_t *src,
     case OPAL_NAME:
         memcpy(&p->data.name, &src->data.name, sizeof(opal_process_name_t));
         break;
+    case OPAL_ENVAR:
+        OBJ_CONSTRUCT(&p->data.envar, opal_envar_t);
+        if (NULL != src->data.envar.envar) {
+            p->data.envar.envar = strdup(src->data.envar.envar);
+        }
+        if (NULL != src->data.envar.value) {
+            p->data.envar.value = strdup(src->data.envar.value);
+        }
+        p->data.envar.separator = src->data.envar.separator;
+        break;
     default:
         opal_output(0, "COPY-OPAL-VALUE: UNSUPPORTED TYPE %d", (int)src->type);
         return OPAL_ERROR;
@@ -409,3 +419,25 @@ int opal_dss_copy_vpid(opal_vpid_t **dest, opal_vpid_t *src, opal_data_type_t ty
 
     return OPAL_SUCCESS;
 }
+
+int opal_dss_copy_envar(opal_envar_t **dest, opal_envar_t *src, opal_data_type_t type)
+{
+    opal_envar_t *val;
+
+    val = OBJ_NEW(opal_envar_t);
+    if (NULL == val) {
+        OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
+    if (NULL != src->envar) {
+        val->envar = strdup(src->envar);
+    }
+    if (NULL != src->value) {
+        val->value = strdup(src->value);
+    }
+    val->separator = src->separator;
+    *dest = val;
+
+    return OPAL_SUCCESS;
+}

opal/dss/dss_internal.h

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
@@ -334,6 +334,8 @@ int opal_dss_pack_vpid(opal_buffer_t *buffer, const void *src,
 
 int opal_dss_pack_status(opal_buffer_t *buffer, const void *src,
                          int32_t num_vals, opal_data_type_t type);
+int opal_dss_pack_envar(opal_buffer_t *buffer, const void *src,
+                        int32_t num_vals, opal_data_type_t type);
 
 /*
  * Internal unpack functions
@@ -407,6 +409,9 @@ int opal_dss_unpack_vpid(opal_buffer_t *buffer, void *dest,
 int opal_dss_unpack_status(opal_buffer_t *buffer, void *dest,
                            int32_t *num_vals, opal_data_type_t type);
 
+int opal_dss_unpack_envar(opal_buffer_t *buffer, void *dest,
+                          int32_t *num_vals, opal_data_type_t type);
+
 /*
  * Internal copy functions
  */
@@ -438,6 +443,8 @@ int opal_dss_copy_jobid(opal_jobid_t **dest, opal_jobid_t *src, opal_data_type_t
 
 int opal_dss_copy_vpid(opal_vpid_t **dest, opal_vpid_t *src, opal_data_type_t type);
 
+int opal_dss_copy_envar(opal_envar_t **dest, opal_envar_t *src, opal_data_type_t type);
+
 
 /*
  * Internal compare functions
@@ -503,6 +510,7 @@ int opal_dss_compare_jobid(opal_jobid_t *value1,
                            opal_data_type_t type);
 
 int opal_dss_compare_status(int *value1, int *value2, opal_data_type_t type);
+int opal_dss_compare_envar(opal_envar_t *value1, opal_envar_t *value2, opal_data_type_t type);
 
 /*
  * Internal print functions
@@ -544,6 +552,8 @@ int opal_dss_print_name(char **output, char *prefix, opal_process_name_t *name,
 int opal_dss_print_jobid(char **output, char *prefix, opal_process_name_t *src, opal_data_type_t type);
 int opal_dss_print_vpid(char **output, char *prefix, opal_process_name_t *src, opal_data_type_t type);
 int opal_dss_print_status(char **output, char *prefix, int *src, opal_data_type_t type);
+int opal_dss_print_envar(char **output, char *prefix,
+                         opal_envar_t *src, opal_data_type_t type);
 
 
 /*

opal/dss/dss_load_unload.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$

opal/dss/dss_open_close.c

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
@@ -232,6 +232,26 @@ OBJ_CLASS_INSTANCE(opal_node_stats_t, opal_object_t,
                    opal_node_stats_destruct);
 
 
+static void opal_envar_construct(opal_envar_t *obj)
+{
+    obj->envar = NULL;
+    obj->value = NULL;
+    obj->separator = '\0';
+}
+static void opal_envar_destruct(opal_envar_t *obj)
+{
+    if (NULL != obj->envar) {
+        free(obj->envar);
+    }
+    if (NULL != obj->value) {
+        free(obj->value);
+    }
+}
+OBJ_CLASS_INSTANCE(opal_envar_t,
+                   opal_list_item_t,
+                   opal_envar_construct,
+                   opal_envar_destruct);
+
 int opal_dss_register_vars (void)
 {
     mca_base_var_enum_t *new_enum;
@@ -624,6 +644,17 @@ int opal_dss_open(void)
                                           "OPAL_STATUS", &tmp))) {
         return rc;
     }
+
+    tmp = OPAL_ENVAR;
+    if (OPAL_SUCCESS != (rc = opal_dss.register_type(opal_dss_pack_envar,
+                                          opal_dss_unpack_envar,
+                                          (opal_dss_copy_fn_t)opal_dss_copy_envar,
+                                          (opal_dss_compare_fn_t)opal_dss_compare_envar,
+                                          (opal_dss_print_fn_t)opal_dss_print_envar,
+                                          OPAL_DSS_UNSTRUCTURED,
+                                          "OPAL_ENVAR", &tmp))) {
+        return rc;
+    }
     /* All done */
 
     opal_dss_initialized = true;

opal/dss/dss_pack.c

@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -836,6 +836,11 @@ int opal_dss_pack_value(opal_buffer_t *buffer, const void *src,
                 return ret;
             }
             break;
+        case OPAL_ENVAR:
+            if (OPAL_SUCCESS != (ret = opal_dss_pack_buffer(buffer, &ptr[i]->data.envar, 1, OPAL_ENVAR))) {
+                return ret;
+            }
+            break;
         default:
             opal_output(0, "PACK-OPAL-VALUE: UNSUPPORTED TYPE %d FOR KEY %s", (int)ptr[i]->type, ptr[i]->key);
             return OPAL_ERROR;
@@ -981,3 +986,24 @@ int opal_dss_pack_status(opal_buffer_t *buffer, const void *src,
 
     return ret;
 }
+
+int opal_dss_pack_envar(opal_buffer_t *buffer, const void *src,
+                        int32_t num_vals, opal_data_type_t type)
+{
+    int ret;
+    int32_t n;
+    opal_envar_t *ptr = (opal_envar_t*)src;
+
+    for (n=0; n < num_vals; n++) {
+        if (OPAL_SUCCESS != (ret = opal_dss_pack_string(buffer, &ptr[n].envar, 1, OPAL_STRING))) {
+            return ret;
+        }
+        if (OPAL_SUCCESS != (ret = opal_dss_pack_string(buffer, &ptr[n].value, 1, OPAL_STRING))) {
+            return ret;
+        }
+        if (OPAL_SUCCESS != (ret = opal_dss_pack_byte(buffer, &ptr[n].separator, 1, OPAL_BYTE))) {
+            return ret;
+        }
+    }
+    return OPAL_SUCCESS;
+}

opal/dss/dss_print.c

@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -806,6 +806,13 @@ int opal_dss_print_value(char **output, char *prefix, opal_value_t *src, opal_da
     case OPAL_PTR:
         asprintf(output, "%sOPAL_VALUE: Data type: OPAL_PTR\tKey: %s", prefx, src->key);
         break;
+    case OPAL_ENVAR:
+        asprintf(output, "%sOPAL_VALUE: Data type: OPAL_ENVAR\tKey: %s\tName: %s\tValue: %s\tSeparator: %c",
+                 prefx, src->key,
+                 (NULL == src->data.envar.envar) ? "NULL" : src->data.envar.envar,
+                 (NULL == src->data.envar.value) ? "NULL" : src->data.envar.value,
+                 ('\0' == src->data.envar.separator) ? ' ' : src->data.envar.separator);
+        break;
     default:
         asprintf(output, "%sOPAL_VALUE: Data type: UNKNOWN\tKey: %s\tValue: UNPRINTABLE",
                  prefx, src->key);
@@ -895,3 +902,25 @@ int opal_dss_print_status(char **output, char *prefix,
     asprintf(output, "%sData type: OPAL_STATUS\tValue: %s", prefx, opal_strerror(*src));
     return OPAL_SUCCESS;
 }
+
+
+int opal_dss_print_envar(char **output, char *prefix,
+                         opal_envar_t *src, opal_data_type_t type)
+{
+    char *prefx = " ";
+
+    /* deal with NULL prefix */
+    if (NULL != prefix) prefx = prefix;
+
+    /* if src is NULL, just print data type and return */
+    if (NULL == src) {
+        asprintf(output, "%sData type: OPAL_ENVAR\tValue: NULL pointer", prefx);
+        return OPAL_SUCCESS;
+    }
+
+    asprintf(output, "%sOPAL_VALUE: Data type: OPAL_ENVAR\tName: %s\tValue: %s\tSeparator: %c",
+             prefx, (NULL == src->envar) ? "NULL" : src->envar,
+             (NULL == src->value) ? "NULL" : src->value,
+             ('\0' == src->separator) ? ' ' : src->separator);
+    return OPAL_SUCCESS;
+}

opal/dss/dss_types.h

@@ -15,7 +15,7 @@
  *                         reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -107,6 +107,7 @@ typedef struct {
 #define    OPAL_INFO_DIRECTIVES     (opal_data_type_t)   36 /**< corresponds to PMIx info directives type (uint32_t) */
 #define    OPAL_PROC_STATE          (opal_data_type_t)   37 /**< corresponds to PMIx proc state type (uint8_t) */
 #define    OPAL_PROC_INFO           (opal_data_type_t)   38 /**< corresponds to PMIx proc_info type */
+#define    OPAL_ENVAR               (opal_data_type_t)   39 /**< corresponds to PMIx envar type */
 
     /* OPAL Dynamic */
 #define    OPAL_DSS_ID_DYNAMIC      (opal_data_type_t)  100
@@ -131,7 +132,16 @@ typedef struct {
     opal_status_t exit_code;
     opal_proc_state_t state;
 } opal_proc_info_t;
-OBJ_CLASS_DECLARATION(opal_proc_info_t);
+OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_proc_info_t);
+
+/* defaine a struct for envar directives */
+typedef struct {
+    opal_list_item_t super;
+    char *envar;
+    char *value;
+    char separator;
+} opal_envar_t;
+OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_envar_t);
 
 /* Data value object */
 typedef struct {
@@ -163,6 +173,7 @@ typedef struct {
         opal_process_name_t name;
         opal_proc_info_t pinfo;
         void *ptr;  // never packed or passed anywhere
+        opal_envar_t envar;
     } data;
 } opal_value_t;
 OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_value_t);