Fix debugger attach and cospawn of debugger daemons for the STAT debugger. Add ability to test the support minus the actual debugger.

Fixes open-mpi#2411

Continue cleanup of STAT debugger attach:

* Limit the number of times we retry sending of a message to avoid an infinite loop

* Don't execute the "init_debugger_after_spawn" state for debugger jobs

* Add a new test program "attach" that takes the debugger attach fifo as its argument, and then simulates attach by writing a byte down the fifo

Output the attach fifo info if we are testing attach so we know where to attach to - otherwise, use the output_verbose

Always send "debugger release" to the job actually being debugged, not the debugger itself

Signed-off-by: Ralph Castain <[email protected]>

Remove debug

Signed-off-by: Ralph Castain <[email protected]>

Conflicts:
	orte/tools/orterun/orterun.c

(cherry picked from commit 8fa33bfdb3f6ca1667c558f06c3c8f0fb5482a8d)

Author: gpaulsen

orte/mca/oob/base/oob_base_stubs.c

@@ -138,6 +138,10 @@ void orte_oob_base_send_nb(int fd, short args, void *cbdata)
                         return;
                     }
                 }
+                opal_output_verbose(5, orte_oob_base_framework.framework_output,
+                                    "%s CANNOT SEND TO %s: TAG %d",
+                                    ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                                    ORTE_NAME_PRINT(&msg->dst), msg->tag);
                 msg->status = ORTE_ERR_ADDRESSEE_UNKNOWN;
                 ORTE_RML_SEND_COMPLETE(msg);
                 return;

orte/mca/rml/base/rml_base_frame.c

@@ -70,6 +70,14 @@ static int orte_rml_base_register(mca_base_register_flag_t flags)
                            MCA_BASE_VAR_SCOPE_READONLY,
                            &orte_rml_base.max_retries);
 
+    orte_rml_base.max_retries = 3;
+    mca_base_var_register("orte", "rml", "base", "max_retries",
+                           "Max #times to retry sending a message",
+                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                           OPAL_INFO_LVL_9,
+                           MCA_BASE_VAR_SCOPE_READONLY,
+                           &orte_rml_base.max_retries);
+
 #if OPAL_ENABLE_TIMING
     orte_rml_base.timing = false;
     (void) mca_base_var_register ("orte", "rml", "base", "timing",

orte/mca/routed/radix/routed_radix.c

@@ -274,6 +274,10 @@ static orte_process_name_t get_route(orte_process_name_t *target)
     daemon.jobid = ORTE_PROC_MY_NAME->jobid;
     /* find out what daemon hosts this proc */
     if (ORTE_VPID_INVALID == (daemon.vpid = orte_get_proc_daemon_vpid(target))) {
+        opal_output_verbose(2, orte_routed_base_framework.framework_output,
+                            "%s ATTEMPTING TO SEND TO %s",
+                            ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                            ORTE_NAME_PRINT(target));
         ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
         ret = ORTE_NAME_INVALID;
         goto found;

orte/mca/state/base/state_base_fns.c

@@ -725,12 +725,9 @@ void orte_state_base_track_procs(int fd, short argc, void *cbdata)
     } else if (ORTE_PROC_STATE_TERMINATED == state) {
         /* update the proc state */
         ORTE_FLAG_UNSET(pdata, ORTE_PROC_FLAG_ALIVE);
-        if (pdata->state < ORTE_PROC_STATE_TERMINATED) {
-            pdata->state = state;
-        }
+            /// opal_pmix.server_deregister_client(proc, NULL, NULL);
+        pdata->state = state;
         if (ORTE_FLAG_TEST(pdata, ORTE_PROC_FLAG_LOCAL)) {
-            /* tell the PMIx subsystem to cleanup this client */
-            opal_pmix.server_deregister_client(proc, NULL, NULL);
             /* Clean up the session directory as if we were the process
              * itself.  This covers the case where the process died abnormally
              * and didn't cleanup its own session directory.
@@ -762,6 +759,7 @@ void orte_state_base_track_procs(int fd, short argc, void *cbdata)
         /* track job status */
         jdata->num_terminated++;
         if (jdata->num_terminated == jdata->num_procs) {
+<<<<<<< HEAD
             /* if requested, check fd status for leaks */
             if (orte_state_base_run_fdcheck) {
                 orte_state_base_check_fds(jdata);
@@ -780,6 +778,9 @@ void orte_state_base_track_procs(int fd, short argc, void *cbdata)
             parent.jobid = jdata->jobid;
             parent.vpid = ORTE_VPID_WILDCARD;
             _send_notification(OPAL_ERR_PROC_ABORTED, pdata->state, &pdata->name, &parent);
+=======
+            ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_TERMINATED);
+>>>>>>> 8fa33bfd... Fix debugger attach and cospawn of debugger daemons for the STAT debugger. Add ability to test the support minus the actual debugger.
         }
     }
 
@@ -989,7 +990,11 @@ void orte_state_base_check_all_complete(int fd, short args, void *cbdata)
                         /* this was a debugger daemon. notify that a debugger has detached */
                         ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_DEBUGGER_DETACH);
                     }
+<<<<<<< HEAD
                     opal_hash_table_set_value_uint32(orte_job_data, jdata->jobid, NULL);
+=======
+                    opal_pointer_array_set_item(orte_job_data, j, NULL);  /* ensure the array has a NULL */
+>>>>>>> 8fa33bfd... Fix debugger attach and cospawn of debugger daemons for the STAT debugger. Add ability to test the support minus the actual debugger.
                     OBJ_RELEASE(jdata);
                 }
             }

orte/tools/orterun/orterun.c

@@ -2005,6 +2005,9 @@ static void setup_debugger_job(void)
         proc = OBJ_NEW(orte_proc_t);
         proc->name.jobid = debugger->jobid;
         proc->name.vpid = vpid++;
+        /* point the proc at the local ORTE daemon as its parent */
+        proc->parent = node->daemon->name.vpid;
+
         /* set the local/node ranks - we don't actually care
          * what these are, but the odls needs them
          */
@@ -2045,14 +2048,19 @@ static bool mpir_breakpoint_fired = false;
 void orte_debugger_init_after_spawn(int fd, short event, void *cbdata)
 {
     orte_state_caddy_t *caddy = (orte_state_caddy_t*)cbdata;
-    orte_job_t *jdata = caddy->jdata;
+    orte_job_t *jdata = caddy->jdata, *target;
     orte_proc_t *proc;
     orte_app_context_t *appctx;
     orte_vpid_t i, j;
     opal_buffer_t *buf;
     int rc;
     char **aliases, *aptr;
 
+    opal_output_verbose(5, orte_debug_output,
+                        "%s INIT AFTER SPAWN FOR %s",
+                        ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                        ORTE_JOBID_PRINT(caddy->jdata->jobid));
+
     /* if we couldn't get thru the mapper stage, we might
      * enter here with no procs. Avoid the "zero byte malloc"
      * message by checking here
@@ -2074,13 +2082,23 @@ void orte_debugger_init_after_spawn(int fd, short event, void *cbdata)
                 /* trigger the debugger */
                 MPIR_Breakpoint();
 
-                /* send a message to rank=0 to release it */
-                if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, 0)) ||
+                /* send a message to rank=0 of the job being debugged to release it */
+                target = (orte_job_t*)opal_pointer_array_get_item(orte_job_data, 1);
+                if (NULL == target) {
+                    /* the job is dead */
+                    return;
+                }
+                if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(target->procs, 0)) ||
                     ORTE_PROC_STATE_UNTERMINATED < proc->state ) {
                     /* proc is already dead */
                     return;
                 }
                 buf = OBJ_NEW(opal_buffer_t); /* don't need anything in this */
+                opal_output_verbose(5, orte_debug_output,
+                                    "%s SENDING DEBUGGER RELEASE TO %s %s:%d",
+                                    ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                                    ORTE_NAME_PRINT(&proc->name),
+                                    __FILE__, __LINE__);
                 if (0 > (rc = orte_rml.send_buffer_nb(&proc->name, buf,
                                                       ORTE_RML_TAG_DEBUGGER_RELEASE,
                                                       orte_rml_send_callback, NULL))) {
@@ -2181,18 +2199,24 @@ void orte_debugger_init_after_spawn(int fd, short event, void *cbdata)
             /* trigger the debugger */
             MPIR_Breakpoint();
 
-            /* send a message to rank=0 to release it */
-            if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, 0)) ||
+            /* send a message to rank=0 of the job being debugged to release it */
+            target = (orte_job_t*)opal_pointer_array_get_item(orte_job_data, 1);
+            if (NULL == target) {
+                /* the job is dead */
+                return;
+            }
+            if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(target->procs, 0)) ||
                 ORTE_PROC_STATE_UNTERMINATED < proc->state) {
                 /* proc is already dead or never registered with us (so we don't have
                  * contact info for him)
                  */
                 return;
             }
             opal_output_verbose(2, orte_debug_output,
-                                "%s sending debugger release to %s",
+                                "%s SENDING DEBUGGER RELEASE TO %s %s:%d",
                                 ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                                ORTE_NAME_PRINT(&proc->name));
+                                ORTE_NAME_PRINT(&proc->name),
+                                __FILE__, __LINE__);
             buf = OBJ_NEW(opal_buffer_t); /* don't need anything in this */
             if (0 > (rc = orte_rml.send_buffer_nb(&proc->name, buf,
                                                   ORTE_RML_TAG_DEBUGGER_RELEASE,
@@ -2252,10 +2276,16 @@ static void open_fifo (void)
         return;
     }
 
-    opal_output_verbose(2, orte_debug_output,
-                        "%s Monitoring debugger attach fifo %s",
-                        ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                        MPIR_attach_fifo);
+    if (orte_debugger_test_attach) {
+        opal_output(0, "%s Monitoring debugger attach fifo %s",
+                    ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                    MPIR_attach_fifo);
+    } else {
+        opal_output_verbose(2, orte_debug_output,
+                            "%s Monitoring debugger attach fifo %s",
+                            ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                            MPIR_attach_fifo);
+    }
     attach = (opal_event_t*)malloc(sizeof(opal_event_t));
     opal_event_set(orte_event_base, attach, attach_fd, OPAL_EV_READ, attach_debugger, attach);