RFC: opal_clock_gettime() and opal_clock_getres()

configure.ac

@@ -10,7 +10,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2006-2021 Cisco Systems, Inc.  All rights reserved
+# Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved
 # Copyright (c) 2006-2008 Sun Microsystems, Inc.  All rights reserved.
 # Copyright (c) 2006-2017 Los Alamos National Security, LLC.  All rights
 #                         reserved.
@@ -694,7 +694,7 @@ AC_CHECK_HEADERS([alloca.h aio.h arpa/inet.h dirent.h \
     sys/fcntl.h sys/ipc.h sys/shm.h \
     sys/ioctl.h sys/mman.h sys/param.h sys/queue.h \
     sys/resource.h sys/select.h sys/socket.h sys/sockio.h \
-    sys/stat.h sys/statfs.h sys/statvfs.h sys/time.h sys/tree.h \
+    sys/stat.h sys/statfs.h sys/statvfs.h time.h sys/time.h sys/tree.h \
     sys/types.h sys/uio.h sys/un.h net/uio.h sys/utsname.h sys/vfs.h sys/wait.h syslog.h \
     termios.h ulimit.h unistd.h util.h utmp.h malloc.h \
     ifaddrs.h crt_externs.h regex.h mntent.h paths.h \
@@ -883,6 +883,15 @@ AC_INCLUDES_DEFAULT
 #endif
 ])
 
+AC_CHECK_MEMBERS([struct timespec.tv_nsec],
+               [], [], [AC_INCLUDES_DEFAULT
+#ifdef HAVE_TIME_H
+#include <time.h>
+#endif
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif])
+
 #
 # Find corresponding types for MPI_Aint, MPI_Count, and MPI_Offset.
 # And if relevant, find the corresponding MPI_ADDRESS_KIND,

ompi/mpi/c/wtick.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation.  All rights reserved.
@@ -31,10 +31,11 @@
 #include <time.h>
 #endif
 
-#include MCA_timer_IMPLEMENTATION_HEADER
 #include "ompi/mpi/c/bindings.h"
 #include "ompi/runtime/mpiruntime.h"
 
+#include "opal/util/clock_gettime.h"
+
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_Wtick = PMPI_Wtick
@@ -44,38 +45,15 @@
 
 double MPI_Wtick(void)
 {
-    /*
-     * See https://github.com/open-mpi/ompi/issues/3003
-     * to get an idea what's going on here.
-     */
-#if 0
-#if OPAL_TIMER_CYCLE_NATIVE
-    {
-        opal_timer_t freq = opal_timer_base_get_freq();
-        if (0 == freq) {
-            /* That should never happen, but if it does, return a bogus value
-             * rather than crashing with a division by zero */
-            return (double)0.0;
-        }
-        return (double)1.0 / (double)freq;
-    }
-#elif OPAL_TIMER_USEC_NATIVE
-    return 0.000001;
-#endif
-#else
-#if defined(__linux__) && OPAL_HAVE_CLOCK_GETTIME
+    // We intentionally don't use the OPAL timer framework here.  See
+    // https://github.com/open-mpi/ompi/issues/3003 for more details.
     struct timespec spec;
     double wtick = 0.0;
-    if (0 == clock_getres(CLOCK_MONOTONIC, &spec)){
+    if (0 == opal_clock_getres(&spec)){
         wtick =  spec.tv_sec + spec.tv_nsec * 1.0e-09;
     } else {
         /* guess */
         wtick = 1.0e-09;
     }
     return wtick;
-#else
-    /* Otherwise, we already return usec precision. */
-    return 0.000001;
-#endif
-#endif
 }

ompi/mpi/c/wtime.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2006-2022 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation.  All rights reserved.
@@ -31,11 +31,12 @@
 #include <time.h>
 #endif  /* HAVE_TIME_H */
 
-#include MCA_timer_IMPLEMENTATION_HEADER
 #include "ompi/mpi/c/bindings.h"
 #include "ompi/runtime/mpiruntime.h"
 #include "ompi/runtime/ompi_spc.h"
 
+#include "opal/util/clock_gettime.h"
+
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
 #pragma weak MPI_Wtime = PMPI_Wtime
@@ -46,17 +47,9 @@
  * and accuracy of the user visible timer.
  * More info: https://github.com/mpi-forum/mpi-issues/issues/77#issuecomment-369663119
  */
-#if defined(__linux__) && OPAL_HAVE_CLOCK_GETTIME
 struct timespec ompi_wtime_time_origin = {.tv_sec = 0};
-#else
-struct timeval ompi_wtime_time_origin = {.tv_sec = 0};
-#endif
 #else  /* OMPI_BUILD_MPI_PROFILING */
-#if defined(__linux__) && OPAL_HAVE_CLOCK_GETTIME
 extern struct timespec ompi_wtime_time_origin;
-#else
-extern struct timeval ompi_wtime_time_origin;
-#endif
 #endif
 
 double MPI_Wtime(void)
@@ -65,36 +58,15 @@ double MPI_Wtime(void)
 
     SPC_RECORD(OMPI_SPC_WTIME, 1);
 
-    /*
-     * See https://github.com/open-mpi/ompi/issues/3003 to find out
-     * what's happening here.
-     */
-#if 0
-#if OPAL_TIMER_CYCLE_NATIVE
-    wtime = ((double) opal_timer_base_get_cycles()) / opal_timer_base_get_freq();
-#elif OPAL_TIMER_USEC_NATIVE
-    wtime = ((double) opal_timer_base_get_usec()) / 1000000.0;
-#endif
-#else
-#if defined(__linux__) && OPAL_HAVE_CLOCK_GETTIME
+    // We intentionally don't use the OPAL timer framework here.  See
+    // https://github.com/open-mpi/ompi/issues/3003 for more details.
     struct timespec tp;
-    (void) clock_gettime(CLOCK_MONOTONIC, &tp);
-    if( OPAL_UNLIKELY(0 == ompi_wtime_time_origin.tv_sec) ) {
+    (void) opal_clock_gettime(&tp);
+    if (OPAL_UNLIKELY(0 == ompi_wtime_time_origin.tv_sec)) {
         ompi_wtime_time_origin = tp;
     }
     wtime  = (double)(tp.tv_nsec - ompi_wtime_time_origin.tv_nsec)/1.0e+9;
     wtime += (tp.tv_sec - ompi_wtime_time_origin.tv_sec);
-#else
-    /* Fall back to gettimeofday() if we have nothing else */
-    struct timeval tv;
-    gettimeofday(&tv, NULL);
-    if( OPAL_UNLIKELY(0 == ompi_wtime_time_origin.tv_sec) ) {
-        ompi_wtime_time_origin = tv;
-    }
-    wtime  = (double)(tv.tv_usec - ompi_wtime_time_origin.tv_usec) / 1.0e+6;
-    wtime += (tv.tv_sec - ompi_wtime_time_origin.tv_sec);
-#endif
-#endif
 
     return wtime;
 }

opal/util/Makefile.am

@@ -9,7 +9,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2007-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2007-2022 Cisco Systems, Inc.  All rights reserved
 # Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
 # Copyright (c) 2013      Intel, Inc. All rights reserved
 # Copyright (c) 2016      Los Alamos National Security, LLC. All rights
@@ -48,6 +48,7 @@ headers = \
 	bipartite_graph.h \
 	bipartite_graph_internal.h \
         bit_ops.h \
+        clock_gettime.h \
         cmd_line.h \
         crc.h \
         daemon_init.h \

opal/util/clock_gettime.h

@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2022 Cisco Systems, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/** @file clock_gettime.h
+ *
+ * Simple, portable wrappers around clock_gettime(3) and
+ * clock_getres(3) to always get monotonically-increasing time.
+ *
+ * If the underlying OS does not have clock_gettime(3), use
+ * gettimeofday(3) instead.
+ *
+ * We intentionally do not use the OPAL timer framework for
+ * high-prevision time here; see
+ * https://github.com/open-mpi/ompi/issues/3003 for more details.
+ *
+ * As of Dec 2021, it turns out that CLOCK_MONOTONIC can actually go
+ * backwards on macOS (!).  CLOCK_MONOTONIC does *not* go backwards on
+ * Linux (or anywhere else we can find), though, even in the presence
+ * of small NTP time adjustments -- e.g., adjtime(3) simply slightly
+ * speeds up or slows down the system clock to make it eventually get
+ * to the desired time.  On macOS, we can use CLOCK_MONOTONIC_RAW,
+ * which never goes backwards.
+ *
+ * Hence, for these wrappers, use CLOCK_MONOTONIC_RAW on Darwin, and
+ * use CLOCK_MONOTONIC everywhere else.
+ *
+ * See
+ * https://github.com/open-mpi/ompi/pull/8057#discussion_r762612710
+ * and
+ * https://github.com/open-mpi/ompi/pull/8057#discussion_r762618783
+ * for more details.
+ */
+
+#ifndef OPAL_UTIL_CLOCK_GETTIME_H_
+#define OPAL_UTIL_CLOCK_GETTIME_H_
+
+#include "opal_config.h"
+
+#if HAVE_TIME_H
+#include <time.h>
+#endif
+#if HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
+#if OPAL_HAVE_CLOCK_GETTIME
+#if defined(__darwin__)
+#define OPAL_CLOCK_TYPE CLOCK_MONOTONIC_RAW
+#else
+#define OPAL_CLOCK_TYPE CLOCK_MONOTONIC
+#endif
+#endif // OPAL_HAVE_CLOCK_GETTIME
+
+#if !defined(HAVE_STRUCT_TIMESPEC_TV_NSEC)
+// Make sure that we have struct timespec; if not, define it.
+struct timespec {
+    time_t tv_sec;
+    long tv_nsec;
+};
+#endif
+
+/**
+ * Simple, portable wrapper around clock_gettime(3) for high-precision time.
+ *
+ * If the underlying system does not have clock_gettime(3), use
+ * gettimeofday(3) instead.
+ *
+ * @param spec (OUT) Struct to return the time
+ * @return Return value from underlying clock_gettime()
+ */
+static inline int opal_clock_gettime(struct timespec *spec)
+{
+#if OPAL_HAVE_CLOCK_GETTIME
+    return clock_gettime(OPAL_CLOCK_TYPE, spec);
+#else
+    // If we do not have clock_gettime(), fall back to gettimeofday()
+    struct timeval tv;
+    int ret = gettimeofday(&tv, NULL);
+
+    spec->tv_sec = tv.tv_sec;
+    // Elevate the micrseconds to nanoseconds
+    spec->tv_nsec = tv.tv_usec * 1000;
+
+    return ret;
+#endif
+}
+
+/**
+ * Simple, portable wrapper around clock_getres(3) for high-precision time.
+ *
+ * If the underlying system does not have clock_gettime(3), return usec
+ * precison (because opal_clock_gettime() will be using gettimeofday(3)).
+ *
+ * @param spec (OUT) Struct to return the resolution
+ * @return Return value from underlying clock_getres()
+ */
+static inline int opal_clock_getres(struct timespec *spec)
+{
+#if OPAL_HAVE_CLOCK_GETTIME
+    return clock_getres(OPAL_CLOCK_TYPE, spec);
+#else
+    // If we don't have clock_gettime(), just return usec precision.
+    spec->tv_sec = 0;
+    spec->tv_nsec = 1000;
+
+    return 0;
+#endif
+}
+
+#endif // OPAL_UTIL_CLOCK_GETTIME_H_