diff --git a/configure.ac b/configure.ac index 2ce2519f9b0..1a8e19cb348 100644 --- a/configure.ac +++ b/configure.ac @@ -10,7 +10,7 @@ # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. -# Copyright (c) 2006-2021 Cisco Systems, Inc. All rights reserved +# Copyright (c) 2006-2022 Cisco Systems, Inc. All rights reserved # Copyright (c) 2006-2008 Sun Microsystems, Inc. All rights reserved. # Copyright (c) 2006-2017 Los Alamos National Security, LLC. All rights # reserved. @@ -694,7 +694,7 @@ AC_CHECK_HEADERS([alloca.h aio.h arpa/inet.h dirent.h \ sys/fcntl.h sys/ipc.h sys/shm.h \ sys/ioctl.h sys/mman.h sys/param.h sys/queue.h \ sys/resource.h sys/select.h sys/socket.h sys/sockio.h \ - sys/stat.h sys/statfs.h sys/statvfs.h sys/time.h sys/tree.h \ + sys/stat.h sys/statfs.h sys/statvfs.h time.h sys/time.h sys/tree.h \ sys/types.h sys/uio.h sys/un.h net/uio.h sys/utsname.h sys/vfs.h sys/wait.h syslog.h \ termios.h ulimit.h unistd.h util.h utmp.h malloc.h \ ifaddrs.h crt_externs.h regex.h mntent.h paths.h \ @@ -883,6 +883,15 @@ AC_INCLUDES_DEFAULT #endif ]) +AC_CHECK_MEMBERS([struct timespec.tv_nsec], + [], [], [AC_INCLUDES_DEFAULT +#ifdef HAVE_TIME_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif]) + # # Find corresponding types for MPI_Aint, MPI_Count, and MPI_Offset. # And if relevant, find the corresponding MPI_ADDRESS_KIND, diff --git a/ompi/mpi/c/wtick.c b/ompi/mpi/c/wtick.c index f6504dccee2..3136a7e1b08 100644 --- a/ompi/mpi/c/wtick.c +++ b/ompi/mpi/c/wtick.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2014 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007-2022 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. @@ -31,10 +31,11 @@ #include #endif -#include MCA_timer_IMPLEMENTATION_HEADER #include "ompi/mpi/c/bindings.h" #include "ompi/runtime/mpiruntime.h" +#include "opal/util/clock_gettime.h" + #if OMPI_BUILD_MPI_PROFILING #if OPAL_HAVE_WEAK_SYMBOLS #pragma weak MPI_Wtick = PMPI_Wtick @@ -44,38 +45,15 @@ double MPI_Wtick(void) { - /* - * See https://github.com/open-mpi/ompi/issues/3003 - * to get an idea what's going on here. - */ -#if 0 -#if OPAL_TIMER_CYCLE_NATIVE - { - opal_timer_t freq = opal_timer_base_get_freq(); - if (0 == freq) { - /* That should never happen, but if it does, return a bogus value - * rather than crashing with a division by zero */ - return (double)0.0; - } - return (double)1.0 / (double)freq; - } -#elif OPAL_TIMER_USEC_NATIVE - return 0.000001; -#endif -#else -#if defined(__linux__) && OPAL_HAVE_CLOCK_GETTIME + // We intentionally don't use the OPAL timer framework here. See + // https://github.com/open-mpi/ompi/issues/3003 for more details. struct timespec spec; double wtick = 0.0; - if (0 == clock_getres(CLOCK_MONOTONIC, &spec)){ + if (0 == opal_clock_getres(&spec)){ wtick = spec.tv_sec + spec.tv_nsec * 1.0e-09; } else { /* guess */ wtick = 1.0e-09; } return wtick; -#else - /* Otherwise, we already return usec precision. */ - return 0.000001; -#endif -#endif } diff --git a/ompi/mpi/c/wtime.c b/ompi/mpi/c/wtime.c index e22d26d943c..b7918ad5d0d 100644 --- a/ompi/mpi/c/wtime.c +++ b/ompi/mpi/c/wtime.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2006-2014 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2006-2022 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. @@ -31,11 +31,12 @@ #include #endif /* HAVE_TIME_H */ -#include MCA_timer_IMPLEMENTATION_HEADER #include "ompi/mpi/c/bindings.h" #include "ompi/runtime/mpiruntime.h" #include "ompi/runtime/ompi_spc.h" +#include "opal/util/clock_gettime.h" + #if OMPI_BUILD_MPI_PROFILING #if OPAL_HAVE_WEAK_SYMBOLS #pragma weak MPI_Wtime = PMPI_Wtime @@ -46,17 +47,9 @@ * and accuracy of the user visible timer. * More info: https://github.com/mpi-forum/mpi-issues/issues/77#issuecomment-369663119 */ -#if defined(__linux__) && OPAL_HAVE_CLOCK_GETTIME struct timespec ompi_wtime_time_origin = {.tv_sec = 0}; -#else -struct timeval ompi_wtime_time_origin = {.tv_sec = 0}; -#endif #else /* OMPI_BUILD_MPI_PROFILING */ -#if defined(__linux__) && OPAL_HAVE_CLOCK_GETTIME extern struct timespec ompi_wtime_time_origin; -#else -extern struct timeval ompi_wtime_time_origin; -#endif #endif double MPI_Wtime(void) @@ -65,36 +58,15 @@ double MPI_Wtime(void) SPC_RECORD(OMPI_SPC_WTIME, 1); - /* - * See https://github.com/open-mpi/ompi/issues/3003 to find out - * what's happening here. - */ -#if 0 -#if OPAL_TIMER_CYCLE_NATIVE - wtime = ((double) opal_timer_base_get_cycles()) / opal_timer_base_get_freq(); -#elif OPAL_TIMER_USEC_NATIVE - wtime = ((double) opal_timer_base_get_usec()) / 1000000.0; -#endif -#else -#if defined(__linux__) && OPAL_HAVE_CLOCK_GETTIME + // We intentionally don't use the OPAL timer framework here. See + // https://github.com/open-mpi/ompi/issues/3003 for more details. struct timespec tp; - (void) clock_gettime(CLOCK_MONOTONIC, &tp); - if( OPAL_UNLIKELY(0 == ompi_wtime_time_origin.tv_sec) ) { + (void) opal_clock_gettime(&tp); + if (OPAL_UNLIKELY(0 == ompi_wtime_time_origin.tv_sec)) { ompi_wtime_time_origin = tp; } wtime = (double)(tp.tv_nsec - ompi_wtime_time_origin.tv_nsec)/1.0e+9; wtime += (tp.tv_sec - ompi_wtime_time_origin.tv_sec); -#else - /* Fall back to gettimeofday() if we have nothing else */ - struct timeval tv; - gettimeofday(&tv, NULL); - if( OPAL_UNLIKELY(0 == ompi_wtime_time_origin.tv_sec) ) { - ompi_wtime_time_origin = tv; - } - wtime = (double)(tv.tv_usec - ompi_wtime_time_origin.tv_usec) / 1.0e+6; - wtime += (tv.tv_sec - ompi_wtime_time_origin.tv_sec); -#endif -#endif return wtime; } diff --git a/opal/util/Makefile.am b/opal/util/Makefile.am index 9018a596dd1..7317427f2f2 100644 --- a/opal/util/Makefile.am +++ b/opal/util/Makefile.am @@ -9,7 +9,7 @@ # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. -# Copyright (c) 2007-2015 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2007-2022 Cisco Systems, Inc. All rights reserved # Copyright (c) 2013 NVIDIA Corporation. All rights reserved. # Copyright (c) 2013 Intel, Inc. All rights reserved # Copyright (c) 2016 Los Alamos National Security, LLC. All rights @@ -48,6 +48,7 @@ headers = \ bipartite_graph.h \ bipartite_graph_internal.h \ bit_ops.h \ + clock_gettime.h \ cmd_line.h \ crc.h \ daemon_init.h \ diff --git a/opal/util/clock_gettime.h b/opal/util/clock_gettime.h new file mode 100644 index 00000000000..87692388cc6 --- /dev/null +++ b/opal/util/clock_gettime.h @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2022 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/** @file clock_gettime.h + * + * Simple, portable wrappers around clock_gettime(3) and + * clock_getres(3) to always get monotonically-increasing time. + * + * If the underlying OS does not have clock_gettime(3), use + * gettimeofday(3) instead. + * + * We intentionally do not use the OPAL timer framework for + * high-prevision time here; see + * https://github.com/open-mpi/ompi/issues/3003 for more details. + * + * As of Dec 2021, it turns out that CLOCK_MONOTONIC can actually go + * backwards on macOS (!). CLOCK_MONOTONIC does *not* go backwards on + * Linux (or anywhere else we can find), though, even in the presence + * of small NTP time adjustments -- e.g., adjtime(3) simply slightly + * speeds up or slows down the system clock to make it eventually get + * to the desired time. On macOS, we can use CLOCK_MONOTONIC_RAW, + * which never goes backwards. + * + * Hence, for these wrappers, use CLOCK_MONOTONIC_RAW on Darwin, and + * use CLOCK_MONOTONIC everywhere else. + * + * See + * https://github.com/open-mpi/ompi/pull/8057#discussion_r762612710 + * and + * https://github.com/open-mpi/ompi/pull/8057#discussion_r762618783 + * for more details. + */ + +#ifndef OPAL_UTIL_CLOCK_GETTIME_H_ +#define OPAL_UTIL_CLOCK_GETTIME_H_ + +#include "opal_config.h" + +#if HAVE_TIME_H +#include +#endif +#if HAVE_SYS_TIME_H +#include +#endif + +#if OPAL_HAVE_CLOCK_GETTIME +#if defined(__darwin__) +#define OPAL_CLOCK_TYPE CLOCK_MONOTONIC_RAW +#else +#define OPAL_CLOCK_TYPE CLOCK_MONOTONIC +#endif +#endif // OPAL_HAVE_CLOCK_GETTIME + +#if !defined(HAVE_STRUCT_TIMESPEC_TV_NSEC) +// Make sure that we have struct timespec; if not, define it. +struct timespec { + time_t tv_sec; + long tv_nsec; +}; +#endif + +/** + * Simple, portable wrapper around clock_gettime(3) for high-precision time. + * + * If the underlying system does not have clock_gettime(3), use + * gettimeofday(3) instead. + * + * @param spec (OUT) Struct to return the time + * @return Return value from underlying clock_gettime() + */ +static inline int opal_clock_gettime(struct timespec *spec) +{ +#if OPAL_HAVE_CLOCK_GETTIME + return clock_gettime(OPAL_CLOCK_TYPE, spec); +#else + // If we do not have clock_gettime(), fall back to gettimeofday() + struct timeval tv; + int ret = gettimeofday(&tv, NULL); + + spec->tv_sec = tv.tv_sec; + // Elevate the micrseconds to nanoseconds + spec->tv_nsec = tv.tv_usec * 1000; + + return ret; +#endif +} + +/** + * Simple, portable wrapper around clock_getres(3) for high-precision time. + * + * If the underlying system does not have clock_gettime(3), return usec + * precison (because opal_clock_gettime() will be using gettimeofday(3)). + * + * @param spec (OUT) Struct to return the resolution + * @return Return value from underlying clock_getres() + */ +static inline int opal_clock_getres(struct timespec *spec) +{ +#if OPAL_HAVE_CLOCK_GETTIME + return clock_getres(OPAL_CLOCK_TYPE, spec); +#else + // If we don't have clock_gettime(), just return usec precision. + spec->tv_sec = 0; + spec->tv_nsec = 1000; + + return 0; +#endif +} + +#endif // OPAL_UTIL_CLOCK_GETTIME_H_