backport improvements on GC scheduler shutdown

Author: d-netto

src/gc.c

@@ -2856,13 +2856,16 @@ JL_EXTENSION NOINLINE void gc_mark_loop_serial(jl_ptls_t ptls)
     gc_drain_own_chunkqueue(ptls, &ptls->mark_queue);
 }
 
-void gc_mark_and_steal(jl_ptls_t ptls)
+int gc_mark_and_steal(jl_ptls_t ptls)
 {
     jl_gc_markqueue_t *mq = &ptls->mark_queue;
     jl_gc_markqueue_t *mq_master = NULL;
     int master_tid = jl_atomic_load(&gc_master_tid);
-    if (master_tid != -1)
-        mq_master = &gc_all_tls_states[master_tid]->mark_queue;
+    if (master_tid == -1) {
+        return 0;
+    }
+    mq_master = &gc_all_tls_states[master_tid]->mark_queue;
+    int marked = 0;
     void *new_obj;
     jl_gc_chunk_t c;
     pop : {
@@ -2878,6 +2881,7 @@ void gc_mark_and_steal(jl_ptls_t ptls)
         goto steal;
     }
     mark : {
+        marked = 1;
         gc_mark_outrefs(ptls, mq, new_obj, 0);
         goto pop;
     }
@@ -2906,12 +2910,10 @@ void gc_mark_and_steal(jl_ptls_t ptls)
             }
         }
         // Try to steal chunk from master thread
-        if (mq_master != NULL) {
-            c = gc_chunkqueue_steal_from(mq_master);
-            if (c.cid != GC_empty_chunk) {
-                gc_mark_chunk(ptls, mq, &c);
-                goto pop;
-            }
+        c = gc_chunkqueue_steal_from(mq_master);
+        if (c.cid != GC_empty_chunk) {
+            gc_mark_chunk(ptls, mq, &c);
+            goto pop;
         }
         // Try to steal pointer from random GC thread
         for (int i = 0; i < 4 * jl_n_gcthreads; i++) {
@@ -2928,37 +2930,141 @@ void gc_mark_and_steal(jl_ptls_t ptls)
             if (new_obj != NULL)
                 goto mark;
         }
-        // Try to steal pointer from master thread
-        if (mq_master != NULL) {
-            new_obj = gc_ptr_queue_steal_from(mq_master);
-            if (new_obj != NULL)
-                goto mark;
+        new_obj = gc_ptr_queue_steal_from(mq_master);
+        if (new_obj != NULL)
+            goto mark;
+    }
+    return marked;
+}
+
+#define GC_BACKOFF_MIN_LG2 (10)
+#define GC_BACKOFF_MAX_LG2 (18)
+
+STATIC_INLINE void gc_backoff_reset_state(gc_backoff_state_t *s) JL_NOTSAFEPOINT
+{
+    s->backoff_phase = GC_SPINNING;
+    s->backoff_lg2 = GC_BACKOFF_MIN_LG2;
+    s->n_spins_at_max = 0;
+}
+
+STATIC_INLINE void gc_backoff(gc_backoff_state_t *s) JL_NOTSAFEPOINT
+{
+    if (s->backoff_phase == GC_SPINNING) {
+        // spin for 2^backoff_lg2 cycles
+        uint64_t c0 = cycleclock();
+        do {
+            jl_cpu_pause();
+        } while (cycleclock() - c0 < (1 << s->backoff_lg2));
+        if (s->backoff_lg2 == GC_BACKOFF_MAX_LG2) {
+            s->n_spins_at_max++;
+            // has been spinning for a while... should
+            // just sleep in the next failed steal attempt
+            if (s->n_spins_at_max >= 8) {
+                s->backoff_phase = GC_SLEEPING;
+            }
+        }
+        else {
+            s->backoff_lg2++;
         }
     }
+    else {
+        // sleep for 1ms
+        uv_sleep(1);
+    }
 }
 
-void gc_mark_loop_parallel(jl_ptls_t ptls, int master)
+int gc_some_work_left_in_queue(jl_ptls_t ptls) JL_NOTSAFEPOINT
+ {
+    if (jl_atomic_load_relaxed(&ptls->mark_queue.ptr_queue.bottom) !=
+        jl_atomic_load_relaxed(&ptls->mark_queue.ptr_queue.top)) {
+        return 1;
+    }
+    if (jl_atomic_load_relaxed(&ptls->mark_queue.chunk_queue.bottom) !=
+        jl_atomic_load_relaxed(&ptls->mark_queue.chunk_queue.top)) {
+        return 1;
+    }
+    return 0;
+ }
+
+ int gc_some_work_left(void) JL_NOTSAFEPOINT
+ {
+    for (int i = gc_first_tid; i < gc_first_tid + jl_n_gcthreads; i++) {
+        jl_ptls_t ptls2 = gc_all_tls_states[i];
+        if (gc_some_work_left_in_queue(ptls2)) {
+            return 1;
+        }
+    }
+    int master_tid = jl_atomic_load(&gc_master_tid);
+    if (master_tid != -1) {
+        jl_ptls_t ptls2 = gc_all_tls_states[master_tid];
+        if (gc_some_work_left_in_queue(ptls2)) {
+            return 1;
+        }
+    }
+    return 0;
+ }
+
+void gc_mark_loop_master_init(jl_ptls_t ptls)
+{
+    jl_atomic_store(&gc_master_tid, ptls->tid);
+    // Wake threads up and try to do some work
+    uv_mutex_lock(&gc_threads_lock);
+    jl_atomic_fetch_add(&gc_n_threads_marking, 1);
+    uv_cond_broadcast(&gc_threads_cond);
+    uv_mutex_unlock(&gc_threads_lock);
+    gc_mark_and_steal(ptls);
+    jl_atomic_fetch_add(&gc_n_threads_marking, -1);
+}
+
+void gc_mark_loop_parallel(jl_ptls_t ptls)
 {
-    int backoff = GC_BACKOFF_MIN;
-    if (master) {
-        jl_atomic_store(&gc_master_tid, ptls->tid);
-        // Wake threads up and try to do some work
+    gc_backoff_state_t s;
+    gc_backoff_reset_state(&s);
+    while (jl_atomic_load(&gc_n_threads_marking) > 0) {
+        if (gc_some_work_left()) {
+            // Try to become a thief while other threads are marking
+            jl_atomic_fetch_add(&gc_n_threads_marking, 1);
+            int marked = gc_mark_and_steal(ptls);
+            jl_atomic_fetch_add(&gc_n_threads_marking, -1);
+            if (marked) {
+                gc_backoff_reset_state(&s);
+            }
+        }
+        gc_backoff(&s);
+    }
+}
+
+void gc_mark_loop_master(jl_ptls_t ptls)
+{
+    gc_mark_loop_master_init(ptls);
+    gc_mark_loop_parallel(ptls);
+}
+
+STATIC_INLINE int gc_may_mark(void) JL_NOTSAFEPOINT
+{
+    return jl_atomic_load(&gc_n_threads_marking) > 0;
+}
+
+STATIC_INLINE int gc_may_sweep(jl_ptls_t ptls) JL_NOTSAFEPOINT
+{
+    return jl_atomic_load(&ptls->gc_sweeps_requested) > 0;
+}
+
+void gc_worker_loop(jl_ptls_t ptls)
+{
+    while (1) {
         uv_mutex_lock(&gc_threads_lock);
-        jl_atomic_fetch_add(&gc_n_threads_marking, 1);
-        uv_cond_broadcast(&gc_threads_cond);
+        while (!gc_may_mark() && !gc_may_sweep(ptls)) {
+            uv_cond_wait(&gc_threads_cond, &gc_threads_lock);
+        }
         uv_mutex_unlock(&gc_threads_lock);
-        gc_mark_and_steal(ptls);
-        jl_atomic_fetch_add(&gc_n_threads_marking, -1);
-    }
-    while (jl_atomic_load(&gc_n_threads_marking) > 0) {
-        // Try to become a thief while other threads are marking
-        jl_atomic_fetch_add(&gc_n_threads_marking, 1);
-        if (jl_atomic_load(&gc_master_tid) != -1) {
-            gc_mark_and_steal(ptls);
+        if (gc_may_mark()) {
+            gc_mark_loop_parallel(ptls);
+        }
+        if (gc_may_sweep(ptls)) { // not an else!
+            gc_sweep_pool_parallel();
+            jl_atomic_fetch_add(&ptls->gc_sweeps_requested, -1);
         }
-        jl_atomic_fetch_add(&gc_n_threads_marking, -1);
-        // Failed to steal
-        gc_backoff(&backoff);
     }
 }
 
@@ -2968,7 +3074,7 @@ void gc_mark_loop(jl_ptls_t ptls)
         gc_mark_loop_serial(ptls);
     }
     else {
-        gc_mark_loop_parallel(ptls, 1);
+        gc_mark_loop_master(ptls);
     }
 }
 

src/gc.h

@@ -113,6 +113,16 @@ typedef struct _jl_gc_chunk_t {
 #define GC_PTR_QUEUE_INIT_SIZE (1 << 18)    // initial size of queue of `jl_value_t *`
 #define GC_CHUNK_QUEUE_INIT_SIZE (1 << 14)  // initial size of chunk-queue
 
+// State used for GC scheduling
+typedef struct {
+#define GC_SPINNING 0
+#define GC_SLEEPING 1
+    uint8_t backoff_phase;  // whether the thread is spinning or sleeping
+                            // between failed steal attempts
+    size_t backoff_lg2;     // expontial backoff log counter
+    size_t n_spins_at_max;  // number of times it spinned at the maximum backoff
+} gc_backoff_state_t;
+
 // layout for big (>2k) objects
 
 JL_EXTENSION typedef struct _bigval_t {
@@ -189,19 +199,6 @@ extern jl_gc_page_stack_t global_page_pool_lazily_freed;
 extern jl_gc_page_stack_t global_page_pool_clean;
 extern jl_gc_page_stack_t global_page_pool_freed;
 
-#define GC_BACKOFF_MIN 4
-#define GC_BACKOFF_MAX 12
-
-STATIC_INLINE void gc_backoff(int *i) JL_NOTSAFEPOINT
-{
-    if (*i < GC_BACKOFF_MAX) {
-        (*i)++;
-    }
-    for (int j = 0; j < (1 << *i); j++) {
-        jl_cpu_pause();
-    }
-}
-
 // Lock-free stack implementation taken
 // from Herlihy's "The Art of Multiprocessor Programming"
 // XXX: this is not a general-purpose lock-free stack. We can
@@ -452,8 +449,7 @@ void gc_mark_finlist_(jl_gc_markqueue_t *mq, jl_value_t **fl_begin, jl_value_t *
 void gc_mark_finlist(jl_gc_markqueue_t *mq, arraylist_t *list, size_t start) JL_NOTSAFEPOINT;
 void gc_mark_loop_serial_(jl_ptls_t ptls, jl_gc_markqueue_t *mq);
 void gc_mark_loop_serial(jl_ptls_t ptls);
-void gc_mark_loop_parallel(jl_ptls_t ptls, int master);
-void gc_sweep_pool_parallel(void);
+void gc_worker_loop(jl_ptls_t ptls);
 void gc_free_pages(void);
 void sweep_stack_pools(void);
 void jl_gc_debug_init(void);

src/partr.c

@@ -108,17 +108,7 @@ void jl_init_threadinginfra(void)
 
 void JL_NORETURN jl_finish_task(jl_task_t *t);
 
-static inline int may_mark(void) JL_NOTSAFEPOINT
-{
-    return (jl_atomic_load(&gc_n_threads_marking) > 0);
-}
-
-static inline int may_sweep(jl_ptls_t ptls) JL_NOTSAFEPOINT
-{
-    return (jl_atomic_load(&ptls->gc_sweeps_requested) > 0);
-}
-
-// gc thread function
+// parallel gc thread function
 void jl_gc_threadfun(void *arg)
 {
     jl_threadarg_t *targ = (jl_threadarg_t*)arg;
@@ -133,20 +123,7 @@ void jl_gc_threadfun(void *arg)
     // free the thread argument here
     free(targ);
 
-    while (1) {
-        uv_mutex_lock(&gc_threads_lock);
-        while (!may_mark() && !may_sweep(ptls)) {
-            uv_cond_wait(&gc_threads_cond, &gc_threads_lock);
-        }
-        uv_mutex_unlock(&gc_threads_lock);
-        if (may_mark()) {
-            gc_mark_loop_parallel(ptls, 0);
-        }
-        if (may_sweep(ptls)) { // not an else!
-            gc_sweep_pool_parallel();
-            jl_atomic_fetch_add(&ptls->gc_sweeps_requested, -1);
-        }
-    }
+    gc_worker_loop(ptls);
 }
 
 // thread function: used by all mutator threads except the main thread