From 6f1483221540ddc26066837621cf81aaac701f55 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sat, 24 Dec 2022 06:43:50 -0500 Subject: [PATCH 01/61] Shutdown changes to address JDK-8288899 and JDK-8286352 --- .../java/util/concurrent/ForkJoinPool.java | 160 +++++++++++------- .../java/util/concurrent/ForkJoinTask.java | 9 +- .../concurrent/forkjoin/AsyncShutdownNow.java | 2 +- 3 files changed, 101 insertions(+), 70 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index dd4950269e62d..35df71e001a59 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1651,8 +1651,12 @@ final void registerWorker(WorkQueue w) { * @param ex the exception causing failure, or null if none */ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { + int cfg = 0; WorkQueue w = (wt == null) ? null : wt.workQueue; - int cfg = (w == null) ? 0 : w.config; + if (w != null) { + cfg = w.config; + w.access = STOP; // may be redundant + } long c = ctl; if ((cfg & TRIMMED) == 0) // decrement counts do {} while (c != (c = compareAndExchangeCtl( @@ -1676,8 +1680,7 @@ else if ((int)c == 0) // was dropped on timeout signalWork(); // possibly replace worker } if (ex != null) { - if (w != null) { - w.access = STOP; // cancel tasks + if (w != null) { // cancel tasks for (ForkJoinTask t; (t = w.nextLocalTask(0)) != null; ) ForkJoinTask.cancelIgnoringExceptions(t); } @@ -1807,7 +1810,6 @@ final void runWorker(WorkQueue w) { r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift } while ((src = scan(w, src, r)) >= 0 || (src = awaitWork(w)) == 0); - w.access = STOP; // record normal termination } } @@ -1869,10 +1871,10 @@ private int awaitWork(WorkQueue w) { } while (pc != (pc = compareAndExchangeCtl( pc, qc = ((pc - RC_UNIT) & UC_MASK) | sp))); if ((qc & RC_MASK) <= 0L) { - if (hasTasks(true) && (w.phase >= 0 || reactivate() == w)) - return 0; // check for stragglers if (runState != 0 && tryTerminate(false, false)) return -1; // quiescent termination + if (hasTasks(true) && (w.phase >= 0 || reactivate() == w)) + return 0; // check for stragglers idle = true; } WorkQueue[] qs = queues; // spin for expected #accesses in scan+signal @@ -2500,23 +2502,26 @@ private boolean tryTerminate(boolean now, boolean enable) { } getAndBitwiseOrRunState(SHUTDOWN | STOP); } - WorkQueue released = reactivate(); // try signalling waiter - int tc = (short)(ctl >>> TC_SHIFT); - if (released == null && tc > 0) { // help unblock and cancel - Thread current = Thread.currentThread(); - WorkQueue w = ((current instanceof ForkJoinWorkerThread) ? - ((ForkJoinWorkerThread)current).workQueue : null); - int r = (w == null) ? 0 : w.config + 1; // stagger traversals - WorkQueue[] qs = queues; + int r = 0; // for queue traversal + Thread current = Thread.currentThread(); + if (current instanceof ForkJoinWorkerThread) { + ForkJoinWorkerThread wt = (ForkJoinWorkerThread)current; + WorkQueue w = wt.workQueue; + if (wt.pool == this && w != null) { + r = w.config; // stagger traversals + w.access = STOP; // may be redundant + } + } + if (reactivate() == null) { // try signalling waiter + WorkQueue[] qs = queues; // or help unblock and cancel int n = (qs == null) ? 0 : qs.length; for (int i = 0; i < n; ++i) { WorkQueue q; Thread thread; - if ((q = qs[(r + i) & (n - 1)]) != null && - (thread = q.owner) != current && q.access != STOP) { + if ((q = qs[(r + i) & (n - 1)]) != null) { for (ForkJoinTask t; (t = q.poll(null)) != null; ) ForkJoinTask.cancelIgnoringExceptions(t); - if (thread != null && !thread.isInterrupted()) { - q.forcePhaseActive(); // for awaitWork + if (q.access != STOP && (thread = q.owner) != null) { + q.forcePhaseActive(); // for awaitWork try { thread.interrupt(); } catch (Throwable ignore) { @@ -2525,7 +2530,7 @@ private boolean tryTerminate(boolean now, boolean enable) { } } } - if ((tc <= 0 || (short)(ctl >>> TC_SHIFT) <= 0) && + if ((short)(ctl >>> TC_SHIFT) <= 0 && (getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0 && (lock = registrationLock) != null) { lock.lock(); // signal when no workers @@ -3037,45 +3042,52 @@ public List> invokeAll(Collection> tasks, } } - // Task to hold results from InvokeAnyTasks + /** + * Task to hold results for invokeAny, or to report exception if + * all subtasks fail or are cancelled or the pool is terminating. + */ static final class InvokeAnyRoot extends ForkJoinTask { private static final long serialVersionUID = 2838392045355241008L; @SuppressWarnings("serial") // Conditionally serializable volatile E result; - final AtomicInteger count; // in case all throw + final AtomicInteger count; // in case all fail @SuppressWarnings("serial") final ForkJoinPool pool; // to check shutdown while collecting InvokeAnyRoot(int n, ForkJoinPool p) { pool = p; count = new AtomicInteger(n); } - final void tryComplete(Callable c) { // called by InvokeAnyTasks - Throwable ex = null; - boolean failed; - if (c == null || Thread.interrupted() || - (pool != null && pool.runState < 0)) - failed = true; - else if (isDone()) - failed = false; - else { - try { - complete(c.call()); - failed = false; - } catch (Throwable tx) { - ex = tx; - failed = true; + final void tryComplete(E v, Throwable ex, boolean fail) { + if (!isDone()) { + if (!fail) { + result = v; + quietlyComplete(); } + else if (pool.runState < 0 || count.getAndDecrement() <= 1) + trySetThrown(ex != null? ex : new CancellationException()); + } + } + final boolean checkDone() { + if (isDone()) + return true; + else if (pool.runState >= 0) + return false; + else { + tryComplete(null, null, true); + return true; } - if ((pool != null && pool.runState < 0) || - (failed && count.getAndDecrement() <= 1)) - trySetThrown(ex != null ? ex : new CancellationException()); } - public final boolean exec() { return false; } // never forked + public final boolean exec() { return false; } public final E getRawResult() { return result; } public final void setRawResult(E v) { result = v; } } - // Variant of AdaptedInterruptibleCallable with results in InvokeAnyRoot + /** + * Variant of AdaptedInterruptibleCallable with results in + * InvokeAnyRoot (and never independently joined). Task + * cancellation status is used to avoid multiple calls to + * root.tryComplete by the same task under async cancellation. + */ static final class InvokeAnyTask extends ForkJoinTask { private static final long serialVersionUID = 2838392045355241008L; final InvokeAnyRoot root; @@ -3087,23 +3099,36 @@ static final class InvokeAnyTask extends ForkJoinTask { this.callable = callable; } public final boolean exec() { + InvokeAnyRoot r; Callable c; Thread.interrupted(); - runner = Thread.currentThread(); - root.tryComplete(callable); - runner = null; - Thread.interrupted(); + if ((c = callable) != null && (r = root) != null && !r.checkDone()) { + runner = Thread.currentThread(); + E v = null; + Throwable ex = null; + boolean fail = false; + try { + v = c.call(); + } catch (Throwable rex) { + ex = rex; + fail = true; + } + runner = null; + if (trySetCancelled() >= 0) // else lost to async cancel + r.tryComplete(v, ex, fail); + } return true; } public final boolean cancel(boolean mayInterruptIfRunning) { - Thread t; - boolean stat = super.cancel(false); + Thread t; InvokeAnyRoot r; + if (trySetCancelled() >= 0 && (r = root) != null) + r.tryComplete(null, null, true); // else lost race to cancel if (mayInterruptIfRunning && (t = runner) != null) { try { t.interrupt(); } catch (Throwable ignore) { } } - return stat; + return isCancelled(); } public final void setRawResult(E v) {} // unused public final E getRawResult() { return null; } @@ -3591,25 +3616,30 @@ public boolean awaitQuiescence(long timeout, TimeUnit unit) { */ @Override public void close() { - if ((config & ISCOMMON) == 0) { - boolean terminated = tryTerminate(false, false); - if (!terminated) { - shutdown(); - boolean interrupted = false; - while (!terminated) { - try { - terminated = awaitTermination(1L, TimeUnit.DAYS); - } catch (InterruptedException e) { - if (!interrupted) { - shutdownNow(); - interrupted = true; - } - } - } - if (interrupted) { - Thread.currentThread().interrupt(); + ReentrantLock lock = registrationLock; + Condition cond = null; + boolean interrupted = false; + if (lock != null && (config & ISCOMMON) == 0 && + (runState & TERMINATED) == 0) { + checkPermission(); + for (;;) { + tryTerminate(interrupted, true); // call outside of lock + if ((runState & TERMINATED) != 0) + break; + lock.lock(); + try { + if (cond == null && (cond = termination) == null) + termination = cond = lock.newCondition(); + if ((runState & TERMINATED) == 0) + cond.await(); + } catch (InterruptedException ex) { + interrupted = true; + } finally { + lock.unlock(); } } + if (interrupted) + Thread.currentThread().interrupt(); } } diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index e0737cde89d19..e238850359bbc 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -322,11 +322,11 @@ private int setDone() { /** * Sets ABNORMAL DONE status unless already done, and wakes up threads * waiting to join this task. - * @return status on exit + * @return previous status */ - private int trySetCancelled() { + final int trySetCancelled() { int s; - do {} while ((s = status) >= 0 && !casStatus(s, s |= (DONE | ABNORMAL))); + do {} while ((s = status) >= 0 && !casStatus(s, s | (DONE | ABNORMAL))); signalWaiters(); return s; } @@ -852,7 +852,8 @@ public static > Collection invokeAll(Collection * @return {@code true} if this task is now cancelled */ public boolean cancel(boolean mayInterruptIfRunning) { - return (trySetCancelled() & (ABNORMAL | THROWN)) == ABNORMAL; + trySetCancelled(); + return isCancelled(); } public final boolean isDone() { diff --git a/test/jdk/java/util/concurrent/forkjoin/AsyncShutdownNow.java b/test/jdk/java/util/concurrent/forkjoin/AsyncShutdownNow.java index 7f7618a2189c9..d742bc0955fd3 100644 --- a/test/jdk/java/util/concurrent/forkjoin/AsyncShutdownNow.java +++ b/test/jdk/java/util/concurrent/forkjoin/AsyncShutdownNow.java @@ -128,7 +128,7 @@ public void testInvokeAll(ExecutorService executor) throws Exception { /** * Test shutdownNow with thread blocked in invokeAny. */ - @Test(dataProvider = "executors", enabled = false) + @Test(dataProvider = "executors") public void testInvokeAny(ExecutorService executor) throws Exception { System.out.format("testInvokeAny: %s%n", executor); try (executor) { From 8a6ff99537b0363044a279eac588cff053194d90 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Wed, 28 Dec 2022 15:45:21 -0500 Subject: [PATCH 02/61] Refactor and regularize use of tryTerminate --- .../java/util/concurrent/ForkJoinPool.java | 102 +++++++++--------- 1 file changed, 54 insertions(+), 48 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 35df71e001a59..bd101fe5a8315 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1665,7 +1665,7 @@ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { (SP_MASK & c))))); else if ((int)c == 0) // was dropped on timeout cfg &= ~SRC; // suppress signal if last - if (!tryTerminate(false, false) && w != null) { + if (tryTerminate(false, false) >= 0 && w != null) { ReentrantLock lock; WorkQueue[] qs; int n, i; long ns = w.nsteals & 0xffffffffL; if ((lock = registrationLock) != null) { @@ -1871,7 +1871,7 @@ private int awaitWork(WorkQueue w) { } while (pc != (pc = compareAndExchangeCtl( pc, qc = ((pc - RC_UNIT) & UC_MASK) | sp))); if ((qc & RC_MASK) <= 0L) { - if (runState != 0 && tryTerminate(false, false)) + if (runState != 0 && tryTerminate(false, false) < 0) return -1; // quiescent termination if (hasTasks(true) && (w.phase >= 0 || reactivate() == w)) return 0; // check for stragglers @@ -2484,25 +2484,31 @@ static int getSurplusQueuedTaskCount() { * @param now if true, unconditionally terminate, else only * if no work and no active workers * @param enable if true, terminate when next possible - * @return true if terminating or terminated - */ - private boolean tryTerminate(boolean now, boolean enable) { - int rs; ReentrantLock lock; Condition cond; - if ((rs = runState) >= 0) { // set SHUTDOWN and/or STOP - if ((config & ISCOMMON) != 0) - return false; // cannot shutdown - if (!now) { - if ((rs & SHUTDOWN) == 0) { - if (!enable) - return false; - getAndBitwiseOrRunState(SHUTDOWN); - } - if (!canStop()) - return false; - } - getAndBitwiseOrRunState(SHUTDOWN | STOP); + * @return runState on exit + */ + private int tryTerminate(boolean now, boolean enable) { + int rs; + if ((rs = runState) >= 0 && (config & ISCOMMON) == 0) { + if (!now && enable && (rs & SHUTDOWN) == 0) + rs = getAndBitwiseOrRunState(SHUTDOWN) | SHUTDOWN; + if (now || ((rs & SHUTDOWN) != 0 && canStop())) + rs = getAndBitwiseOrRunState(SHUTDOWN | STOP) | STOP; + else + rs = runState; + } + if (rs < 0 && (rs & TERMINATED) == 0) { + helpTerminate(); + rs = runState; } - int r = 0; // for queue traversal + return rs; + } + + /** + * Helps complete termination by cancelling tasks, unblocking + * workers and possibly triggering TERMINATED state. + */ + private void helpTerminate() { + int r = 1; // for queue traversal Thread current = Thread.currentThread(); if (current instanceof ForkJoinWorkerThread) { ForkJoinWorkerThread wt = (ForkJoinWorkerThread)current; @@ -2512,16 +2518,20 @@ private boolean tryTerminate(boolean now, boolean enable) { w.access = STOP; // may be redundant } } - if (reactivate() == null) { // try signalling waiter - WorkQueue[] qs = queues; // or help unblock and cancel - int n = (qs == null) ? 0 : qs.length; - for (int i = 0; i < n; ++i) { - WorkQueue q; Thread thread; - if ((q = qs[(r + i) & (n - 1)]) != null) { - for (ForkJoinTask t; (t = q.poll(null)) != null; ) - ForkJoinTask.cancelIgnoringExceptions(t); - if (q.access != STOP && (thread = q.owner) != null) { - q.forcePhaseActive(); // for awaitWork + WorkQueue[] qs; WorkQueue q; Thread thread; + int n = ((qs = queues) == null) ? 0 : qs.length; + for (int i = 0; i < n; ++i) { // help cancel tasks + if ((q = qs[(r + i) & (n - 1)]) != null) { + for (ForkJoinTask t; (t = q.poll(null)) != null; ) + ForkJoinTask.cancelIgnoringExceptions(t); + } + } + if (reactivate() == null) { // activate or help unblock + n = ((qs = queues) == null) ? 0 : qs.length; + for (int i = 0; i < n; i += 2) { + if ((q = qs[(r + i) & (n - 1)]) != null && q.access != STOP) { + q.forcePhaseActive(); // for awaitWork + if ((thread = q.owner) != null && !thread.isInterrupted()) { try { thread.interrupt(); } catch (Throwable ignore) { @@ -2530,16 +2540,16 @@ private boolean tryTerminate(boolean now, boolean enable) { } } } - if ((short)(ctl >>> TC_SHIFT) <= 0 && + ReentrantLock lock; Condition cond; // signal when no workers + if ((short)(ctl >>> TC_SHIFT) <= 0 && (runState & TERMINATED) == 0 && (getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0 && (lock = registrationLock) != null) { - lock.lock(); // signal when no workers + lock.lock(); if ((cond = termination) != null) cond.signalAll(); lock.unlock(); container.close(); } - return true; } // Exported methods @@ -3504,7 +3514,7 @@ public List shutdownNow() { * @return {@code true} if all tasks have completed following shut down */ public boolean isTerminated() { - return (runState & TERMINATED) != 0; + return (tryTerminate(false, false) & TERMINATED) != 0; } /** @@ -3549,30 +3559,29 @@ public boolean isShutdown() { */ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { - ReentrantLock lock; Condition cond; boolean terminated; + ReentrantLock lock; Condition cond = null; long nanos = unit.toNanos(timeout); if ((config & ISCOMMON) != 0) { if (helpQuiescePool(this, nanos, true) < 0) throw new InterruptedException(); - terminated = false; + return false; } - else if (!(terminated = ((runState & TERMINATED) != 0))) { - tryTerminate(false, false); // reduce transient blocking - if ((lock = registrationLock) != null && - !(terminated = (((runState & TERMINATED) != 0)))) { + else if ((lock = registrationLock) != null) { + while ((tryTerminate(false, false) & TERMINATED) == 0) { + if (nanos <= 0L) + return false; lock.lock(); try { - if ((cond = termination) == null) + if (cond == null && (cond = termination) == null) termination = cond = lock.newCondition(); - while (!(terminated = ((runState & TERMINATED) != 0)) && - nanos > 0L) + if ((runState & TERMINATED) == 0) nanos = cond.awaitNanos(nanos); } finally { lock.unlock(); } } } - return terminated; + return true; } /** @@ -3622,10 +3631,7 @@ public void close() { if (lock != null && (config & ISCOMMON) == 0 && (runState & TERMINATED) == 0) { checkPermission(); - for (;;) { - tryTerminate(interrupted, true); // call outside of lock - if ((runState & TERMINATED) != 0) - break; + while ((tryTerminate(interrupted, true) & TERMINATED) == 0) { lock.lock(); try { if (cond == null && (cond = termination) == null) From 4bfaac195c68d4293976f408da4d0a0fc5f5e665 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sun, 1 Jan 2023 15:25:43 -0500 Subject: [PATCH 03/61] More uniform termnination for interruptibles --- .../java/util/concurrent/ForkJoinPool.java | 302 ++++++++++-------- .../java/util/concurrent/ForkJoinTask.java | 36 ++- 2 files changed, 187 insertions(+), 151 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index bd101fe5a8315..8645631a552d4 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -582,25 +582,18 @@ public class ForkJoinPool extends AbstractExecutorService { * * Shutdown and Termination. A call to shutdownNow invokes * tryTerminate to atomically set a mode bit. The calling thread, - * as well as every other worker thereafter terminating, helps - * terminate others by cancelling their unprocessed tasks, and - * interrupting other workers. Calls to non-abrupt shutdown() - * preface this by checking isQuiescent before triggering the - * "STOP" phase of termination. During termination, workers are - * stopped using all three of (often in parallel): releasing via - * ctl (method reactivate), interrupts, and cancelling tasks that - * will cause workers to not find work and exit. To support this, - * worker references not removed from the queues array during - * termination. It is possible for late thread creations to still - * be in progress after a quiescent termination reports terminated - * status, but they will also immediately terminate. To conform to - * ExecutorService invoke, invokeAll, and invokeAny specs, we must - * track pool status while waiting in ForkJoinTask.awaitDone, and - * interrupt interruptible callers on termination, while also - * avoiding cancelling other tasks that are normally completing - * during quiescent termination. This is tracked by recording - * ForkJoinTask.POOLSUBMIT in task status and/or as a bit flag - * argument to joining methods. + * as well as every other worker thereafter terminating invokes + * helpTerminate, which cancels queued tasks, reactivates idle + * workers, and interrupts others (in addtion to deregistering + * itself). Calls to non-abrupt shutdown() preface this by + * checking canStop before triggering the "STOP" phase of + * termination. Like most eveything else, we try to speed up + * termination by balancing parallelism and contention (which may + * occur while trying to cancel tasks or unblock workers), by + * starting task scans at different indices, and by fanning out + * reactivation of idle workers. The main remaining cost is the + * potential need to re-interrupt active workers with tasks that + * swallow them. * * Trimming workers. To release resources after periods of lack of * use, a worker starting to wait when the pool is quiescent will @@ -727,20 +720,40 @@ public class ForkJoinPool extends AbstractExecutorService { * may be JVM-dependent and must access particular Thread class * fields to achieve this effect. * - * Interrupt handling - * ================== + * Interrupt handling and Cancellation + * =================================== * - * The framework is designed to manage task cancellation + * The framework is primarily designed to manage task cancellation * (ForkJoinTask.cancel) independently from the interrupt status * of threads running tasks. (See the public ForkJoinTask - * documentation for rationale.) Interrupts are issued only in - * tryTerminate, when workers should be terminating and tasks - * should be cancelled anyway. Interrupts are cleared only when - * necessary to ensure that calls to LockSupport.park do not loop - * indefinitely (park returns immediately if the current thread is - * interrupted). For cases in which task bodies are specified or - * desired to interrupt upon cancellation, ForkJoinTask.cancel can - * be overridden to do so (as is done for invoke{Any,All}). + * documentation for rationale.) Interrupts are issued internally + * only in helpTerminate, when workers should be terminating and + * tasks should be cancelled anyway. By default, interrupts are + * cleared only when necessary to ensure that calls to + * LockSupport.park do not loop indefinitely (park returns + * immediately if the current thread is interrupted). + * + * To conform to specs and expectations surrounding execution, + * cancellation and termination of ExecutorService invoke, submit, + * invokeAll, and invokeAny methods (without penalizing others) we + * use AdaptedInterruptibleCallables in ExecutorService methods + * accepting Callables (actually, for InvokeAny, a variant that + * gathers a single result). We also ensure that external + * submitters do not help run such tasks by recording + * ForkJoinTask.POOLSUBMIT in task status and/or as a bit flag + * argument to joining methods. External callers of task.get etc + * are not directly interrupted on shutdown, but must be woken due + * to the task itself being cancelled during termination. + * Interruptible tasks always clear interrupts and check for + * termination and cancellation before invoking user-supplied + * Callables These rechecks are needed because the cleared + * interrupt could be due to cancellation, termination, or any + * other reason. These tasks also record their runner thread so + * that cancel(true) can interrupt them later. As is the case with + * any kind of pool, it is possible that an interrupt designed to + * cancel one task occurs both late and unnecessarily, so instead + * "spuriously" interrupts the thread while performing a later + * task. * * Memory placement * ================ @@ -1440,6 +1453,11 @@ else if (a[nk] == null) // misc + final void cancelAllTasks() { + for (ForkJoinTask t; (t = poll(null)) != null; ) + ForkJoinTask.cancelIgnoringExceptions(t); + } + /** * Returns true if owned by a worker thread and not known to be blocked. */ @@ -1676,16 +1694,12 @@ else if ((int)c == 0) // was dropped on timeout stealCount += ns; // accumulate steals lock.unlock(); } + w.cancelAllTasks(); if ((cfg & SRC) != 0) signalWork(); // possibly replace worker } - if (ex != null) { - if (w != null) { // cancel tasks - for (ForkJoinTask t; (t = w.nextLocalTask(0)) != null; ) - ForkJoinTask.cancelIgnoringExceptions(t); - } + if (ex != null) ForkJoinTask.rethrow(ex); - } } /* @@ -1819,13 +1833,15 @@ final void runWorker(WorkQueue w) { * returning source id or retry indicator. * * @param w caller's WorkQueue - * @param prevSrc the two previous queues (if nonzero) stolen from in current phase, packed as int + * @param prevSrc the two previous queues (if nonzero) stolen from + * in current phase, packed as int * @param r random seed * @return the next prevSrc value to use, or negative if none found */ private int scan(WorkQueue w, int prevSrc, int r) { + int rs = runState; WorkQueue[] qs = queues; - int n = (w == null || qs == null) ? 0 : qs.length; + int n = (rs < 0 || qs == null || w == null) ? 0 : qs.length; for (int step = (r >>> 16) | 1, i = n; i > 0; --i, r += step) { int j, cap; WorkQueue q; ForkJoinTask[] a; if ((q = qs[j = r & (n - 1)]) != null && @@ -1871,22 +1887,20 @@ private int awaitWork(WorkQueue w) { } while (pc != (pc = compareAndExchangeCtl( pc, qc = ((pc - RC_UNIT) & UC_MASK) | sp))); if ((qc & RC_MASK) <= 0L) { - if (runState != 0 && tryTerminate(false, false) < 0) - return -1; // quiescent termination if (hasTasks(true) && (w.phase >= 0 || reactivate() == w)) return 0; // check for stragglers + if (runState != 0 && tryTerminate(false, false) < 0) + return -1; // quiescent termination idle = true; } WorkQueue[] qs = queues; // spin for expected #accesses in scan+signal int spins = ((qs == null) ? 0 : ((qs.length & SMASK) << 1)) | 0xf; while ((p = w.phase) < 0 && --spins > 0) Thread.onSpinWait(); - if (p < 0) { + if (p < 0) { // await signal long deadline = idle ? keepAlive + System.currentTimeMillis() : 0L; LockSupport.setCurrentBlocker(this); - for (;;) { // await signal or termination - if (runState < 0) - return -1; + for (;;) { w.access = PARKED; // enable unpark if (w.phase < 0) { if (idle) @@ -2487,18 +2501,18 @@ static int getSurplusQueuedTaskCount() { * @return runState on exit */ private int tryTerminate(boolean now, boolean enable) { - int rs; - if ((rs = runState) >= 0 && (config & ISCOMMON) == 0) { - if (!now && enable && (rs & SHUTDOWN) == 0) - rs = getAndBitwiseOrRunState(SHUTDOWN) | SHUTDOWN; - if (now || ((rs & SHUTDOWN) != 0 && canStop())) - rs = getAndBitwiseOrRunState(SHUTDOWN | STOP) | STOP; - else + int rs = runState; + if ((config & ISCOMMON) == 0) { + if (rs >= 0) { + if (!now && enable && (rs & SHUTDOWN) == 0) + rs = getAndBitwiseOrRunState(SHUTDOWN) | SHUTDOWN; + if (now || ((rs & SHUTDOWN) != 0 && canStop())) + rs = getAndBitwiseOrRunState(SHUTDOWN | STOP) | STOP; + } + if ((rs < 0 || (rs = runState) < 0) && (rs & TERMINATED) == 0) { + helpTerminate(); rs = runState; - } - if (rs < 0 && (rs & TERMINATED) == 0) { - helpTerminate(); - rs = runState; + } } return rs; } @@ -2520,18 +2534,21 @@ private void helpTerminate() { } WorkQueue[] qs; WorkQueue q; Thread thread; int n = ((qs = queues) == null) ? 0 : qs.length; - for (int i = 0; i < n; ++i) { // help cancel tasks - if ((q = qs[(r + i) & (n - 1)]) != null) { - for (ForkJoinTask t; (t = q.poll(null)) != null; ) - ForkJoinTask.cancelIgnoringExceptions(t); - } - } - if (reactivate() == null) { // activate or help unblock + for (int i = 0; i < n; ++i) { // help cancel tasks + if ((q = qs[(r + i) & (n - 1)]) != null) + q.cancelAllTasks(); + } + if (reactivate() != null) // activate <= 2 idle workers + reactivate(); + else if ((runState & TERMINATED) == 0) { // or unblock active workers + boolean canTerminate = true; n = ((qs = queues) == null) ? 0 : qs.length; for (int i = 0; i < n; i += 2) { - if ((q = qs[(r + i) & (n - 1)]) != null && q.access != STOP) { + if ((q = qs[(r + i) & (n - 1)]) != null && + (thread = q.owner) != null && q.access != STOP) { + canTerminate = false; q.forcePhaseActive(); // for awaitWork - if ((thread = q.owner) != null && !thread.isInterrupted()) { + if (!thread.isInterrupted()) { try { thread.interrupt(); } catch (Throwable ignore) { @@ -2539,16 +2556,16 @@ private void helpTerminate() { } } } - } - ReentrantLock lock; Condition cond; // signal when no workers - if ((short)(ctl >>> TC_SHIFT) <= 0 && (runState & TERMINATED) == 0 && - (getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0 && - (lock = registrationLock) != null) { - lock.lock(); - if ((cond = termination) != null) - cond.signalAll(); - lock.unlock(); - container.close(); + ReentrantLock lock; Condition cond; // signal when no workers + if (canTerminate && (short)(ctl >>> TC_SHIFT) <= 0 && + (getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0 && + (lock = registrationLock) != null) { + lock.lock(); + if ((cond = termination) != null) + cond.signalAll(); + lock.unlock(); + container.close(); + } } } @@ -2889,7 +2906,7 @@ public ForkJoinTask submit(ForkJoinTask task) { */ @Override public ForkJoinTask submit(Callable task) { - return poolSubmit(true, new ForkJoinTask.AdaptedCallable(task)); + return poolSubmit(true, new ForkJoinTask.AdaptedInterruptibleCallable(task)); } /** @@ -3008,13 +3025,12 @@ public List> invokeAll(Collection> tasks) { futures.add(f); poolSubmit(true, f); } - for (int i = futures.size() - 1; i >= 0; --i) + for (int i = futures.size() - 1; i >= 0 && runState >= 0; --i) ((ForkJoinTask)futures.get(i)).quietlyJoin(); return futures; - } catch (Throwable t) { + } finally { for (Future e : futures) ForkJoinTask.cancelIgnoringExceptions(e); - throw t; } } @@ -3033,7 +3049,7 @@ public List> invokeAll(Collection> tasks, } long startTime = System.nanoTime(), ns = nanos; boolean timedOut = (ns < 0L); - for (int i = futures.size() - 1; i >= 0; --i) { + for (int i = futures.size() - 1; i >= 0 && runState >= 0; --i) { ForkJoinTask f = (ForkJoinTask)futures.get(i); if (!f.isDone()) { if (!timedOut) @@ -3045,10 +3061,9 @@ public List> invokeAll(Collection> tasks, } } return futures; - } catch (Throwable t) { + } finally { for (Future e : futures) ForkJoinTask.cancelIgnoringExceptions(e); - throw t; } } @@ -3062,41 +3077,38 @@ static final class InvokeAnyRoot extends ForkJoinTask { volatile E result; final AtomicInteger count; // in case all fail @SuppressWarnings("serial") - final ForkJoinPool pool; // to check shutdown while collecting - InvokeAnyRoot(int n, ForkJoinPool p) { - pool = p; + InvokeAnyRoot(int n) { count = new AtomicInteger(n); } - final void tryComplete(E v, Throwable ex, boolean fail) { + final boolean checkDone() { + ForkJoinPool p; // force done if caller's pool terminating + if (!isDone()) { + if ((p = getPool()) == null || p.runState >= 0) + return false; + cancel(false); + } + return true; + } + final void tryComplete(E v, Throwable ex, boolean completed) { if (!isDone()) { - if (!fail) { + if (completed) { result = v; quietlyComplete(); } - else if (pool.runState < 0 || count.getAndDecrement() <= 1) - trySetThrown(ex != null? ex : new CancellationException()); - } - } - final boolean checkDone() { - if (isDone()) - return true; - else if (pool.runState >= 0) - return false; - else { - tryComplete(null, null, true); - return true; + else if (count.getAndDecrement() <= 1) + trySetThrown(ex != null ? ex : new CancellationException()); } } - public final boolean exec() { return false; } + public final boolean exec() { return false; } // never forked public final E getRawResult() { return result; } public final void setRawResult(E v) { result = v; } } /** * Variant of AdaptedInterruptibleCallable with results in - * InvokeAnyRoot (and never independently joined). Task - * cancellation status is used to avoid multiple calls to - * root.tryComplete by the same task under async cancellation. + * InvokeAnyRoot (and never independently joined). Cancellation + * status is used to avoid multiple calls to tryComplete by the + * same task under async cancellation. */ static final class InvokeAnyTask extends ForkJoinTask { private static final long serialVersionUID = 2838392045355241008L; @@ -3109,36 +3121,42 @@ static final class InvokeAnyTask extends ForkJoinTask { this.callable = callable; } public final boolean exec() { - InvokeAnyRoot r; Callable c; Thread.interrupted(); - if ((c = callable) != null && (r = root) != null && !r.checkDone()) { - runner = Thread.currentThread(); - E v = null; - Throwable ex = null; - boolean fail = false; + runner = Thread.currentThread(); + InvokeAnyRoot r = root; + Callable c = callable; + E v = null; + Throwable ex = null; + boolean completed = false; + if (r != null && !r.checkDone() && !isDone()) { try { - v = c.call(); + if (c != null) { + v = c.call(); + completed = true; + } } catch (Throwable rex) { ex = rex; - fail = true; } - runner = null; - if (trySetCancelled() >= 0) // else lost to async cancel - r.tryComplete(v, ex, fail); + if (trySetCancelled() >= 0) + r.tryComplete(v, ex, completed); } + runner = null; return true; } public final boolean cancel(boolean mayInterruptIfRunning) { - Thread t; InvokeAnyRoot r; - if (trySetCancelled() >= 0 && (r = root) != null) - r.tryComplete(null, null, true); // else lost race to cancel - if (mayInterruptIfRunning && (t = runner) != null) { - try { - t.interrupt(); - } catch (Throwable ignore) { + int s; Thread t; InvokeAnyRoot r; + if ((s = trySetCancelled()) >= 0) { + if ((r = root) != null) + r.tryComplete(null, null, false); + if (mayInterruptIfRunning && (t = runner) != null) { + try { + t.interrupt(); + } catch (Throwable ignore) { + } } + return true; } - return isCancelled(); + return ((s & (ABNORMAL | THROWN)) == ABNORMAL); } public final void setRawResult(E v) {} // unused public final E getRawResult() { return null; } @@ -3150,7 +3168,7 @@ public T invokeAny(Collection> tasks) int n = tasks.size(); if (n <= 0) throw new IllegalArgumentException(); - InvokeAnyRoot root = new InvokeAnyRoot(n, this); + InvokeAnyRoot root = new InvokeAnyRoot(n); ArrayList> fs = new ArrayList<>(n); try { for (Callable c : tasks) { @@ -3162,7 +3180,11 @@ public T invokeAny(Collection> tasks) if (root.isDone()) break; } - return root.get(); + try { + return root.get(); + } catch (CancellationException cx) { + throw new ExecutionException(cx); + } } finally { for (InvokeAnyTask f : fs) ForkJoinTask.cancelIgnoringExceptions(f); @@ -3177,7 +3199,7 @@ public T invokeAny(Collection> tasks, int n = tasks.size(); if (n <= 0) throw new IllegalArgumentException(); - InvokeAnyRoot root = new InvokeAnyRoot(n, this); + InvokeAnyRoot root = new InvokeAnyRoot(n); ArrayList> fs = new ArrayList<>(n); try { for (Callable c : tasks) { @@ -3189,7 +3211,11 @@ public T invokeAny(Collection> tasks, if (root.isDone()) break; } - return root.get(nanos, TimeUnit.NANOSECONDS); + try { + return root.get(nanos, TimeUnit.NANOSECONDS); + } catch (CancellationException cx) { + throw new ExecutionException(cx); + } } finally { for (InvokeAnyTask f : fs) ForkJoinTask.cancelIgnoringExceptions(f); @@ -3514,6 +3540,7 @@ public List shutdownNow() { * @return {@code true} if all tasks have completed following shut down */ public boolean isTerminated() { + // reduce false negatives during termination by helping return (tryTerminate(false, false) & TERMINATED) != 0; } @@ -3531,7 +3558,7 @@ public boolean isTerminated() { * @return {@code true} if terminating but not yet terminated */ public boolean isTerminating() { - return (runState & (STOP | TERMINATED)) == STOP; + return (tryTerminate(false, false) & (STOP | TERMINATED)) == STOP; } /** @@ -3559,7 +3586,7 @@ public boolean isShutdown() { */ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { - ReentrantLock lock; Condition cond = null; + ReentrantLock lock; long nanos = unit.toNanos(timeout); if ((config & ISCOMMON) != 0) { if (helpQuiescePool(this, nanos, true) < 0) @@ -3567,6 +3594,7 @@ public boolean awaitTermination(long timeout, TimeUnit unit) return false; } else if ((lock = registrationLock) != null) { + Condition cond = null; while ((tryTerminate(false, false) & TERMINATED) == 0) { if (nanos <= 0L) return false; @@ -3574,8 +3602,9 @@ else if ((lock = registrationLock) != null) { try { if (cond == null && (cond = termination) == null) termination = cond = lock.newCondition(); - if ((runState & TERMINATED) == 0) - nanos = cond.awaitNanos(nanos); + if ((runState & TERMINATED) != 0) + break; + nanos = cond.awaitNanos(nanos); } finally { lock.unlock(); } @@ -3625,19 +3654,20 @@ public boolean awaitQuiescence(long timeout, TimeUnit unit) { */ @Override public void close() { - ReentrantLock lock = registrationLock; - Condition cond = null; + ReentrantLock lock; boolean interrupted = false; - if (lock != null && (config & ISCOMMON) == 0 && - (runState & TERMINATED) == 0) { + if ((runState & TERMINATED) == 0 && (config & ISCOMMON) == 0 && + (lock = registrationLock) != null) { checkPermission(); + Condition cond = null; while ((tryTerminate(interrupted, true) & TERMINATED) == 0) { lock.lock(); try { if (cond == null && (cond = termination) == null) termination = cond = lock.newCondition(); - if ((runState & TERMINATED) == 0) - cond.await(); + if ((runState & TERMINATED) != 0) + break; + cond.await(); } catch (InterruptedException ex) { interrupted = true; } finally { @@ -3827,7 +3857,7 @@ protected RunnableFuture newTaskFor(Runnable runnable, T value) { @Override protected RunnableFuture newTaskFor(Callable callable) { - return new ForkJoinTask.AdaptedCallable(callable); + return new ForkJoinTask.AdaptedInterruptibleCallable(callable); } static { diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index e238850359bbc..1897572ed4124 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -413,7 +413,9 @@ private int awaitDone(int how, long deadline) { q = (wt = (ForkJoinWorkerThread)t).workQueue; p = wt.pool; } - else if ((p = ForkJoinPool.common) != null && (how & POOLSUBMIT) == 0) + else if ((how & POOLSUBMIT) != 0) + p = null; + else if ((p = ForkJoinPool.common) != null) q = p.externalQueue(); if (q != null && p != null) { // try helping if (this instanceof CountedCompleter) @@ -433,6 +435,8 @@ else if ((how & RAN) != 0 || Aux a; if ((s = status) < 0) break; + else if (p != null && p.runState < 0) + cancelIgnoringExceptions(this); // cancel on shutdown else if (node == null) node = new Aux(Thread.currentThread(), null); else if (!queued) { @@ -446,9 +450,7 @@ else if (timed && (ns = deadline - System.nanoTime()) <= 0) { } else if (Thread.interrupted()) { interrupted = true; - if ((how & POOLSUBMIT) != 0 && p != null && p.runState < 0) - cancelIgnoringExceptions(this); // cancel on shutdown - else if ((how & INTERRUPTIBLE) != 0) { + if ((how & INTERRUPTIBLE) != 0) { s = ABNORMAL; break; } @@ -852,8 +854,8 @@ public static > Collection invokeAll(Collection * @return {@code true} if this task is now cancelled */ public boolean cancel(boolean mayInterruptIfRunning) { - trySetCancelled(); - return isCancelled(); + int s = trySetCancelled(); + return (s >= 0 || (s & (ABNORMAL | THROWN)) == ABNORMAL); } public final boolean isDone() { @@ -1483,10 +1485,13 @@ static final class AdaptedInterruptibleCallable extends ForkJoinTask public final T getRawResult() { return result; } public final void setRawResult(T v) { result = v; } public final boolean exec() { + ForkJoinPool p; // for termination check Thread.interrupted(); runner = Thread.currentThread(); try { - if (!isDone()) // recheck + if ((p = getPool()) != null && p.runState < 0) + trySetCancelled(); + else if (!isDone()) result = callable.call(); return true; } catch (RuntimeException rex) { @@ -1495,20 +1500,21 @@ public final boolean exec() { throw new RuntimeException(ex); } finally { runner = null; - Thread.interrupted(); } } public final void run() { invoke(); } public final boolean cancel(boolean mayInterruptIfRunning) { - Thread t; - boolean stat = super.cancel(false); - if (mayInterruptIfRunning && (t = runner) != null) { - try { - t.interrupt(); - } catch (Throwable ignore) { + int s; Thread t; + if ((s = trySetCancelled()) >= 0) { + if (mayInterruptIfRunning && (t = runner) != null) { + try { + t.interrupt(); + } catch (Throwable ignore) { + } } + return true; } - return stat; + return ((s & (ABNORMAL | THROWN)) == ABNORMAL); } public String toString() { return super.toString() + "[Wrapped task = " + callable + "]"; From 9412a0ee25fd1f54983d2ce92d5320643d0fce62 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Thu, 5 Jan 2023 13:52:04 -0500 Subject: [PATCH 04/61] Make ExecutorService method exceptions conform to others. --- .../java/util/concurrent/ForkJoinPool.java | 204 ++++++++---------- .../java/util/concurrent/ForkJoinTask.java | 41 ++-- 2 files changed, 112 insertions(+), 133 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 8645631a552d4..a343af9c6203c 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -582,18 +582,17 @@ public class ForkJoinPool extends AbstractExecutorService { * * Shutdown and Termination. A call to shutdownNow invokes * tryTerminate to atomically set a mode bit. The calling thread, - * as well as every other worker thereafter terminating invokes - * helpTerminate, which cancels queued tasks, reactivates idle - * workers, and interrupts others (in addtion to deregistering - * itself). Calls to non-abrupt shutdown() preface this by - * checking canStop before triggering the "STOP" phase of + * as well as other workers thereafter terminating, and external + * callers checking termination helps cancel queued tasks, + * reactivate idle workers, and interrupt others (in addtion to + * deregistering itself). Calls to non-abrupt shutdown() preface + * this by checking canStop before triggering the "STOP" phase of * termination. Like most eveything else, we try to speed up * termination by balancing parallelism and contention (which may * occur while trying to cancel tasks or unblock workers), by * starting task scans at different indices, and by fanning out * reactivation of idle workers. The main remaining cost is the - * potential need to re-interrupt active workers with tasks that - * swallow them. + * potential need to re-interrupt active workers. * * Trimming workers. To release resources after periods of lack of * use, a worker starting to wait when the pool is quiescent will @@ -727,7 +726,7 @@ public class ForkJoinPool extends AbstractExecutorService { * (ForkJoinTask.cancel) independently from the interrupt status * of threads running tasks. (See the public ForkJoinTask * documentation for rationale.) Interrupts are issued internally - * only in helpTerminate, when workers should be terminating and + * only in tryTerminate, when workers should be terminating and * tasks should be cancelled anyway. By default, interrupts are * cleared only when necessary to ensure that calls to * LockSupport.park do not loop indefinitely (park returns @@ -1093,8 +1092,8 @@ static boolean casSlotToNull(ForkJoinTask[] a, int i, return U.compareAndSetReference(a, ((long)i << ASHIFT) + ABASE, c, null); } - final void forcePhaseActive() { // clear sign bit - U.getAndBitwiseAndInt(this, PHASE, 0x7fffffff); + final int forcePhaseActive() { // clear sign bit + return U.getAndBitwiseAndInt(this, PHASE, 0x7fffffff); } final int getAndSetAccess(int v) { return U.getAndSetInt(this, ACCESS, v); @@ -1453,11 +1452,6 @@ else if (a[nk] == null) // misc - final void cancelAllTasks() { - for (ForkJoinTask t; (t = poll(null)) != null; ) - ForkJoinTask.cancelIgnoringExceptions(t); - } - /** * Returns true if owned by a worker thread and not known to be blocked. */ @@ -1669,12 +1663,8 @@ final void registerWorker(WorkQueue w) { * @param ex the exception causing failure, or null if none */ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { - int cfg = 0; WorkQueue w = (wt == null) ? null : wt.workQueue; - if (w != null) { - cfg = w.config; - w.access = STOP; // may be redundant - } + int cfg = (w == null) ? 0 : w.config; long c = ctl; if ((cfg & TRIMMED) == 0) // decrement counts do {} while (c != (c = compareAndExchangeCtl( @@ -1694,12 +1684,17 @@ else if ((int)c == 0) // was dropped on timeout stealCount += ns; // accumulate steals lock.unlock(); } - w.cancelAllTasks(); if ((cfg & SRC) != 0) signalWork(); // possibly replace worker } - if (ex != null) + if (ex != null) { + if (w != null) { + w.access = STOP; // cancel tasks + for (ForkJoinTask t; (t = w.nextLocalTask(0)) != null; ) + ForkJoinTask.cancelIgnoringExceptions(t); + } ForkJoinTask.rethrow(ex); + } } /* @@ -1824,6 +1819,7 @@ final void runWorker(WorkQueue w) { r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift } while ((src = scan(w, src, r)) >= 0 || (src = awaitWork(w)) == 0); + w.access = STOP; // record normal termination } } @@ -1839,9 +1835,8 @@ final void runWorker(WorkQueue w) { * @return the next prevSrc value to use, or negative if none found */ private int scan(WorkQueue w, int prevSrc, int r) { - int rs = runState; WorkQueue[] qs = queues; - int n = (rs < 0 || qs == null || w == null) ? 0 : qs.length; + int n = (w == null || qs == null) ? 0 : qs.length; for (int step = (r >>> 16) | 1, i = n; i > 0; --i, r += step) { int j, cap; WorkQueue q; ForkJoinTask[] a; if ((q = qs[j = r & (n - 1)]) != null && @@ -1877,7 +1872,6 @@ private int awaitWork(WorkQueue w) { if (w == null) return -1; // currently impossible int p = (w.phase + SS_SEQ) & ~INACTIVE; // advance phase - boolean idle = false; // true if possibly quiescent if (runState < 0) return -1; // terminating long sp = p & SP_MASK, pc = ctl, qc; @@ -1886,13 +1880,11 @@ private int awaitWork(WorkQueue w) { w.stackPred = (int)pc; // set ctl stack link } while (pc != (pc = compareAndExchangeCtl( pc, qc = ((pc - RC_UNIT) & UC_MASK) | sp))); - if ((qc & RC_MASK) <= 0L) { - if (hasTasks(true) && (w.phase >= 0 || reactivate() == w)) - return 0; // check for stragglers - if (runState != 0 && tryTerminate(false, false) < 0) - return -1; // quiescent termination - idle = true; - } + boolean idle; + if ((idle = ((qc & RC_MASK) <= 0L)) && // possibly quiescent + (hasTasks(true) || (runState != 0 && canStop(true))) && + w.phase < 0) + reactivate(); // enable rescan or termination WorkQueue[] qs = queues; // spin for expected #accesses in scan+signal int spins = ((qs == null) ? 0 : ((qs.length & SMASK) << 1)) | 0xf; while ((p = w.phase) < 0 && --spins > 0) @@ -1924,21 +1916,25 @@ private int awaitWork(WorkQueue w) { } } } - return 0; + return runState & STOP; // negative if terminating } /** * Non-overridable version of isQuiescent. Returns true if * quiescent or already terminating. */ - private boolean canStop() { - long c = ctl; - do { + private boolean canStop(boolean enableTermination) { + for (long c = ctl;;) { if (runState < 0) break; if ((c & RC_MASK) > 0L || hasTasks(false)) return false; - } while (c != (c = ctl)); // validate + if (c == (c = ctl)) { // validate + if (enableTermination) // transition now + getAndBitwiseOrRunState(STOP); + break; + } + } return true; } @@ -2276,7 +2272,7 @@ private int externalHelpQuiesce(long nanos, boolean interruptible) { t.doExec(); parkTime = 0L; } - else if (canStop()) + else if (canStop(false)) return 1; else if (parkTime == 0L) { parkTime = 1L << 10; @@ -2493,7 +2489,10 @@ static int getSurplusQueuedTaskCount() { // Termination /** - * Possibly initiates and/or completes pool termination. + * Possibly initiates and/or completes pool termination. If + * terminating, helps complete termination by cancelling tasks, + * reactivating idle workers or interrupting active workers, and + * possibly triggering TERMINATED state. * * @param now if true, unconditionally terminate, else only * if no work and no active workers @@ -2502,71 +2501,50 @@ static int getSurplusQueuedTaskCount() { */ private int tryTerminate(boolean now, boolean enable) { int rs = runState; - if ((config & ISCOMMON) == 0) { - if (rs >= 0) { - if (!now && enable && (rs & SHUTDOWN) == 0) - rs = getAndBitwiseOrRunState(SHUTDOWN) | SHUTDOWN; - if (now || ((rs & SHUTDOWN) != 0 && canStop())) - rs = getAndBitwiseOrRunState(SHUTDOWN | STOP) | STOP; - } - if ((rs < 0 || (rs = runState) < 0) && (rs & TERMINATED) == 0) { - helpTerminate(); + if ((config & ISCOMMON) == 0 && rs >= 0) { + if (!now && enable && (rs & SHUTDOWN) == 0) + rs = getAndBitwiseOrRunState(SHUTDOWN) | SHUTDOWN; + if (now || ((rs & SHUTDOWN) != 0 && canStop(false))) + rs = getAndBitwiseOrRunState(SHUTDOWN | STOP) | STOP; + else rs = runState; - } } - return rs; - } - - /** - * Helps complete termination by cancelling tasks, unblocking - * workers and possibly triggering TERMINATED state. - */ - private void helpTerminate() { - int r = 1; // for queue traversal + if ((rs & (STOP | TERMINATED)) != STOP) + return rs; // not terminating + WorkQueue[] qs; WorkQueue q; Thread thread; WorkQueue w; + ReentrantLock lock; Condition cond; + boolean more = (reactivate() != null); // try activating idle worker Thread current = Thread.currentThread(); - if (current instanceof ForkJoinWorkerThread) { - ForkJoinWorkerThread wt = (ForkJoinWorkerThread)current; - WorkQueue w = wt.workQueue; - if (wt.pool == this && w != null) { - r = w.config; // stagger traversals - w.access = STOP; // may be redundant - } - } - WorkQueue[] qs; WorkQueue q; Thread thread; + int r = (((current instanceof ForkJoinWorkerThread) && + (w = ((ForkJoinWorkerThread)current).workQueue) != null) ? + w.config : 0); // stagger traversals int n = ((qs = queues) == null) ? 0 : qs.length; - for (int i = 0; i < n; ++i) { // help cancel tasks - if ((q = qs[(r + i) & (n - 1)]) != null) - q.cancelAllTasks(); - } - if (reactivate() != null) // activate <= 2 idle workers - reactivate(); - else if ((runState & TERMINATED) == 0) { // or unblock active workers - boolean canTerminate = true; - n = ((qs = queues) == null) ? 0 : qs.length; - for (int i = 0; i < n; i += 2) { - if ((q = qs[(r + i) & (n - 1)]) != null && - (thread = q.owner) != null && q.access != STOP) { - canTerminate = false; - q.forcePhaseActive(); // for awaitWork - if (!thread.isInterrupted()) { - try { - thread.interrupt(); - } catch (Throwable ignore) { - } + for (int i = 0; i < n; ++i) { + if ((q = qs[(r + i) & (n - 1)]) != null) { + for (ForkJoinTask t; (t = q.poll(null)) != null; ) + ForkJoinTask.cancelIgnoringExceptions(t); + if (!more && q.access != STOP && + (thread = q.owner) != null && thread != current && + (q.forcePhaseActive() < 0 || !thread.isInterrupted())) { + try { + thread.interrupt(); // help unblock others + } catch (Throwable ignore) { } } } - ReentrantLock lock; Condition cond; // signal when no workers - if (canTerminate && (short)(ctl >>> TC_SHIFT) <= 0 && - (getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0 && - (lock = registrationLock) != null) { - lock.lock(); - if ((cond = termination) != null) - cond.signalAll(); - lock.unlock(); - container.close(); - } } + if (reactivate() == null && // transition if no workers + (short)(ctl >>> TC_SHIFT) <= 0 && + (runState & TERMINATED) == 0 && + (getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0 && + (lock = registrationLock) != null) { + lock.lock(); + if ((cond = termination) != null) + cond.signalAll(); + lock.unlock(); + container.close(); + } + return runState; } // Exported methods @@ -3025,12 +3003,13 @@ public List> invokeAll(Collection> tasks) { futures.add(f); poolSubmit(true, f); } - for (int i = futures.size() - 1; i >= 0 && runState >= 0; --i) + for (int i = futures.size() - 1; i >= 0; --i) ((ForkJoinTask)futures.get(i)).quietlyJoin(); return futures; - } finally { + } catch (Throwable t) { for (Future e : futures) ForkJoinTask.cancelIgnoringExceptions(e); + throw t; } } @@ -3049,7 +3028,7 @@ public List> invokeAll(Collection> tasks, } long startTime = System.nanoTime(), ns = nanos; boolean timedOut = (ns < 0L); - for (int i = futures.size() - 1; i >= 0 && runState >= 0; --i) { + for (int i = futures.size() - 1; i >= 0; --i) { ForkJoinTask f = (ForkJoinTask)futures.get(i); if (!f.isDone()) { if (!timedOut) @@ -3061,9 +3040,10 @@ public List> invokeAll(Collection> tasks, } } return futures; - } finally { + } catch (Throwable t) { for (Future e : futures) ForkJoinTask.cancelIgnoringExceptions(e); + throw t; } } @@ -3095,8 +3075,12 @@ final void tryComplete(E v, Throwable ex, boolean completed) { result = v; quietlyComplete(); } - else if (count.getAndDecrement() <= 1) - trySetThrown(ex != null ? ex : new CancellationException()); + else if (count.getAndDecrement() <= 1) { + if (ex == null) + cancel(false); + else + trySetThrown(ex); + } } } public final boolean exec() { return false; } // never forked @@ -3180,11 +3164,7 @@ public T invokeAny(Collection> tasks) if (root.isDone()) break; } - try { - return root.get(); - } catch (CancellationException cx) { - throw new ExecutionException(cx); - } + return root.get(); } finally { for (InvokeAnyTask f : fs) ForkJoinTask.cancelIgnoringExceptions(f); @@ -3211,11 +3191,7 @@ public T invokeAny(Collection> tasks, if (root.isDone()) break; } - try { - return root.get(nanos, TimeUnit.NANOSECONDS); - } catch (CancellationException cx) { - throw new ExecutionException(cx); - } + return root.get(nanos, TimeUnit.NANOSECONDS); } finally { for (InvokeAnyTask f : fs) ForkJoinTask.cancelIgnoringExceptions(f); @@ -3325,7 +3301,7 @@ public int getActiveThreadCount() { * @return {@code true} if all threads are currently idle */ public boolean isQuiescent() { - return canStop(); + return canStop(false); } /** @@ -3540,7 +3516,7 @@ public List shutdownNow() { * @return {@code true} if all tasks have completed following shut down */ public boolean isTerminated() { - // reduce false negatives during termination by helping + // reduce premature negatives during termination by helping return (tryTerminate(false, false) & TERMINATED) != 0; } diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index 1897572ed4124..a357816109799 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -448,19 +448,20 @@ else if (timed && (ns = deadline - System.nanoTime()) <= 0) { s = 0; break; } - else if (Thread.interrupted()) { - interrupted = true; - if ((how & INTERRUPTIBLE) != 0) { + else { + if (Thread.interrupted()) + interrupted = true; + if ((s = status) < 0) // prefer result to IE + break; + else if (interrupted && (how & INTERRUPTIBLE) != 0) { s = ABNORMAL; break; } + else if (timed) + LockSupport.parkNanos(ns); + else + LockSupport.park(); } - else if ((s = status) < 0) // recheck - break; - else if (timed) - LockSupport.parkNanos(ns); - else - LockSupport.park(); } if (uncompensate) p.uncompensate(); @@ -574,13 +575,17 @@ private void reportException(int s) { * necessary in an ExecutionException. */ private void reportExecutionException(int s) { - Throwable ex = null, rx; + Throwable ex, rx; if (s == ABNORMAL) ex = new InterruptedException(); else if (s >= 0) ex = new TimeoutException(); else if ((rx = getThrowableException()) != null) ex = new ExecutionException(rx); + else if ((status & POOLSUBMIT) != 0) + ex = new ExecutionException(new CancellationException()); + else + ex = new CancellationException(); ForkJoinTask.uncheckedThrow(ex); } @@ -1504,17 +1509,15 @@ else if (!isDone()) } public final void run() { invoke(); } public final boolean cancel(boolean mayInterruptIfRunning) { - int s; Thread t; - if ((s = trySetCancelled()) >= 0) { - if (mayInterruptIfRunning && (t = runner) != null) { - try { - t.interrupt(); - } catch (Throwable ignore) { - } + Thread t; + boolean stat = super.cancel(false); + if (mayInterruptIfRunning && (t = runner) != null) { + try { + t.interrupt(); + } catch (Throwable ignore) { } - return true; } - return ((s & (ABNORMAL | THROWN)) == ABNORMAL); + return stat; } public String toString() { return super.toString() + "[Wrapped task = " + callable + "]"; From 6c7690805bb766fc7f72bb7f2a4df0ca3267cdd5 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sat, 7 Jan 2023 09:31:38 -0500 Subject: [PATCH 05/61] More exception compatibility --- .../java/util/concurrent/ForkJoinPool.java | 121 ++++++++++-------- .../java/util/concurrent/ForkJoinTask.java | 21 +-- 2 files changed, 84 insertions(+), 58 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index a343af9c6203c..347a6ab566751 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -581,18 +581,22 @@ public class ForkJoinPool extends AbstractExecutorService { * occasionally adds some extra unnecessary processing. * * Shutdown and Termination. A call to shutdownNow invokes - * tryTerminate to atomically set a mode bit. The calling thread, - * as well as other workers thereafter terminating, and external - * callers checking termination helps cancel queued tasks, - * reactivate idle workers, and interrupt others (in addtion to - * deregistering itself). Calls to non-abrupt shutdown() preface - * this by checking canStop before triggering the "STOP" phase of - * termination. Like most eveything else, we try to speed up - * termination by balancing parallelism and contention (which may - * occur while trying to cancel tasks or unblock workers), by - * starting task scans at different indices, and by fanning out - * reactivation of idle workers. The main remaining cost is the - * potential need to re-interrupt active workers. + * tryTerminate to atomically set a mode bit. However, termination + * is intrinsically non-atomic. The calling thread, as well as + * other workers thereafter terminating, and external callers + * checking termination help cancel queued tasks, reactivate idle + * workers, and interrupt others. To support this, worker + * references not removed from the queues array during + * termination. It is possible for late thread creations to still + * be in progress after a quiescent termination reports terminated + * status, but they will also immediately terminate. Calls to + * non-abrupt shutdown() preface this by checking canStop before + * triggering the "STOP" phase of termination. Like most eveything + * else, we try to speed up termination by balancing parallelism + * and contention, by starting task scans at different indices, + * and by fanning out reactivation of idle workers. The main + * remaining cost is the potential need to re-interrupt active + * workers (checks for this are racy but suffice here). * * Trimming workers. To release resources after periods of lack of * use, a worker starting to wait when the pool is quiescent will @@ -733,26 +737,26 @@ public class ForkJoinPool extends AbstractExecutorService { * immediately if the current thread is interrupted). * * To conform to specs and expectations surrounding execution, - * cancellation and termination of ExecutorService invoke, submit, - * invokeAll, and invokeAny methods (without penalizing others) we - * use AdaptedInterruptibleCallables in ExecutorService methods - * accepting Callables (actually, for InvokeAny, a variant that - * gathers a single result). We also ensure that external + * cancellation and termination of Callable-based ExecutorService + * invoke, submit, invokeAll, and invokeAny methods (without + * penalizing others) we use AdaptedInterruptibleCallables in + * ExecutorService methods (actually, for InvokeAny, a variant + * that gathers a single result). We also ensure that external * submitters do not help run such tasks by recording * ForkJoinTask.POOLSUBMIT in task status and/or as a bit flag * argument to joining methods. External callers of task.get etc * are not directly interrupted on shutdown, but must be woken due * to the task itself being cancelled during termination. - * Interruptible tasks always clear interrupts and check for - * termination and cancellation before invoking user-supplied - * Callables These rechecks are needed because the cleared - * interrupt could be due to cancellation, termination, or any - * other reason. These tasks also record their runner thread so - * that cancel(true) can interrupt them later. As is the case with - * any kind of pool, it is possible that an interrupt designed to - * cancel one task occurs both late and unnecessarily, so instead - * "spuriously" interrupts the thread while performing a later - * task. + * AdaptedInterruptibleCallables tasks always clear interrupts and + * check for termination and cancellation before invoking + * user-supplied Callables. The rechecks are needed because the + * cleared interrupt could be due to cancellation, termination, or + * any other reason. These tasks also record their runner thread + * so that cancel(true) can interrupt them later. As is the case + * with any kind of pool, it is possible that an interrupt + * designed to cancel one task occurs both late and unnecessarily, + * so instead "spuriously" interrupts the thread while performing + * a later task. * * Memory placement * ================ @@ -1663,8 +1667,12 @@ final void registerWorker(WorkQueue w) { * @param ex the exception causing failure, or null if none */ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { + int cfg = 0; WorkQueue w = (wt == null) ? null : wt.workQueue; - int cfg = (w == null) ? 0 : w.config; + if (w != null) { + cfg = w.config; + w.access = STOP; // may be redundant + } long c = ctl; if ((cfg & TRIMMED) == 0) // decrement counts do {} while (c != (c = compareAndExchangeCtl( @@ -1688,8 +1696,7 @@ else if ((int)c == 0) // was dropped on timeout signalWork(); // possibly replace worker } if (ex != null) { - if (w != null) { - w.access = STOP; // cancel tasks + if (w != null) { // cancel tasks for (ForkJoinTask t; (t = w.nextLocalTask(0)) != null; ) ForkJoinTask.cancelIgnoringExceptions(t); } @@ -1819,7 +1826,6 @@ final void runWorker(WorkQueue w) { r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift } while ((src = scan(w, src, r)) >= 0 || (src = awaitWork(w)) == 0); - w.access = STOP; // record normal termination } } @@ -1882,7 +1888,7 @@ private int awaitWork(WorkQueue w) { pc, qc = ((pc - RC_UNIT) & UC_MASK) | sp))); boolean idle; if ((idle = ((qc & RC_MASK) <= 0L)) && // possibly quiescent - (hasTasks(true) || (runState != 0 && canStop(true))) && + (hasTasks(true) || (runState != 0 && canStop(true) < 0)) && w.phase < 0) reactivate(); // enable rescan or termination WorkQueue[] qs = queues; // spin for expected #accesses in scan+signal @@ -1916,26 +1922,27 @@ private int awaitWork(WorkQueue w) { } } } - return runState & STOP; // negative if terminating + return runState & STOP; // 0 unless terminating } /** - * Non-overridable version of isQuiescent. Returns true if - * quiescent or already terminating. + * Non-overridable version of isQuiescent + * @param enableTransition true if should set runState if can stop + * @return negative if quiescent or already terminating, else runState */ - private boolean canStop(boolean enableTermination) { + private int canStop(boolean enableTransition) { + int rs; for (long c = ctl;;) { - if (runState < 0) + if ((rs = runState) < 0 || ((c & RC_MASK) > 0L || hasTasks(false))) break; - if ((c & RC_MASK) > 0L || hasTasks(false)) - return false; if (c == (c = ctl)) { // validate - if (enableTermination) // transition now + if (enableTransition) getAndBitwiseOrRunState(STOP); + rs |= STOP; break; } } - return true; + return rs; } /** @@ -2272,7 +2279,7 @@ private int externalHelpQuiesce(long nanos, boolean interruptible) { t.doExec(); parkTime = 0L; } - else if (canStop(false)) + else if (canStop(false) < 0) return 1; else if (parkTime == 0L) { parkTime = 1L << 10; @@ -2502,12 +2509,14 @@ static int getSurplusQueuedTaskCount() { private int tryTerminate(boolean now, boolean enable) { int rs = runState; if ((config & ISCOMMON) == 0 && rs >= 0) { - if (!now && enable && (rs & SHUTDOWN) == 0) - rs = getAndBitwiseOrRunState(SHUTDOWN) | SHUTDOWN; - if (now || ((rs & SHUTDOWN) != 0 && canStop(false))) - rs = getAndBitwiseOrRunState(SHUTDOWN | STOP) | STOP; + if (!now) { + if (enable && (rs & SHUTDOWN) == 0) + rs = getAndBitwiseOrRunState(SHUTDOWN) | SHUTDOWN; + if ((rs & SHUTDOWN) != 0) + rs = canStop(true); + } else - rs = runState; + rs = getAndBitwiseOrRunState(SHUTDOWN | STOP) | STOP; } if ((rs & (STOP | TERMINATED)) != STOP) return rs; // not terminating @@ -3050,6 +3059,10 @@ public List> invokeAll(Collection> tasks, /** * Task to hold results for invokeAny, or to report exception if * all subtasks fail or are cancelled or the pool is terminating. + * Note: Among other oddities, the ExecutorService.invokeAny spec + * requires that the all-cancelled case (including cancellation of + * the root because of pool termination) be reported as an + * ExecutionException, not a CancellationException. */ static final class InvokeAnyRoot extends ForkJoinTask { private static final long serialVersionUID = 2838392045355241008L; @@ -3164,7 +3177,11 @@ public T invokeAny(Collection> tasks) if (root.isDone()) break; } - return root.get(); + try { + return root.get(); + } catch (CancellationException cx) { + throw new ExecutionException(cx); + } } finally { for (InvokeAnyTask f : fs) ForkJoinTask.cancelIgnoringExceptions(f); @@ -3191,7 +3208,11 @@ public T invokeAny(Collection> tasks, if (root.isDone()) break; } - return root.get(nanos, TimeUnit.NANOSECONDS); + try { + return root.get(nanos, TimeUnit.NANOSECONDS); + } catch (CancellationException cx) { + throw new ExecutionException(cx); + } } finally { for (InvokeAnyTask f : fs) ForkJoinTask.cancelIgnoringExceptions(f); @@ -3301,7 +3322,7 @@ public int getActiveThreadCount() { * @return {@code true} if all threads are currently idle */ public boolean isQuiescent() { - return canStop(false); + return canStop(false) < 0; } /** diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index a357816109799..8c2c9ae027f77 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -575,17 +575,13 @@ private void reportException(int s) { * necessary in an ExecutionException. */ private void reportExecutionException(int s) { - Throwable ex, rx; + Throwable ex = null, rx; if (s == ABNORMAL) ex = new InterruptedException(); else if (s >= 0) ex = new TimeoutException(); else if ((rx = getThrowableException()) != null) ex = new ExecutionException(rx); - else if ((status & POOLSUBMIT) != 0) - ex = new ExecutionException(new CancellationException()); - else - ex = new CancellationException(); ForkJoinTask.uncheckedThrow(ex); } @@ -1476,6 +1472,15 @@ public String toString() { private static final long serialVersionUID = 2838392045355241008L; } + /** + * Adapter for Callable-based tasks that are designed to be + * interruptible when cancelled, including cases of cancellation + * upon pool termination. In addition to recording the running + * thread to enable interrupt in cancel(true), the task checks for + * termination before executing the compute method, to cover + * shutdown races in which the task has not yet been cancelled on + * entry but might not otherwise be re-interrupted by others. + */ static final class AdaptedInterruptibleCallable extends ForkJoinTask implements RunnableFuture { @SuppressWarnings("serial") // Conditionally serializable @@ -1490,12 +1495,12 @@ static final class AdaptedInterruptibleCallable extends ForkJoinTask public final T getRawResult() { return result; } public final void setRawResult(T v) { result = v; } public final boolean exec() { - ForkJoinPool p; // for termination check Thread.interrupted(); runner = Thread.currentThread(); + ForkJoinPool p = getPool(); try { - if ((p = getPool()) != null && p.runState < 0) - trySetCancelled(); + if (p != null && p.runState < 0) + trySetCancelled(); // termination check else if (!isDone()) result = callable.call(); return true; From 6d1578e91e77f59652891d686e3d019bbaeff687 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sun, 8 Jan 2023 09:37:58 -0500 Subject: [PATCH 06/61] Fix more exception policy mismatches --- .../java/util/concurrent/ForkJoinPool.java | 31 +++++++------ .../java/util/concurrent/ForkJoinTask.java | 43 ++++++++++--------- 2 files changed, 38 insertions(+), 36 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 347a6ab566751..854bfd18cd388 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -3073,15 +3073,6 @@ static final class InvokeAnyRoot extends ForkJoinTask { InvokeAnyRoot(int n) { count = new AtomicInteger(n); } - final boolean checkDone() { - ForkJoinPool p; // force done if caller's pool terminating - if (!isDone()) { - if ((p = getPool()) == null || p.runState >= 0) - return false; - cancel(false); - } - return true; - } final void tryComplete(E v, Throwable ex, boolean completed) { if (!isDone()) { if (completed) { @@ -3090,7 +3081,7 @@ final void tryComplete(E v, Throwable ex, boolean completed) { } else if (count.getAndDecrement() <= 1) { if (ex == null) - cancel(false); + trySetCancelled(); else trySetThrown(ex); } @@ -3118,21 +3109,29 @@ static final class InvokeAnyTask extends ForkJoinTask { this.callable = callable; } public final boolean exec() { + ForkJoinPool p; + Thread t = Thread.currentThread(); Thread.interrupted(); - runner = Thread.currentThread(); + runner = t; InvokeAnyRoot r = root; Callable c = callable; E v = null; Throwable ex = null; boolean completed = false; - if (r != null && !r.checkDone() && !isDone()) { - try { - if (c != null) { + if (r != null && c != null) { + if ((t instanceof ForkJoinWorkerThread) && + (p = ((ForkJoinWorkerThread) t).pool) != null && + p.runState < 0) { // termination check + r.trySetCancelled(); + t.interrupt(); // restore interrupt + } + else if (!r.isDone() && !isDone()) { + try { v = c.call(); completed = true; + } catch (Throwable rex) { + ex = rex; } - } catch (Throwable rex) { - ex = rex; } if (trySetCancelled() >= 0) r.tryComplete(v, ex, completed); diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index 8c2c9ae027f77..3948068b39d28 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -413,9 +413,7 @@ private int awaitDone(int how, long deadline) { q = (wt = (ForkJoinWorkerThread)t).workQueue; p = wt.pool; } - else if ((how & POOLSUBMIT) != 0) - p = null; - else if ((p = ForkJoinPool.common) != null) + else if ((p = ForkJoinPool.common) != null && (how & POOLSUBMIT) == 0) q = p.externalQueue(); if (q != null && p != null) { // try helping if (this instanceof CountedCompleter) @@ -435,8 +433,6 @@ else if ((how & RAN) != 0 || Aux a; if ((s = status) < 0) break; - else if (p != null && p.runState < 0) - cancelIgnoringExceptions(this); // cancel on shutdown else if (node == null) node = new Aux(Thread.currentThread(), null); else if (!queued) { @@ -448,20 +444,21 @@ else if (timed && (ns = deadline - System.nanoTime()) <= 0) { s = 0; break; } - else { - if (Thread.interrupted()) - interrupted = true; - if ((s = status) < 0) // prefer result to IE - break; - else if (interrupted && (how & INTERRUPTIBLE) != 0) { + else if (Thread.interrupted()) { + interrupted = true; + if ((how & POOLSUBMIT) != 0 && p != null && p.runState < 0) + cancelIgnoringExceptions(this); // cancel on shutdown + else if ((how & INTERRUPTIBLE) != 0) { s = ABNORMAL; break; } - else if (timed) - LockSupport.parkNanos(ns); - else - LockSupport.park(); } + else if ((s = status) < 0) // recheck + break; + else if (timed) + LockSupport.parkNanos(ns); + else + LockSupport.park(); } if (uncompensate) p.uncompensate(); @@ -485,8 +482,11 @@ else if (casAux(a, next)) } } } + int stat = status; // prefer completion result + if (stat < 0) + s = stat; } - else { + if (s < 0) { signalWaiters(); // help clean or signal if (interrupted) Thread.currentThread().interrupt(); @@ -1495,12 +1495,15 @@ static final class AdaptedInterruptibleCallable extends ForkJoinTask public final T getRawResult() { return result; } public final void setRawResult(T v) { result = v; } public final boolean exec() { + ForkJoinPool p; + Thread t = Thread.currentThread(); Thread.interrupted(); - runner = Thread.currentThread(); - ForkJoinPool p = getPool(); + runner = t; try { - if (p != null && p.runState < 0) - trySetCancelled(); // termination check + if ((t instanceof ForkJoinWorkerThread) && + (p = ((ForkJoinWorkerThread) t).pool) != null && + p.runState < 0) // termination check + cancelIgnoringExceptions(this); else if (!isDone()) result = callable.call(); return true; From c09461f2e7a7f2841cbcfc016a320075dd5a6536 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Wed, 11 Jan 2023 17:14:30 -0500 Subject: [PATCH 07/61] More exception policy fixes --- .../java/util/concurrent/ForkJoinPool.java | 91 ++++++++----------- .../java/util/concurrent/ForkJoinTask.java | 13 ++- 2 files changed, 49 insertions(+), 55 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 854bfd18cd388..7beff8f682d40 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1824,8 +1824,8 @@ final void runWorker(WorkQueue w) { int r = w.stackPred, src = 0; // use seed from registerWorker do { r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift - } while ((src = scan(w, src, r)) >= 0 || - (src = awaitWork(w)) == 0); + } while (runState >= 0 && ((src = scan(w, src, r)) >= 0 || + (src = awaitWork(w)) == 0)); } } @@ -1922,7 +1922,7 @@ private int awaitWork(WorkQueue w) { } } } - return runState & STOP; // 0 unless terminating + return 0; } /** @@ -3071,6 +3071,8 @@ static final class InvokeAnyRoot extends ForkJoinTask { final AtomicInteger count; // in case all fail @SuppressWarnings("serial") InvokeAnyRoot(int n) { + if (n <= 0) + throw new IllegalArgumentException(); count = new AtomicInteger(n); } final void tryComplete(E v, Throwable ex, boolean completed) { @@ -3090,6 +3092,32 @@ else if (count.getAndDecrement() <= 1) { public final boolean exec() { return false; } // never forked public final E getRawResult() { return result; } public final void setRawResult(E v) { result = v; } + // Common support for timed and untimed versions of invokeAny + final E invokeAny(Collection> tasks, + ForkJoinPool pool, boolean timed, long nanos) + throws InterruptedException, ExecutionException, TimeoutException { + Thread t = Thread.currentThread(); + if (!(t instanceof ForkJoinWorkerThread) || + ((ForkJoinWorkerThread)t).pool != pool) + markPoolSubmission(); // for exception reporting + ArrayList> fs = new ArrayList<>(count.get()); + try { + for (Callable c : tasks) { + InvokeAnyTask f = new InvokeAnyTask(this, c); + fs.add(f); + pool.poolSubmit(true, f); + if (isDone()) + break; + } + if (timed) + return get(nanos, TimeUnit.NANOSECONDS); + else + return get(); + } finally { + for (InvokeAnyTask f : fs) + ForkJoinTask.cancelIgnoringExceptions(f); + } + } } /** @@ -3105,8 +3133,9 @@ static final class InvokeAnyTask extends ForkJoinTask { final Callable callable; transient volatile Thread runner; InvokeAnyTask(InvokeAnyRoot root, Callable callable) { - this.root = root; + if (callable == null) throw new NullPointerException(); this.callable = callable; + this.root = root; } public final boolean exec() { ForkJoinPool p; @@ -3161,29 +3190,12 @@ public final void setRawResult(E v) {} // unused @Override public T invokeAny(Collection> tasks) throws InterruptedException, ExecutionException { - int n = tasks.size(); - if (n <= 0) - throw new IllegalArgumentException(); - InvokeAnyRoot root = new InvokeAnyRoot(n); - ArrayList> fs = new ArrayList<>(n); try { - for (Callable c : tasks) { - if (c == null) - throw new NullPointerException(); - InvokeAnyTask f = new InvokeAnyTask(root, c); - fs.add(f); - poolSubmit(true, f); - if (root.isDone()) - break; - } - try { - return root.get(); - } catch (CancellationException cx) { - throw new ExecutionException(cx); - } - } finally { - for (InvokeAnyTask f : fs) - ForkJoinTask.cancelIgnoringExceptions(f); + return new InvokeAnyRoot(tasks.size()). + invokeAny(tasks, this, false, 0L); + } catch (TimeoutException cannotHappen) { + assert false; + return null; } } @@ -3191,31 +3203,8 @@ public T invokeAny(Collection> tasks) public T invokeAny(Collection> tasks, long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { - long nanos = unit.toNanos(timeout); - int n = tasks.size(); - if (n <= 0) - throw new IllegalArgumentException(); - InvokeAnyRoot root = new InvokeAnyRoot(n); - ArrayList> fs = new ArrayList<>(n); - try { - for (Callable c : tasks) { - if (c == null) - throw new NullPointerException(); - InvokeAnyTask f = new InvokeAnyTask(root, c); - fs.add(f); - poolSubmit(true, f); - if (root.isDone()) - break; - } - try { - return root.get(nanos, TimeUnit.NANOSECONDS); - } catch (CancellationException cx) { - throw new ExecutionException(cx); - } - } finally { - for (InvokeAnyTask f : fs) - ForkJoinTask.cancelIgnoringExceptions(f); - } + return new InvokeAnyRoot(tasks.size()). + invokeAny(tasks, this, true, unit.toNanos(timeout)); } /** diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index 3948068b39d28..421f8c402c6d5 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -446,9 +446,7 @@ else if (timed && (ns = deadline - System.nanoTime()) <= 0) { } else if (Thread.interrupted()) { interrupted = true; - if ((how & POOLSUBMIT) != 0 && p != null && p.runState < 0) - cancelIgnoringExceptions(this); // cancel on shutdown - else if ((how & INTERRUPTIBLE) != 0) { + if ((how & INTERRUPTIBLE) != 0) { s = ABNORMAL; break; } @@ -575,13 +573,20 @@ private void reportException(int s) { * necessary in an ExecutionException. */ private void reportExecutionException(int s) { - Throwable ex = null, rx; + Throwable ex, rx; if (s == ABNORMAL) ex = new InterruptedException(); else if (s >= 0) ex = new TimeoutException(); else if ((rx = getThrowableException()) != null) ex = new ExecutionException(rx); + else { + rx = new CancellationException(); + if ((status & POOLSUBMIT) != 0) // wrap if external + ex = new ExecutionException(rx); + else + ex = rx; + } ForkJoinTask.uncheckedThrow(ex); } From 72b617771348134f4a751e25ddeb6d2ee97f1832 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sat, 21 Jan 2023 12:56:00 -0500 Subject: [PATCH 08/61] Ensure consistency across quiescence and runState decisions --- .../java/util/concurrent/ForkJoinPool.java | 423 ++++++++++-------- .../java/util/concurrent/ForkJoinTask.java | 49 +- 2 files changed, 250 insertions(+), 222 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 7beff8f682d40..7a6a091ace52d 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -482,7 +482,8 @@ public class ForkJoinPool extends AbstractExecutorService { * available (and so is never checked in this way). When queued, * the lower 16 bits of its phase must hold its pool index. So we * place the index there upon initialization and never modify - * these bits. + * these bits, which also allows use as a versioned id in other + * contexts. * * The ctl field also serves as the basis for memory * synchronization surrounding activation. This uses a more @@ -566,37 +567,40 @@ public class ForkJoinPool extends AbstractExecutorService { * fields in ctl allow efficient and accurate discovery of * quiescent states (i.e., when all workers are idle) after * deactivation. However, this voting mechanism alone does not - * guarantee that a pool can become dormant (quiesced or - * terminated), because external racing producers do not vote, and - * can asynchronously submit new tasks. To deal with this, the - * final unparked thread (in awaitWork) scans external queues to + * guarantee that a pool can become dormant (or terminated), + * because external racing producers do not vote, and can + * asynchronously submit new tasks. To deal with this, the final + * unparked thread (in awaitWork) uses canStop (see below) to * check for tasks that could have been added during a race window * that would not be accompanied by a signal, in which case - * re-activating itself (or any other worker) to recheck. The same - * sets of checks are used in tryTerminate, to correctly trigger - * delayed termination (shutDown, followed by quiescence) in the - * presence of racing submissions. In all cases, the notion of the - * "final" unparked thread is an approximation, because new - * workers could be in the process of being constructed, which - * occasionally adds some extra unnecessary processing. - * - * Shutdown and Termination. A call to shutdownNow invokes - * tryTerminate to atomically set a mode bit. However, termination - * is intrinsically non-atomic. The calling thread, as well as - * other workers thereafter terminating, and external callers + * re-activating itself (or any other worker) to recheck. + * + * Termination. A call to shutdownNow invokes tryTerminate to + * atomically set a mode bit. However, termination is + * intrinsically non-atomic. The calling thread, as well as other + * workers thereafter terminating, and any external callers * checking termination help cancel queued tasks, reactivate idle - * workers, and interrupt others. To support this, worker - * references not removed from the queues array during - * termination. It is possible for late thread creations to still - * be in progress after a quiescent termination reports terminated - * status, but they will also immediately terminate. Calls to - * non-abrupt shutdown() preface this by checking canStop before - * triggering the "STOP" phase of termination. Like most eveything - * else, we try to speed up termination by balancing parallelism - * and contention, by starting task scans at different indices, - * and by fanning out reactivation of idle workers. The main - * remaining cost is the potential need to re-interrupt active - * workers (checks for this are racy but suffice here). + * workers, and interrupt others; repeating until no more are + * found by checking and marking ctl counts and phase and access + * fields. These actions race with non-terminated workers. + * Except when using Interruptible tasks (see below), there are no + * quarantees after an abrupt shutdown whether remaining tasks + * complete normally or exceptionally or are cancelled, and + * termination may fail if tasks repeatedly ignore both + * cancellation status and interrupts. + * + * Quiescent shutdown. Calls to non-abrupt shutdown() use method + * canStop to trigger the "STOP" phase of termination upon + * quiescence, which is intrinsically racy, and requires + * convergent sweeps through queues to detect whether all workers + * are idle and there are no submissions that they could poll if + * they were not idle. If not immediately triggered, whenever all + * workers become idle in awaitWork, canStop is rechecked. Method + * helpQuiesce also uses canStop to help determine quiescence. It + * differs in that it does not trigger termination (even if + * enabled) and cannot rely on ctl counts to determine that all + * workers are inactive (because any executing helpQuiesce are not + * included). * * Trimming workers. To release resources after periods of lack of * use, a worker starting to wait when the pool is quiescent will @@ -736,27 +740,27 @@ public class ForkJoinPool extends AbstractExecutorService { * LockSupport.park do not loop indefinitely (park returns * immediately if the current thread is interrupted). * - * To conform to specs and expectations surrounding execution, - * cancellation and termination of Callable-based ExecutorService - * invoke, submit, invokeAll, and invokeAny methods (without - * penalizing others) we use AdaptedInterruptibleCallables in - * ExecutorService methods (actually, for InvokeAny, a variant - * that gathers a single result). We also ensure that external - * submitters do not help run such tasks by recording - * ForkJoinTask.POOLSUBMIT in task status and/or as a bit flag - * argument to joining methods. External callers of task.get etc - * are not directly interrupted on shutdown, but must be woken due - * to the task itself being cancelled during termination. - * AdaptedInterruptibleCallables tasks always clear interrupts and - * check for termination and cancellation before invoking - * user-supplied Callables. The rechecks are needed because the - * cleared interrupt could be due to cancellation, termination, or - * any other reason. These tasks also record their runner thread - * so that cancel(true) can interrupt them later. As is the case - * with any kind of pool, it is possible that an interrupt - * designed to cancel one task occurs both late and unnecessarily, - * so instead "spuriously" interrupts the thread while performing - * a later task. + * However, to conform to specs and expectations surrounding + * execution, cancellation and termination of Callable-based + * ExecutorService invoke, submit, invokeAll, and invokeAny + * methods (without penalizing others) we use "Interruptible" + * tasks -- AdaptedInterruptibleCallables, and a variant + * InvokeAnyTask. We ensure that external submitters do not help + * run such tasks by recording ForkJoinTask.POOLSUBMIT in task + * status and/or as a bit flag argument to other methods. In + * addition to recording a "runner" field (similarly to + * FutureTask) to support cancel(true), we ensure that upon pool + * shutdown, either the task is cancelled by the runner or the + * runner is interrupted so it can cancel. Often both, but since + * external joining callers never run these tasks, they must await + * cancellation (or a result; reporting the correct result or + * exception for task.get etc depending on pool status). As is + * the case with any kind of pool, it is possible that an + * interrupt designed to cancel one task occurs both late and + * unnecessarily, so instead "spuriously" interrupts the worker + * thread while performing a later task. Users should check + * cancellation status upon interrupt, but usually cannot because + * their Callables are wrapped and so cannot access status. * * Memory placement * ================ @@ -848,15 +852,10 @@ public class ForkJoinPool extends AbstractExecutorService { * * The main sources of differences from previous version are: * - * * Use of Unsafe vs VarHandle, including re-instatement of some - * constructions from pre-VarHandle versions. - * * Reduced memory and signal contention, mainly by distinguishing - * failure cases. - * * Improved initialization, in part by preparing for possible - * removal of SecurityManager - * * Enable resizing (includes refactoring quiescence/termination) - * * Unification of most internal vs external operations; some made - * possible via use of WorkQueue.access, and POOLSUBMIT status in tasks. + * * Handling of Interruptible tasks consistent with + * ExecutorService specs, including uniform use of revised + * canStop for quiescence-related checks and corresponding + * revisions to termination checks */ // static configuration constants @@ -1074,7 +1073,7 @@ static final class WorkQueue { @jdk.internal.vm.annotation.Contended("w") volatile int access; // values 0, 1 (locked), PARKED, STOP @jdk.internal.vm.annotation.Contended("w") - volatile int phase; // versioned, negative if inactive + volatile int phase; // versioned, negative if inactive, 0 on exit @jdk.internal.vm.annotation.Contended("w") volatile int source; // source queue id in topLevelExec @jdk.internal.vm.annotation.Contended("w") @@ -1623,6 +1622,7 @@ final void registerWorker(WorkQueue w) { w.array = new ForkJoinTask[INITIAL_QUEUE_CAPACITY]; cfg |= w.config | SRC; w.stackPred = seed; + int seq = (seed ^ (seed >>> 16)) << 16; // initial phase seq int id = (seed << 1) | 1; // initial index guess lock.lock(); try { @@ -1632,7 +1632,8 @@ final void registerWorker(WorkQueue w) { for (; qs[id &= m] != null && k > 0; id -= 2, k -= 2); if (k == 0) id = n | 1; // resize below - w.phase = w.config = id | cfg; // now publishable + w.config = id | cfg; + w.phase = (id | seq) & ~INACTIVE; // now publishable if (id < n) qs[id] = w; @@ -1671,7 +1672,10 @@ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { WorkQueue w = (wt == null) ? null : wt.workQueue; if (w != null) { cfg = w.config; - w.access = STOP; // may be redundant + w.access = STOP; // mark as done; cancel tasks + w.phase = 0; + for (ForkJoinTask t; (t = w.nextLocalTask(0)) != null; ) + ForkJoinTask.cancelIgnoringExceptions(t); } long c = ctl; if ((cfg & TRIMMED) == 0) // decrement counts @@ -1695,13 +1699,8 @@ else if ((int)c == 0) // was dropped on timeout if ((cfg & SRC) != 0) signalWork(); // possibly replace worker } - if (ex != null) { - if (w != null) { // cancel tasks - for (ForkJoinTask t; (t = w.nextLocalTask(0)) != null; ) - ForkJoinTask.cancelIgnoringExceptions(t); - } + if (ex != null) ForkJoinTask.rethrow(ex); - } } /* @@ -1777,7 +1776,7 @@ private boolean tryTrim(WorkQueue w) { int pred = w.stackPred, cfg = w.config | TRIMMED; long c = ctl; int sp = (int)c & ~INACTIVE; - if ((sp & SMASK) == (cfg & SMASK) && + if ((sp & SMASK) == (cfg & SMASK) && runState >= 0 && compareAndSetCtl(c, ((pred & SP_MASK) | (UC_MASK & (c - TC_UNIT))))) { w.config = cfg; // add sentinel for deregisterWorker @@ -1789,27 +1788,51 @@ private boolean tryTrim(WorkQueue w) { } /** - * Returns true if any queue is detectably nonempty. Accurate - * only when workers are quiescent; else conservatively - * approximate. - * @param submissionsOnly if true, only check submission queues - */ - private boolean hasTasks(boolean submissionsOnly) { - int step = submissionsOnly ? 2 : 1; - for (int checkSum = 0;;) { // repeat until stable (normally twice) - U.loadFence(); - WorkQueue[] qs = queues; - int n = (qs == null) ? 0 : qs.length, sum = 0; - for (int i = 0; i < n; i += step) { - WorkQueue q; int s; + * Internal version of isQuiescent and related functionality. + * Returns true if terminating or all submission queues are empty + * and unlocked, and unless !all, all workers are idle. To obtain + * a reliable snapshot, scans at least twice, and continues in + * case of inconsistencies. + * + * @param all if false, only check submission queues + * @param stopIfEnabled true if should transition runState to STOP + * if quiescent, in which case returns false + */ + private boolean canStop(boolean all, boolean stopIfEnabled) { + long checkSum = 0L, sum = 0L; // for consistency checks + for (int scans = 2;;) { // min remaining scans + if (runState < 0) + return true; // terminating + else if (all && (ctl & RC_MASK) > 0L) + return false; // not idle + else if (scans > 0) + --scans; // checkSum not valid + else if (checkSum == sum) { + if (stopIfEnabled && (runState & SHUTDOWN) != 0) { + getAndBitwiseOrRunState(STOP); + return false; // caller cannot stop yet + } + return true; + } + checkSum = sum; + sum = 0L; + WorkQueue[] qs; WorkQueue q; + int n = ((qs = queues) == null) ? 0 : qs.length; + for (int i = 0; i < n; ++i) { if ((q = qs[i]) != null) { - if (q.access > 0 || (s = q.top) != q.base) - return true; - sum += (s << 16) + i + 1; + int x = q.access, p = q.phase, s = q.top, d = s - q.base; + if ((i & 1) != 0) { + if (all && (p | x) > 0) + return false; // active worker + } + else if (x != 0 || d > 0 || q.access != 0) + return false; // locked or nonempty + else if (d != 0 || q.top != s) + scans = 1; // inconsistent + else // hash to associate s with q + sum = (sum << 5) - sum + s + p; } } - if (checkSum == (checkSum = sum)) - return false; } } @@ -1878,6 +1901,7 @@ private int awaitWork(WorkQueue w) { if (w == null) return -1; // currently impossible int p = (w.phase + SS_SEQ) & ~INACTIVE; // advance phase + boolean idle; // true if possibly quiescent if (runState < 0) return -1; // terminating long sp = p & SP_MASK, pc = ctl, qc; @@ -1886,19 +1910,17 @@ private int awaitWork(WorkQueue w) { w.stackPred = (int)pc; // set ctl stack link } while (pc != (pc = compareAndExchangeCtl( pc, qc = ((pc - RC_UNIT) & UC_MASK) | sp))); - boolean idle; - if ((idle = ((qc & RC_MASK) <= 0L)) && // possibly quiescent - (hasTasks(true) || (runState != 0 && canStop(true) < 0)) && - w.phase < 0) - reactivate(); // enable rescan or termination + if ((idle = ((qc & RC_MASK) <= 0L)) && // check for stragglers + !canStop(true, true) && (w.phase >= 0 || reactivate() == w)) + return 0; // rescan or quiescent terminate WorkQueue[] qs = queues; // spin for expected #accesses in scan+signal int spins = ((qs == null) ? 0 : ((qs.length & SMASK) << 1)) | 0xf; while ((p = w.phase) < 0 && --spins > 0) Thread.onSpinWait(); - if (p < 0) { // await signal + if (p < 0) { long deadline = idle ? keepAlive + System.currentTimeMillis() : 0L; LockSupport.setCurrentBlocker(this); - for (;;) { + for (;;) { // await signal w.access = PARKED; // enable unpark if (w.phase < 0) { if (idle) @@ -1925,26 +1947,6 @@ private int awaitWork(WorkQueue w) { return 0; } - /** - * Non-overridable version of isQuiescent - * @param enableTransition true if should set runState if can stop - * @return negative if quiescent or already terminating, else runState - */ - private int canStop(boolean enableTransition) { - int rs; - for (long c = ctl;;) { - if ((rs = runState) < 0 || ((c & RC_MASK) > 0L || hasTasks(false))) - break; - if (c == (c = ctl)) { // validate - if (enableTransition) - getAndBitwiseOrRunState(STOP); - rs |= STOP; - break; - } - } - return rs; - } - /** * Scans for and returns a polled task, if available. Used only * for untracked polls. Begins scan at a random index to avoid @@ -2190,18 +2192,21 @@ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { if (w == null || (phase = w.phase) < 0) return 0; int activePhase = phase, inactivePhase = phase | INACTIVE; - int wsrc = w.source, r = 0; + int wsrc = w.source, r = 0, returnStatus = 1; for (boolean locals = true;;) { WorkQueue[] qs; WorkQueue q; - if (runState < 0) { // terminating - w.phase = activePhase; - return 1; + if (runState < 0) + break; // terminating + if (interruptible && Thread.interrupted()) { + returnStatus = -1; + break; } if (locals) { // run local tasks before (re)polling + locals = false; for (ForkJoinTask u; (u = w.nextLocalTask()) != null;) u.doExec(); } - boolean rescan = false, busy = locals = false, interrupted; + boolean rescan = false, busy = false; int n = ((qs = queues) == null) ? 0 : qs.length, m = n - 1; scan: for (int i = n, j; i > 0; --i, ++r) { if ((q = qs[j = m & r]) != null && q != w) { @@ -2210,17 +2215,17 @@ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { int b = q.base, cap; if (a == null || (cap = a.length) <= 0) break; - int k = (cap - 1) & b, nb = b + 1, nk = (cap - 1) & nb; + int k = (cap - 1) & b, nb = b + 1; ForkJoinTask t = a[k]; U.loadFence(); // for re-reads if (q.base != b || q.array != a || a[k] != t) ; else if (t == null) { if (!rescan) { - if (a[nk] != null || q.top - b > 0) + int p = q.phase, s = q.top; + if (s - b > 0) rescan = true; - else if (!busy && - q.owner != null && q.phase >= 0) + else if (p > 0) busy = true; } break; @@ -2232,37 +2237,40 @@ else if (WorkQueue.casSlotToNull(a, k, t)) { w.source = src; t.doExec(); w.source = wsrc; - rescan = locals = true; + locals = true; break scan; } } } } - if (rescan) - ; // retry - else if (phase >= 0) { + if (locals) + ; // remain active + else if (rescan || !canStop(false, false)) { + if (phase < 0) // tentatively reactivate + w.phase = phase = activePhase; + } + else if (phase >= 0) { // tentatively inactivate parkTime = 0L; w.phase = phase = inactivePhase; } - else if (!busy) { - w.phase = activePhase; - return 1; - } + else if (!busy) + break; // inactive and quiescent else if (parkTime == 0L) { - parkTime = 1L << 10; // initially about 1 usec + parkTime = 1L << 10; // initially about 1 usec Thread.yield(); } - else if ((interrupted = interruptible && Thread.interrupted()) || - System.nanoTime() - startTime > nanos) { - w.phase = activePhase; - return interrupted ? -1 : 0; + else if (System.nanoTime() - startTime > nanos) { + returnStatus = 0; + break; } else { LockSupport.parkNanos(this, parkTime); if (parkTime < nanos >>> 8 && parkTime < 1L << 20) - parkTime <<= 1; // max sleep approx 1 sec or 1% nanos + parkTime <<= 1; // max sleep approx 1 sec or 1% nanos } } + w.phase = activePhase; + return returnStatus; } /** @@ -2279,7 +2287,7 @@ private int externalHelpQuiesce(long nanos, boolean interruptible) { t.doExec(); parkTime = 0L; } - else if (canStop(false) < 0) + else if (canStop(true, false)) return 1; else if (parkTime == 0L) { parkTime = 1L << 10; @@ -2351,6 +2359,7 @@ final WorkQueue submissionQueue(boolean isSubmit) { break; else if ((q = qs[i = (n - 1) & id]) == null) { WorkQueue w = new WorkQueue(null, id | SRC); + w.phase = (id >>> 1) | INACTIVE; // for use as unique id w.array = new ForkJoinTask[INITIAL_QUEUE_CAPACITY]; lock.lock(); // install under lock if (queues == qs && qs[i] == null) @@ -2498,8 +2507,8 @@ static int getSurplusQueuedTaskCount() { /** * Possibly initiates and/or completes pool termination. If * terminating, helps complete termination by cancelling tasks, - * reactivating idle workers or interrupting active workers, and - * possibly triggering TERMINATED state. + * reactivating idle workers or interrupting active workers, until + * no more are found, then possibly triggering TERMINATED state. * * @param now if true, unconditionally terminate, else only * if no work and no active workers @@ -2509,42 +2518,61 @@ static int getSurplusQueuedTaskCount() { private int tryTerminate(boolean now, boolean enable) { int rs = runState; if ((config & ISCOMMON) == 0 && rs >= 0) { - if (!now) { + if (now) + rs = getAndBitwiseOrRunState(SHUTDOWN | STOP) | STOP; + else { if (enable && (rs & SHUTDOWN) == 0) rs = getAndBitwiseOrRunState(SHUTDOWN) | SHUTDOWN; if ((rs & SHUTDOWN) != 0) - rs = canStop(true); + canStop(true, true); // sets STOP if quiescent + rs = runState; } - else - rs = getAndBitwiseOrRunState(SHUTDOWN | STOP) | STOP; } - if ((rs & (STOP | TERMINATED)) != STOP) - return rs; // not terminating - WorkQueue[] qs; WorkQueue q; Thread thread; WorkQueue w; - ReentrantLock lock; Condition cond; - boolean more = (reactivate() != null); // try activating idle worker - Thread current = Thread.currentThread(); - int r = (((current instanceof ForkJoinWorkerThread) && - (w = ((ForkJoinWorkerThread)current).workQueue) != null) ? - w.config : 0); // stagger traversals - int n = ((qs = queues) == null) ? 0 : qs.length; - for (int i = 0; i < n; ++i) { - if ((q = qs[(r + i) & (n - 1)]) != null) { - for (ForkJoinTask t; (t = q.poll(null)) != null; ) - ForkJoinTask.cancelIgnoringExceptions(t); - if (!more && q.access != STOP && - (thread = q.owner) != null && thread != current && - (q.forcePhaseActive() < 0 || !thread.isInterrupted())) { - try { - thread.interrupt(); // help unblock others - } catch (Throwable ignore) { + if ((rs & (STOP | TERMINATED)) != STOP) // not terminating + return rs; + Thread current = Thread.currentThread(); // help terminate + WorkQueue w = ((current instanceof ForkJoinWorkerThread) ? + ((ForkJoinWorkerThread)current).workQueue : null); + int r = (w != null) ? w.config : 0; // stagger traversals + boolean rescan; + do { // repeat until cannot cancel, reactivate, or interrupt + boolean reactivated = rescan = (reactivate() != null); + WorkQueue[] qs = queues; + int n = (qs == null) ? 0 : qs.length; + for (int i = 0; i < n; ++i) { + WorkQueue q; Thread thread; int k, p; + if ((q = qs[k = (r + i) & (n - 1)]) != null && q.phase != 0 && + q.access != STOP) { + for (ForkJoinTask t; (t = q.poll(null)) != null; ) { + ForkJoinTask.cancelIgnoringExceptions(t); + rescan = true; // cancel tasks + } + if ((k & 1) == 0) { // is submission queue + if (q.getAndSetAccess(1) != 0) + rescan = true; // locked + else + q.access = q.phase = 0; // mark as done + } + else if (!reactivated) { + if ((p = q.phase) < 0) // reactivate next pass + rescan = true; + else if (p != 0 && (thread = q.owner) != null && + !thread.isInterrupted()) { + rescan = true; + try { + thread.interrupt(); // try to unblock + } catch (Throwable ignore) { + } + } } } } - } - if (reactivate() == null && // transition if no workers - (short)(ctl >>> TC_SHIFT) <= 0 && - (runState & TERMINATED) == 0 && + if (((rs = runState) & TERMINATED) != 0) + return rs; + } while (rescan); + + ReentrantLock lock; Condition cond; // transition when no workers + if ((short)(ctl >>> TC_SHIFT) <= 0 && (getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0 && (lock = registrationLock) != null) { lock.lock(); @@ -3138,34 +3166,33 @@ static final class InvokeAnyTask extends ForkJoinTask { this.root = root; } public final boolean exec() { - ForkJoinPool p; + InvokeAnyRoot r; ForkJoinPool p; Thread t = Thread.currentThread(); - Thread.interrupted(); - runner = t; - InvokeAnyRoot r = root; - Callable c = callable; - E v = null; - Throwable ex = null; - boolean completed = false; - if (r != null && c != null) { - if ((t instanceof ForkJoinWorkerThread) && + if ((r = root) != null && !r.isDone()) { + if ((t instanceof ForkJoinWorkerThread) && // termination check (p = ((ForkJoinWorkerThread) t).pool) != null && - p.runState < 0) { // termination check + p.runState < 0) r.trySetCancelled(); - t.interrupt(); // restore interrupt - } - else if (!r.isDone() && !isDone()) { - try { - v = c.call(); - completed = true; - } catch (Throwable rex) { - ex = rex; + else { + Thread.interrupted(); + runner = t; + E v = null; + Throwable ex = null; + boolean completed = false; + if (!isDone()) { + try { + v = callable.call(); + completed = true; + } catch (Throwable rex) { + ex = rex; + } } + runner = null; + if (trySetCancelled() >= 0) // avoid race with cancel + r.tryComplete(v, ex, completed); + Thread.interrupted(); } - if (trySetCancelled() >= 0) - r.tryComplete(v, ex, completed); } - runner = null; return true; } public final boolean cancel(boolean mayInterruptIfRunning) { @@ -3310,7 +3337,7 @@ public int getActiveThreadCount() { * @return {@code true} if all threads are currently idle */ public boolean isQuiescent() { - return canStop(false) < 0; + return canStop(true, false); } /** @@ -3385,7 +3412,7 @@ public int getQueuedSubmissionCount() { * @return {@code true} if there are any queued submissions */ public boolean hasQueuedSubmissions() { - return hasTasks(true); + return !canStop(false, false); } /** @@ -3587,9 +3614,8 @@ else if ((lock = registrationLock) != null) { try { if (cond == null && (cond = termination) == null) termination = cond = lock.newCondition(); - if ((runState & TERMINATED) != 0) - break; - nanos = cond.awaitNanos(nanos); + if ((runState & TERMINATED) == 0) + nanos = cond.awaitNanos(nanos); } finally { lock.unlock(); } @@ -3640,19 +3666,18 @@ public boolean awaitQuiescence(long timeout, TimeUnit unit) { @Override public void close() { ReentrantLock lock; - boolean interrupted = false; if ((runState & TERMINATED) == 0 && (config & ISCOMMON) == 0 && (lock = registrationLock) != null) { checkPermission(); Condition cond = null; + boolean interrupted = Thread.interrupted(); while ((tryTerminate(interrupted, true) & TERMINATED) == 0) { lock.lock(); try { if (cond == null && (cond = termination) == null) termination = cond = lock.newCondition(); - if ((runState & TERMINATED) != 0) - break; - cond.await(); + if ((runState & TERMINATED) == 0) + cond.await(); } catch (InterruptedException ex) { interrupted = true; } finally { diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index 421f8c402c6d5..4aa48bf10f877 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -425,6 +425,8 @@ else if ((how & RAN) != 0 || return s; if (s == UNCOMPENSATE) uncompensate = true; + if (p.runState < 0) // recheck below if interrupted + cancelIgnoringExceptions(this); } Aux node = null; long ns = 0L; @@ -446,13 +448,14 @@ else if (timed && (ns = deadline - System.nanoTime()) <= 0) { } else if (Thread.interrupted()) { interrupted = true; + if (p != null && p.runState < 0) + cancelIgnoringExceptions(this); if ((how & INTERRUPTIBLE) != 0) { - s = ABNORMAL; + if ((s = status) >= 0) + s = ABNORMAL; // prefer reporting result to IE break; } } - else if ((s = status) < 0) // recheck - break; else if (timed) LockSupport.parkNanos(ns); else @@ -480,11 +483,8 @@ else if (casAux(a, next)) } } } - int stat = status; // prefer completion result - if (stat < 0) - s = stat; } - if (s < 0) { + else { signalWaiters(); // help clean or signal if (interrupted) Thread.currentThread().interrupt(); @@ -1502,23 +1502,26 @@ static final class AdaptedInterruptibleCallable extends ForkJoinTask public final boolean exec() { ForkJoinPool p; Thread t = Thread.currentThread(); - Thread.interrupted(); - runner = t; - try { - if ((t instanceof ForkJoinWorkerThread) && - (p = ((ForkJoinWorkerThread) t).pool) != null && - p.runState < 0) // termination check - cancelIgnoringExceptions(this); - else if (!isDone()) - result = callable.call(); - return true; - } catch (RuntimeException rex) { - throw rex; - } catch (Exception ex) { - throw new RuntimeException(ex); - } finally { - runner = null; + if ((t instanceof ForkJoinWorkerThread) && + (p = ((ForkJoinWorkerThread) t).pool) != null && + p.runState < 0) // termination check + cancelIgnoringExceptions(this); + else { + Thread.interrupted(); + runner = t; + try { + if (!isDone()) + result = callable.call(); + } catch (RuntimeException rex) { + throw rex; + } catch (Exception ex) { + throw new RuntimeException(ex); + } finally { + runner = null; + Thread.interrupted(); + } } + return true; } public final void run() { invoke(); } public final boolean cancel(boolean mayInterruptIfRunning) { From b01fe39ae12b567e2da8d3dcfa35bd9dd2427fed Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sun, 22 Jan 2023 17:02:51 -0500 Subject: [PATCH 09/61] Check termination in ManagedBlockers --- .../share/classes/java/util/concurrent/ForkJoinPool.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 7a6a091ace52d..a8612717d2df0 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1920,7 +1920,11 @@ private int awaitWork(WorkQueue w) { if (p < 0) { long deadline = idle ? keepAlive + System.currentTimeMillis() : 0L; LockSupport.setCurrentBlocker(this); - for (;;) { // await signal + for (;;) { // await signal or termination + if (runState < 0) { // activate before exit + do {} while (w.phase < 0 && reactivate() != null); + return -1; + } w.access = PARKED; // enable unpark if (w.phase < 0) { if (idle) @@ -3810,6 +3814,8 @@ private void compensatedBlock(ManagedBlocker blocker) long c = ctl; if (blocker.isReleasable()) break; + if (runState < 0) // will be interrupted on cancellation + throw new InterruptedException(); if ((comp = tryCompensate(c, false)) >= 0) { long post = (comp == 0) ? 0L : RC_UNIT; try { From 212dc07e78ba007f27bac0db0f6db27d7cf5a9e1 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sat, 28 Jan 2023 16:28:01 -0500 Subject: [PATCH 10/61] More uniform handling of Interruptibles --- .../java/util/concurrent/ForkJoinPool.java | 498 ++++++++---------- .../java/util/concurrent/ForkJoinTask.java | 158 ++++-- 2 files changed, 336 insertions(+), 320 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index a8612717d2df0..70ce1c16fbcb1 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -579,28 +579,30 @@ public class ForkJoinPool extends AbstractExecutorService { * atomically set a mode bit. However, termination is * intrinsically non-atomic. The calling thread, as well as other * workers thereafter terminating, and any external callers - * checking termination help cancel queued tasks, reactivate idle - * workers, and interrupt others; repeating until no more are - * found by checking and marking ctl counts and phase and access - * fields. These actions race with non-terminated workers. - * Except when using Interruptible tasks (see below), there are no - * quarantees after an abrupt shutdown whether remaining tasks - * complete normally or exceptionally or are cancelled, and - * termination may fail if tasks repeatedly ignore both - * cancellation status and interrupts. + * checking termination help cancel queued tasks and unblock + * workers, repeating until no more are found. These actions race + * with non-terminated workers. Except when using Interruptible + * tasks (see below), there are no quarantees after an abrupt + * shutdown whether remaining tasks complete normally or + * exceptionally or are cancelled, and termination may fail if + * tasks repeatedly ignore both cancellation status and + * interrupts. * * Quiescent shutdown. Calls to non-abrupt shutdown() use method * canStop to trigger the "STOP" phase of termination upon - * quiescence, which is intrinsically racy, and requires - * convergent sweeps through queues to detect whether all workers - * are idle and there are no submissions that they could poll if - * they were not idle. If not immediately triggered, whenever all - * workers become idle in awaitWork, canStop is rechecked. Method - * helpQuiesce also uses canStop to help determine quiescence. It - * differs in that it does not trigger termination (even if - * enabled) and cannot rely on ctl counts to determine that all - * workers are inactive (because any executing helpQuiesce are not - * included). + * quiescence, which is intrinsically racy, and requires multiple + * scans of queues to detect whether all workers are idle and + * there are no submissions that they could poll if they were not + * idle. If two scans agree on state, then there was at least one + * point at which quiescence held, and so further submissions may + * be rejected. To avoid false-positives, we use a linear hash. + * (A similar scheme is used in tryTerminate.) If not immediately + * triggered, whenever all workers become idle in awaitWork, + * canStop is rechecked. Method helpQuiesce also uses canStop to + * help determine quiescence. It differs in that it does not + * trigger termination (even if enabled) and cannot rely on ctl + * counts to determine that all workers are inactive (because any + * executing helpQuiesce are not included). * * Trimming workers. To release resources after periods of lack of * use, a worker starting to wait when the pool is quiescent will @@ -740,27 +742,23 @@ public class ForkJoinPool extends AbstractExecutorService { * LockSupport.park do not loop indefinitely (park returns * immediately if the current thread is interrupted). * - * However, to conform to specs and expectations surrounding - * execution, cancellation and termination of Callable-based - * ExecutorService invoke, submit, invokeAll, and invokeAny - * methods (without penalizing others) we use "Interruptible" - * tasks -- AdaptedInterruptibleCallables, and a variant - * InvokeAnyTask. We ensure that external submitters do not help - * run such tasks by recording ForkJoinTask.POOLSUBMIT in task - * status and/or as a bit flag argument to other methods. In - * addition to recording a "runner" field (similarly to - * FutureTask) to support cancel(true), we ensure that upon pool - * shutdown, either the task is cancelled by the runner or the - * runner is interrupted so it can cancel. Often both, but since - * external joining callers never run these tasks, they must await - * cancellation (or a result; reporting the correct result or - * exception for task.get etc depending on pool status). As is - * the case with any kind of pool, it is possible that an + * Interruptible tasks. To conform to ExecutorService specs and + * expectations externally invoked ExecutorService methods use + * InterruptibleForkJoinTasks. External submitters do not help + * run such tasks (implemented by marking with + * ForkJoinTask.POOLSUBMIT in task status and/or as a bit flag + * argument to other methods). These tasks include a "runner" + * field (similarly to FutureTask) to support cancel(true), + * minimizing impact of "stray" interrupts and those in which an * interrupt designed to cancel one task occurs both late and - * unnecessarily, so instead "spuriously" interrupts the worker - * thread while performing a later task. Users should check - * cancellation status upon interrupt, but usually cannot because - * their Callables are wrapped and so cannot access status. + * unnecessarily. Upon pool shutdown, the task is cancelled, and + * runners are interrupted so they can cancel. Since external + * joining callers never run these tasks, they must await + * cancellation. + * + * See the ForkJoinTask internal documentation for an account of + * how these correspond to how and when different exceptions are + * thrown. * * Memory placement * ================ @@ -852,7 +850,7 @@ public class ForkJoinPool extends AbstractExecutorService { * * The main sources of differences from previous version are: * - * * Handling of Interruptible tasks consistent with + * * Handling of Interruptible tasks is now consistent with * ExecutorService specs, including uniform use of revised * canStop for quiescence-related checks and corresponding * revisions to termination checks @@ -1073,7 +1071,7 @@ static final class WorkQueue { @jdk.internal.vm.annotation.Contended("w") volatile int access; // values 0, 1 (locked), PARKED, STOP @jdk.internal.vm.annotation.Contended("w") - volatile int phase; // versioned, negative if inactive, 0 on exit + volatile int phase; // versioned, negative if inactive @jdk.internal.vm.annotation.Contended("w") volatile int source; // source queue id in topLevelExec @jdk.internal.vm.annotation.Contended("w") @@ -1095,9 +1093,6 @@ static boolean casSlotToNull(ForkJoinTask[] a, int i, return U.compareAndSetReference(a, ((long)i << ASHIFT) + ABASE, c, null); } - final int forcePhaseActive() { // clear sign bit - return U.getAndBitwiseAndInt(this, PHASE, 0x7fffffff); - } final int getAndSetAccess(int v) { return U.getAndSetInt(this, ACCESS, v); } @@ -1279,7 +1274,7 @@ else if (t != null && casSlotToNull(a, k, t)) { } else if (array != a || a[k] != null) ; // stale - else if (a[nk] == null && top - b <= 0) + else if (a[nk] == null && access <= 0 && top - b <= 0) break; // empty } return null; @@ -1672,10 +1667,9 @@ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { WorkQueue w = (wt == null) ? null : wt.workQueue; if (w != null) { cfg = w.config; - w.access = STOP; // mark as done; cancel tasks - w.phase = 0; - for (ForkJoinTask t; (t = w.nextLocalTask(0)) != null; ) - ForkJoinTask.cancelIgnoringExceptions(t); + w.access = STOP; // mark as done + do {} while (w.phase < 0 && // ensure released + reactivate() != null); } long c = ctl; if ((cfg & TRIMMED) == 0) // decrement counts @@ -1696,6 +1690,8 @@ else if ((int)c == 0) // was dropped on timeout stealCount += ns; // accumulate steals lock.unlock(); } + for (ForkJoinTask t; (t = w.nextLocalTask(0)) != null; ) + ForkJoinTask.cancelIgnoringExceptions(t); if ((cfg & SRC) != 0) signalWork(); // possibly replace worker } @@ -1787,55 +1783,6 @@ private boolean tryTrim(WorkQueue w) { return false; } - /** - * Internal version of isQuiescent and related functionality. - * Returns true if terminating or all submission queues are empty - * and unlocked, and unless !all, all workers are idle. To obtain - * a reliable snapshot, scans at least twice, and continues in - * case of inconsistencies. - * - * @param all if false, only check submission queues - * @param stopIfEnabled true if should transition runState to STOP - * if quiescent, in which case returns false - */ - private boolean canStop(boolean all, boolean stopIfEnabled) { - long checkSum = 0L, sum = 0L; // for consistency checks - for (int scans = 2;;) { // min remaining scans - if (runState < 0) - return true; // terminating - else if (all && (ctl & RC_MASK) > 0L) - return false; // not idle - else if (scans > 0) - --scans; // checkSum not valid - else if (checkSum == sum) { - if (stopIfEnabled && (runState & SHUTDOWN) != 0) { - getAndBitwiseOrRunState(STOP); - return false; // caller cannot stop yet - } - return true; - } - checkSum = sum; - sum = 0L; - WorkQueue[] qs; WorkQueue q; - int n = ((qs = queues) == null) ? 0 : qs.length; - for (int i = 0; i < n; ++i) { - if ((q = qs[i]) != null) { - int x = q.access, p = q.phase, s = q.top, d = s - q.base; - if ((i & 1) != 0) { - if (all && (p | x) > 0) - return false; // active worker - } - else if (x != 0 || d > 0 || q.access != 0) - return false; // locked or nonempty - else if (d != 0 || q.top != s) - scans = 1; // inconsistent - else // hash to associate s with q - sum = (sum << 5) - sum + s + p; - } - } - } - } - /** * Top-level runloop for workers, called by ForkJoinWorkerThread.run. * See above for explanation. @@ -1912,7 +1859,7 @@ private int awaitWork(WorkQueue w) { pc, qc = ((pc - RC_UNIT) & UC_MASK) | sp))); if ((idle = ((qc & RC_MASK) <= 0L)) && // check for stragglers !canStop(true, true) && (w.phase >= 0 || reactivate() == w)) - return 0; // rescan or quiescent terminate + return 0; // rescan or terminate WorkQueue[] qs = queues; // spin for expected #accesses in scan+signal int spins = ((qs == null) ? 0 : ((qs.length & SMASK) << 1)) | 0xf; while ((p = w.phase) < 0 && --spins > 0) @@ -1921,10 +1868,8 @@ private int awaitWork(WorkQueue w) { long deadline = idle ? keepAlive + System.currentTimeMillis() : 0L; LockSupport.setCurrentBlocker(this); for (;;) { // await signal or termination - if (runState < 0) { // activate before exit - do {} while (w.phase < 0 && reactivate() != null); - return -1; - } + if (runState < 0) + break; w.access = PARKED; // enable unpark if (w.phase < 0) { if (idle) @@ -1951,6 +1896,49 @@ private int awaitWork(WorkQueue w) { return 0; } + /** + * Internal version of isQuiescent and related functionality. + * Returns true if terminating or all submission queues are empty + * and unlocked, and all workers are idle. + * + * @param idle false if caller need not be counted as idle. + * @param stopIfEnabled true if should transition runState to STOP + * if quiescent, in which case returns false + */ + private boolean canStop(boolean idle, boolean stopIfEnabled) { + long prevSum = 0L; // at least 2 scans + for (boolean stable = false, rescan = true;;) { + if (runState < 0) + return true; // terminating + else if (idle && (ctl & RC_MASK) > 0L) + return false; // active workers exist + else if (stable) { + if (stopIfEnabled && (runState & SHUTDOWN) != 0) { + getAndBitwiseOrRunState(STOP); + return false; // caller cannot stop yet + } + return true; + } + long sum = 0L; + WorkQueue[] qs; WorkQueue q; + int n = ((qs = queues) == null) ? 0 : qs.length; + for (int i = 0; i < n; ++i) { + if ((q = qs[i]) != null) { + int x = q.access, p = q.phase, s = q.top, d = s - q.base; + if (x > 0 || p > 0 || d > 0 || q.access > 0) + return false; // locked, active or nonempty + if (d != 0 || q.top != s) + rescan = true; // inconsistent reads + sum = (sum << 5) - sum + s + p; // sequence hash + } + } + if (!rescan && sum == prevSum) + stable = true; + prevSum = sum; + rescan = false; + } + } + /** * Scans for and returns a polled task, if available. Used only * for untracked polls. Begins scan at a random index to avoid @@ -2191,12 +2179,13 @@ else if ((f = f.completer) == null) * @return positive if quiescent, negative if interrupted, else 0 */ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { - long startTime = System.nanoTime(), parkTime = 0L; + long startTime = System.nanoTime(); int phase; // w.phase set negative when temporarily quiescent if (w == null || (phase = w.phase) < 0) return 0; int activePhase = phase, inactivePhase = phase | INACTIVE; int wsrc = w.source, r = 0, returnStatus = 1; + long initialSleep = -parallelism, parkTime = 0L; for (boolean locals = true;;) { WorkQueue[] qs; WorkQueue q; if (runState < 0) @@ -2210,7 +2199,7 @@ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { for (ForkJoinTask u; (u = w.nextLocalTask()) != null;) u.doExec(); } - boolean rescan = false, busy = false; + boolean rescan = false; int n = ((qs = queues) == null) ? 0 : qs.length, m = n - 1; scan: for (int i = n, j; i > 0; --i, ++r) { if ((q = qs[j = m & r]) != null && q != w) { @@ -2225,13 +2214,8 @@ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { if (q.base != b || q.array != a || a[k] != t) ; else if (t == null) { - if (!rescan) { - int p = q.phase, s = q.top; - if (s - b > 0) - rescan = true; - else if (p > 0) - busy = true; - } + if (!rescan && (q.access > 0 || q.top - b > 0)) + rescan = true; break; } else if (phase < 0) // reactivate before taking @@ -2241,36 +2225,31 @@ else if (WorkQueue.casSlotToNull(a, k, t)) { w.source = src; t.doExec(); w.source = wsrc; - locals = true; + rescan = locals = true; break scan; } } } } - if (locals) - ; // remain active - else if (rescan || !canStop(false, false)) { - if (phase < 0) // tentatively reactivate - w.phase = phase = activePhase; - } - else if (phase >= 0) { // tentatively inactivate - parkTime = 0L; - w.phase = phase = inactivePhase; - } - else if (!busy) - break; // inactive and quiescent - else if (parkTime == 0L) { - parkTime = 1L << 10; // initially about 1 usec - Thread.yield(); - } - else if (System.nanoTime() - startTime > nanos) { - returnStatus = 0; - break; - } - else { - LockSupport.parkNanos(this, parkTime); - if (parkTime < nanos >>> 8 && parkTime < 1L << 20) - parkTime <<= 1; // max sleep approx 1 sec or 1% nanos + if (!rescan) { + if (phase >= 0) { // tentatively inactivate + w.phase = phase = inactivePhase; + parkTime = initialSleep; + } + if (canStop(false, false)) + break; // quiescent + else if (System.nanoTime() - startTime > nanos) { + returnStatus = 0; // timed out + break; + } + else if (parkTime <= 0L) // spin before sleep + ++parkTime; + else { // initial sleep approx 1 usec + parkTime = Math.max(parkTime, 1L << 10); + LockSupport.parkNanos(this, parkTime); + if (parkTime < nanos >>> 8 && parkTime < 1L << 20) + parkTime <<= 1; // max sleep approx 1 sec or 1% nanos + } } } w.phase = activePhase; @@ -2285,23 +2264,26 @@ else if (System.nanoTime() - startTime > nanos) { * @return positive if quiescent, negative if interrupted, else 0 */ private int externalHelpQuiesce(long nanos, boolean interruptible) { - for (long startTime = System.nanoTime(), parkTime = 0L;;) { + long startTime = System.nanoTime(); + long initialSleep = -parallelism, parkTime = 0L; + for (;;) { // same structure as helpQuiesce ForkJoinTask t; - if ((t = pollScan(false)) != null) { + if (runState < 0) + return 1; + else if (interruptible && Thread.interrupted()) + return -1; + else if ((t = pollScan(false)) != null) { + parkTime = initialSleep; t.doExec(); - parkTime = 0L; } else if (canStop(true, false)) return 1; - else if (parkTime == 0L) { - parkTime = 1L << 10; - Thread.yield(); - } - else if ((System.nanoTime() - startTime) > nanos) + else if (System.nanoTime() - startTime > nanos) return 0; - else if (interruptible && Thread.interrupted()) - return -1; + else if (parkTime <= 0L) + ++parkTime; else { + parkTime = Math.max(parkTime, 1L << 10); LockSupport.parkNanos(this, parkTime); if (parkTime < nanos >>> 8 && parkTime < 1L << 20) parkTime <<= 1; @@ -2509,10 +2491,7 @@ static int getSurplusQueuedTaskCount() { // Termination /** - * Possibly initiates and/or completes pool termination. If - * terminating, helps complete termination by cancelling tasks, - * reactivating idle workers or interrupting active workers, until - * no more are found, then possibly triggering TERMINATED state. + * Possibly initiates and/or completes pool termination. * * @param now if true, unconditionally terminate, else only * if no work and no active workers @@ -2527,65 +2506,58 @@ private int tryTerminate(boolean now, boolean enable) { else { if (enable && (rs & SHUTDOWN) == 0) rs = getAndBitwiseOrRunState(SHUTDOWN) | SHUTDOWN; - if ((rs & SHUTDOWN) != 0) - canStop(true, true); // sets STOP if quiescent - rs = runState; + if ((rs & SHUTDOWN) != 0) { + canStop(true, true); // sets STOP if quiescent + rs = runState; + } } } - if ((rs & (STOP | TERMINATED)) != STOP) // not terminating + if ((rs & (STOP | TERMINATED)) != STOP) // else help terminate return rs; - Thread current = Thread.currentThread(); // help terminate + reactivate(); // try to fan out + Thread current = Thread.currentThread(); WorkQueue w = ((current instanceof ForkJoinWorkerThread) ? ((ForkJoinWorkerThread)current).workQueue : null); - int r = (w != null) ? w.config : 0; // stagger traversals - boolean rescan; - do { // repeat until cannot cancel, reactivate, or interrupt - boolean reactivated = rescan = (reactivate() != null); + int r = (w != null) ? w.config : 0; // stagger traversals + // loop until all queues empty and all workers stopped or interrupted + long prevSum = 0L; // similar to canStop + for (boolean stable = false, rescan = true;;) { + if (((rs = runState) & TERMINATED) != 0) + return rs; + if (stable) { + ReentrantLock lock; Condition cond; // transition if no workers + if ((short)(ctl >>> TC_SHIFT) <= 0 && + (getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0 && + (lock = registrationLock) != null) { + lock.lock(); + if ((cond = termination) != null) + cond.signalAll(); + lock.unlock(); + container.close(); + } + return runState; + } + long sum = 0L; WorkQueue[] qs = queues; int n = (qs == null) ? 0 : qs.length; for (int i = 0; i < n; ++i) { - WorkQueue q; Thread thread; int k, p; - if ((q = qs[k = (r + i) & (n - 1)]) != null && q.phase != 0 && - q.access != STOP) { - for (ForkJoinTask t; (t = q.poll(null)) != null; ) { - ForkJoinTask.cancelIgnoringExceptions(t); - rescan = true; // cancel tasks - } - if ((k & 1) == 0) { // is submission queue - if (q.getAndSetAccess(1) != 0) - rescan = true; // locked - else - q.access = q.phase = 0; // mark as done - } - else if (!reactivated) { - if ((p = q.phase) < 0) // reactivate next pass - rescan = true; - else if (p != 0 && (thread = q.owner) != null && - !thread.isInterrupted()) { - rescan = true; - try { - thread.interrupt(); // try to unblock - } catch (Throwable ignore) { - } - } + WorkQueue q; Thread qt; + if ((q = qs[(r + i) & (n - 1)]) != null) { + for (ForkJoinTask f; (f = q.poll(null)) != null; ) + ForkJoinTask.cancelIgnoringExceptions(f); + if ((qt = q.owner) != null && q.access != STOP && + !qt.isInterrupted()) { + rescan = true; + ForkJoinTask.interruptIgnoringExceptions(qt); } + sum = (sum << 5) - sum + (q.phase & ~INACTIVE); } } - if (((rs = runState) & TERMINATED) != 0) - return rs; - } while (rescan); - - ReentrantLock lock; Condition cond; // transition when no workers - if ((short)(ctl >>> TC_SHIFT) <= 0 && - (getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0 && - (lock = registrationLock) != null) { - lock.lock(); - if ((cond = termination) != null) - cond.signalAll(); - lock.unlock(); - container.close(); + if (!rescan && sum == prevSum) + stable = true; + prevSum = sum; + rescan = false; } - return runState; } // Exported methods @@ -2935,7 +2907,7 @@ public ForkJoinTask submit(Callable task) { */ @Override public ForkJoinTask submit(Runnable task, T result) { - return poolSubmit(true, new ForkJoinTask.AdaptedRunnable(task, result)); + return poolSubmit(true, new ForkJoinTask.AdaptedInterruptibleRunnable(task, result)); } /** @@ -2948,7 +2920,7 @@ public ForkJoinTask submit(Runnable task, T result) { public ForkJoinTask submit(Runnable task) { return poolSubmit(true, (task instanceof ForkJoinTask) ? (ForkJoinTask) task // avoid re-wrap - : new ForkJoinTask.AdaptedRunnableAction(task)); + : new ForkJoinTask.AdaptedInterruptibleRunnable(task, null)); } // Added mainly for possible use in Loom @@ -3039,17 +3011,21 @@ public List> invokeAll(Collection> tasks) { ArrayList> futures = new ArrayList<>(tasks.size()); try { for (Callable t : tasks) { - ForkJoinTask f = + ForkJoinTask.AdaptedInterruptibleCallable f = new ForkJoinTask.AdaptedInterruptibleCallable(t); futures.add(f); poolSubmit(true, f); } - for (int i = futures.size() - 1; i >= 0; --i) - ((ForkJoinTask)futures.get(i)).quietlyJoin(); + for (int i = futures.size() - 1; i >= 0; --i) { + ForkJoinTask.AdaptedInterruptibleCallable f = + (ForkJoinTask.AdaptedInterruptibleCallable)futures.get(i); + f.quietlyJoin(); + } return futures; } catch (Throwable t) { for (Future e : futures) - ForkJoinTask.cancelIgnoringExceptions(e); + ForkJoinTask.cancelIgnoringExceptions( + (ForkJoinTask.AdaptedInterruptibleCallable)e); throw t; } } @@ -3062,7 +3038,7 @@ public List> invokeAll(Collection> tasks, ArrayList> futures = new ArrayList<>(tasks.size()); try { for (Callable t : tasks) { - ForkJoinTask f = + ForkJoinTask.AdaptedInterruptibleCallable f = new ForkJoinTask.AdaptedInterruptibleCallable(t); futures.add(f); poolSubmit(true, f); @@ -3070,7 +3046,8 @@ public List> invokeAll(Collection> tasks, long startTime = System.nanoTime(), ns = nanos; boolean timedOut = (ns < 0L); for (int i = futures.size() - 1; i >= 0; --i) { - ForkJoinTask f = (ForkJoinTask)futures.get(i); + ForkJoinTask.AdaptedInterruptibleCallable f = + (ForkJoinTask.AdaptedInterruptibleCallable)futures.get(i); if (!f.isDone()) { if (!timedOut) timedOut = !f.quietlyJoin(ns, TimeUnit.NANOSECONDS); @@ -3083,18 +3060,18 @@ public List> invokeAll(Collection> tasks, return futures; } catch (Throwable t) { for (Future e : futures) - ForkJoinTask.cancelIgnoringExceptions(e); + ForkJoinTask.cancelIgnoringExceptions( + (ForkJoinTask.AdaptedInterruptibleCallable)e); throw t; } } /** - * Task to hold results for invokeAny, or to report exception if - * all subtasks fail or are cancelled or the pool is terminating. - * Note: Among other oddities, the ExecutorService.invokeAny spec - * requires that the all-cancelled case (including cancellation of - * the root because of pool termination) be reported as an - * ExecutionException, not a CancellationException. + * Task (that is never forked) to hold results for invokeAny, or + * to report exception if all subtasks fail or are cancelled or + * the pool is terminating. ForkJoinTask tags are used to avoid + * multiple calls to tryComplete by the same task under async + * cancellation. */ static final class InvokeAnyRoot extends ForkJoinTask { private static final long serialVersionUID = 2838392045355241008L; @@ -3107,13 +3084,17 @@ static final class InvokeAnyRoot extends ForkJoinTask { throw new IllegalArgumentException(); count = new AtomicInteger(n); } - final void tryComplete(E v, Throwable ex, boolean completed) { - if (!isDone()) { + final void tryComplete(InvokeAnyTask f, E v, Throwable ex, + boolean completed) { + ForkJoinPool p; + if (!isDone() && f != null && + f.compareAndSetForkJoinTaskTag((short)0, (short)1)) { if (completed) { result = v; quietlyComplete(); } - else if (count.getAndDecrement() <= 1) { + else if (count.getAndDecrement() <= 1 || + ((p = getPool()) != null && p.runState < 0)) { if (ex == null) trySetCancelled(); else @@ -3153,69 +3134,37 @@ final E invokeAny(Collection> tasks, } /** - * Variant of AdaptedInterruptibleCallable with results in - * InvokeAnyRoot (and never independently joined). Cancellation - * status is used to avoid multiple calls to tryComplete by the - * same task under async cancellation. + * AdaptedInterruptibleCallable with results in InvokeAnyRoot (and + * never independently joined). */ - static final class InvokeAnyTask extends ForkJoinTask { + static final class InvokeAnyTask extends ForkJoinTask.AdaptedInterruptibleCallable { private static final long serialVersionUID = 2838392045355241008L; final InvokeAnyRoot root; - @SuppressWarnings("serial") // Conditionally serializable - final Callable callable; - transient volatile Thread runner; InvokeAnyTask(InvokeAnyRoot root, Callable callable) { - if (callable == null) throw new NullPointerException(); - this.callable = callable; + super(callable); this.root = root; } - public final boolean exec() { - InvokeAnyRoot r; ForkJoinPool p; - Thread t = Thread.currentThread(); + final E compute() throws Exception { + InvokeAnyRoot r; if ((r = root) != null && !r.isDone()) { - if ((t instanceof ForkJoinWorkerThread) && // termination check - (p = ((ForkJoinWorkerThread) t).pool) != null && - p.runState < 0) - r.trySetCancelled(); - else { - Thread.interrupted(); - runner = t; - E v = null; - Throwable ex = null; - boolean completed = false; - if (!isDone()) { - try { - v = callable.call(); - completed = true; - } catch (Throwable rex) { - ex = rex; - } - } - runner = null; - if (trySetCancelled() >= 0) // avoid race with cancel - r.tryComplete(v, ex, completed); - Thread.interrupted(); - } - } - return true; - } - public final boolean cancel(boolean mayInterruptIfRunning) { - int s; Thread t; InvokeAnyRoot r; - if ((s = trySetCancelled()) >= 0) { - if ((r = root) != null) - r.tryComplete(null, null, false); - if (mayInterruptIfRunning && (t = runner) != null) { - try { - t.interrupt(); - } catch (Throwable ignore) { - } + E v = null; Throwable ex = null; boolean completed = false; + try { + v = callable.call(); + completed = true; + } catch (Throwable rex) { + ex = rex; + } finally { + r.tryComplete(this, v, ex, completed); } - return true; } - return ((s & (ABNORMAL | THROWN)) == ABNORMAL); + return null; + } + public boolean cancel(boolean mayInterruptIfRunning) { + InvokeAnyRoot r = root; + if (r != null) + r.tryComplete(this, null, null, false); + return super.cancel(mayInterruptIfRunning); } - public final void setRawResult(E v) {} // unused - public final E getRawResult() { return null; } } @Override @@ -3416,7 +3365,14 @@ public int getQueuedSubmissionCount() { * @return {@code true} if there are any queued submissions */ public boolean hasQueuedSubmissions() { - return !canStop(false, false); + WorkQueue[] qs; WorkQueue q; + if (runState >= 0 && (qs = queues) != null) { + for (int i = 0; i < qs.length; i += 2) { + if ((q = qs[i]) != null && q.queueSize() > 0) + return true; + } + } + return false; } /** @@ -3868,7 +3824,7 @@ private static void unmanagedBlock(ManagedBlocker blocker) @Override protected RunnableFuture newTaskFor(Runnable runnable, T value) { - return new ForkJoinTask.AdaptedRunnable(runnable, value); + return new ForkJoinTask.AdaptedInterruptibleRunnable(runnable, value); } @Override diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index 4aa48bf10f877..aff45d931a4e5 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -208,7 +208,36 @@ public abstract class ForkJoinTask implements Future, Serializable { * See the internal documentation of class ForkJoinPool for a * general implementation overview. ForkJoinTasks are mainly * responsible for maintaining their "status" field amidst relays - * to methods in ForkJoinWorkerThread and ForkJoinPool. + * to methods in ForkJoinWorkerThread and ForkJoinPool, along with + * recording and reporting exceptions. The status field mainly + * holds bits recording completion status. There is no bit + * representing "running", to recording whether incomplete tasks + * are queued vs executing (although these cases can be + * distinguished in InterruptibleForkJoinTasks). Cancellation is + * recorded in status bits (ABNORMAL but not THROWN), but reported + * in joining methods by throwing an exception. Other exceptions + * of completed (THROWN) tasks are recorded in the "aux" field, + * but may be reconstructed (in getThrowableException) to produce + * more useful stack traces when reported. Sentinels for being + * interrupted or timing out while waiting for completion are not + * recorded as status bits but are included in return values of + * methods in which they occur. + * + * The rules for how these are managed and issued in public + * methods have evolved across versions of this class. Those for + * tasks with results accessed via join() differ from those via + * Future.get(), which may also vary when invoked using pool + * submit methods by non-workers. In particular, internal usages + * of ForkJoinTasks ignore interrupt status when executing or + * awaiting completion. In all other cases, reporting task status + * (results or exceptions while executing) is always preferred to + * interrupts (also timeouts), restoring caller interrupt status + * in case of races. Similarly, completion status is preferred to + * reporting cancellation (ensured by atomically setting status), + * including cases where tasks are cancelled during termination. + * Cancellation is reported as an unchecked exception by join(), + * and by worker calls to get(), but is otherwise wrapped by a + * (checked) ExecutionException. * * The methods of this class are more-or-less layered into * (1) basic status maintenance @@ -216,12 +245,6 @@ public abstract class ForkJoinTask implements Future, Serializable { * (3) user-level methods that additionally report results. * This is sometimes hard to see because this file orders exported * methods in a way that flows well in javadocs. - * - * Revision notes: This class uses jdk-internal Unsafe for atomics - * and special memory modes, rather than VarHandles, to avoid - * initialization dependencies in other jdk components that - * require early parallelism. It also simplifies handling of - * pool-submitted tasks, among other minor improvements. */ /** @@ -498,7 +521,7 @@ else if (casAux(a, next)) * spec'ed not to throw any exceptions, but if it does anyway, we * have no recourse, so guard against this case. */ - static final void cancelIgnoringExceptions(Future t) { + static final void cancelIgnoringExceptions(ForkJoinTask t) { if (t != null) { try { t.cancel(true); @@ -507,6 +530,16 @@ static final void cancelIgnoringExceptions(Future t) { } } + /* Similarly, for interrupts */ + static final void interruptIgnoringExceptions(Thread t) { + if (t != null) { + try { + t.interrupt(); + } catch (Throwable ignore) { + } + } + } + /** * Returns a rethrowable exception for this task, if available. * To provide accurate stack traces, if the exception was not @@ -1478,69 +1511,97 @@ public String toString() { } /** - * Adapter for Callable-based tasks that are designed to be - * interruptible when cancelled, including cases of cancellation - * upon pool termination. In addition to recording the running - * thread to enable interrupt in cancel(true), the task checks for - * termination before executing the compute method, to cover - * shutdown races in which the task has not yet been cancelled on - * entry but might not otherwise be re-interrupted by others. - */ - static final class AdaptedInterruptibleCallable extends ForkJoinTask - implements RunnableFuture { - @SuppressWarnings("serial") // Conditionally serializable - final Callable callable; + * Tasks that are designed to be interruptible when cancelled, + * including cases of cancellation upon pool termination. In + * addition to recording the running thread to enable interrupt in + * cancel(true), the task checks for termination before executing + * the compute method, to cover shutdown races in which the task + * has not yet been cancelled on entry but might not otherwise be + * re-interrupted by others. + */ + static abstract class InterruptibleForkJoinTask extends ForkJoinTask { transient volatile Thread runner; - @SuppressWarnings("serial") // Conditionally serializable - T result; - AdaptedInterruptibleCallable(Callable callable) { - if (callable == null) throw new NullPointerException(); - this.callable = callable; + abstract T compute() throws Exception; + static boolean isTerminating(Thread t) { + ForkJoinPool p; + return ((t instanceof ForkJoinWorkerThread) && + (p = ((ForkJoinWorkerThread) t).pool) != null && + p.runState < 0); } - public final T getRawResult() { return result; } - public final void setRawResult(T v) { result = v; } public final boolean exec() { - ForkJoinPool p; Thread t = Thread.currentThread(); - if ((t instanceof ForkJoinWorkerThread) && - (p = ((ForkJoinWorkerThread) t).pool) != null && - p.runState < 0) // termination check - cancelIgnoringExceptions(this); + if (isTerminating(t)) + cancel(false); else { Thread.interrupted(); runner = t; try { if (!isDone()) - result = callable.call(); - } catch (RuntimeException rex) { - throw rex; + setRawResult(compute()); } catch (Exception ex) { - throw new RuntimeException(ex); + trySetThrown(ex); } finally { runner = null; - Thread.interrupted(); } } return true; } - public final void run() { invoke(); } - public final boolean cancel(boolean mayInterruptIfRunning) { - Thread t; - boolean stat = super.cancel(false); - if (mayInterruptIfRunning && (t = runner) != null) { - try { - t.interrupt(); - } catch (Throwable ignore) { - } - } - return stat; + public boolean cancel(boolean mayInterruptIfRunning) { + int s = trySetCancelled(); + if (s >= 0 && mayInterruptIfRunning) + ForkJoinTask.interruptIgnoringExceptions(runner); + return (s >= 0 || (s & (ABNORMAL | THROWN)) == ABNORMAL); + } + private static final long serialVersionUID = 2838392045355241008L; + } + + /** + * Adapter for Callable-based interruptible tasks. + */ + static class AdaptedInterruptibleCallable extends InterruptibleForkJoinTask + implements RunnableFuture { + @SuppressWarnings("serial") // Conditionally serializable + final Callable callable; + @SuppressWarnings("serial") // Conditionally serializable + T result; + AdaptedInterruptibleCallable(Callable callable) { + if (callable == null) throw new NullPointerException(); + this.callable = callable; } + public final T getRawResult() { return result; } + public final void setRawResult(T v) { result = v; } + T compute() throws Exception { return callable.call(); } + public final void run() { invoke(); } public String toString() { return super.toString() + "[Wrapped task = " + callable + "]"; } private static final long serialVersionUID = 2838392045355241008L; } + /** + * Adapter for Runnable-based interruptible tasks. + */ + static final class AdaptedInterruptibleRunnable extends InterruptibleForkJoinTask + implements RunnableFuture { + @SuppressWarnings("serial") // Conditionally serializable + final Runnable runnable; + @SuppressWarnings("serial") // Conditionally serializable + final T result; + AdaptedInterruptibleRunnable(Runnable runnable, T result) { + if (runnable == null) throw new NullPointerException(); + this.runnable = runnable; + this.result = result; + } + public final T getRawResult() { return result; } + public final void setRawResult(T v) { } + final T compute() { runnable.run(); return result; } + public final void run() { invoke(); } + public String toString() { + return super.toString() + "[Wrapped task = " + runnable + "]"; + } + private static final long serialVersionUID = 2838392045355241008L; + } + /** * Returns a new {@code ForkJoinTask} that performs the {@code run} * method of the given {@code Runnable} as its action, and returns @@ -1596,7 +1657,6 @@ public static ForkJoinTask adapt(Callable callable) { * @since 19 */ public static ForkJoinTask adaptInterruptible(Callable callable) { - // https://bugs.openjdk.org/browse/JDK-8246587 return new AdaptedInterruptibleCallable(callable); } From d34879ee8b5419cb191232040987b64adfd396d7 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Mon, 13 Feb 2023 10:28:58 -0500 Subject: [PATCH 11/61] Redesign using ExecutorServiceTasks for external submissions --- .../util/concurrent/CountedCompleter.java | 17 +- .../java/util/concurrent/ForkJoinPool.java | 935 ++++++++++-------- .../java/util/concurrent/ForkJoinTask.java | 357 ++++--- .../concurrent/tck/ForkJoinPool19Test.java | 1 + 4 files changed, 718 insertions(+), 592 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java b/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java index 3278d191074ea..76d9378b7db7e 100644 --- a/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java +++ b/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java @@ -724,13 +724,13 @@ public final void quietlyCompleteRoot() { * processed. */ public final void helpComplete(int maxTasks) { - ForkJoinPool.WorkQueue q; Thread t; boolean owned; - if (owned = (t = Thread.currentThread()) instanceof ForkJoinWorkerThread) + ForkJoinPool.WorkQueue q; Thread t; boolean internal; + if (internal = (t = Thread.currentThread()) instanceof ForkJoinWorkerThread) q = ((ForkJoinWorkerThread)t).workQueue; else q = ForkJoinPool.commonQueue(); if (q != null && maxTasks > 0) - q.helpComplete(this, owned, maxTasks); + q.helpComplete(this, internal, maxTasks); } // ForkJoinTask overrides @@ -777,6 +777,17 @@ protected final boolean exec() { @Override protected void setRawResult(T t) { } + /** + * Overrride default recursive helping to instead help along + * completion chains. + */ + @Override + final int tryHelpJoin(ForkJoinPool p, ForkJoinPool.WorkQueue q, + boolean internal, int how) { + return ((q == null || p == null) ? 0 : + p.helpComplete(this, q, internal, (how & TIMED) != 0)); + } + /* * This class uses jdk-internal Unsafe for atomics and special * memory modes, rather than VarHandles, to avoid initialization diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 70ce1c16fbcb1..e2d0cfbd2352d 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -49,9 +49,9 @@ import java.util.List; import java.util.function.Predicate; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.locks.LockSupport; -import java.util.concurrent.locks.ReentrantLock; -import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.AbstractQueuedSynchronizer; import jdk.internal.access.JavaUtilConcurrentFJPAccess; import jdk.internal.access.SharedSecrets; import jdk.internal.misc.Unsafe; @@ -309,9 +309,11 @@ public class ForkJoinPool extends AbstractExecutorService { * the poll operation. Stalls can be detected by observing that * q.base doesn't change on repeated reads of null t and when * no other alternatives apply, spin-wait for it to settle. To - * reduce producing these kinds of stalls by other stealers, we - * encourage timely writes to indices using store fences when - * memory ordering is not already constrained by context. + * reduce producing these kinds of stalls by other stealers (as + * well as quiescence checks etc described below), we encourage + * timely writes to indices using otherwise unnecessarily + * strong writes or store fences when memory ordering is not + * already constrained by context. * * * The CAS may fail, in which case we may want to retry unless * there is too much contention. One goal is to balance and @@ -364,12 +366,14 @@ public class ForkJoinPool extends AbstractExecutorService { * Insertion of tasks in shared mode requires a lock. We use only * a simple spinlock because submitters encountering a busy queue * move to a different position to use or create other queues. - * They (spin) block when registering new queues, and less - * often in tryRemove and helpComplete. The lock needed for - * external queues is generalized (as field "access") for - * operations on owned queues that require a fully-fenced write - * (including push, parking status, and termination) in order to - * deal with Dekker-like signalling constructs described below. + * They (spin) block when registering new queues, or indirectly + * elsewhere (by revisiting later. The lock needed for external + * queues is generalized (as field "access") for operations on + * internal queues that require a fully-fenced write to deal with + * Dekker-like signalling constructs described below. Because of + * the lock, checking non-quiescence of submission queues can be + * easier than worker queues because of the added check that if + * the lock is held, it is not quiescent. * * Management * ========== @@ -405,19 +409,25 @@ public class ForkJoinPool extends AbstractExecutorService { * * Field "runState" holds lifetime status, atomically and * monotonically setting SHUTDOWN, STOP, and finally TERMINATED - * bits. It is updated only via bitwise atomics (getAndBitwiseOr). + * bits. It is updated only via bitwise atomics (getAndBitwiseOr), + * although triggering STOP is performed under the registration + * lock to disable updates. The bits are arranged such that if + * runState is 0, the pool is not in a shutdown-able state, and if + * negative, it is stopping or terminated. * * Array "queues" holds references to WorkQueues. It is updated * (only during worker creation and termination) under the - * registrationLock, but is otherwise concurrently readable (often - * prefaced by a volatile read of mode to check termination, that - * is required anyway, and serves as an acquire fence). To - * simplify index-based operations, the array size is always a - * power of two, and all readers must tolerate null slots. Worker - * queues are at odd indices. Worker ids masked with SMASK match - * their index. Shared (submission) queues are at even - * indices. Grouping them together in this way simplifies and - * speeds up task scanning. + * registrationLock, but is otherwise concurrently readable but + * reads are always prefaced by a volatile read of runState to + * check termination, that also serves as an acquire fence. The + * registration lock is a simple AQS-based Sequence Lock, to + * enable retries if held during queiscence checks. To simplify + * index-based operations, the array size is always a power of + * two, and all readers must tolerate null slots. Worker queues + * are at odd indices. Worker ids masked with SMASK match their + * index. Shared (submission) queues are at even indices. Grouping + * them together in this way simplifies and speeds up task + * scanning. * * All worker thread creation is on-demand, triggered by task * submissions, replacement of terminated workers, and/or @@ -429,7 +439,13 @@ public class ForkJoinPool extends AbstractExecutorService { * one source of some of the messy code constructions here). In * essence, the queues array serves as a weak reference * mechanism. In particular, the stack top subfield of ctl stores - * indices, not references. + * indices, not references. Operations on queues obtained from + * these indices remain OK (with at most some unnecessary extra + * work) even if an underlying worker failed and was replaced by + * another at the same index. There are two contexts requiring + * precautions about this: In checkQuiescence, the SeqLock is used + * to recheck upon queue array updates. And during termination, + * worker queue array updates are disabled. * * Queuing Idle Workers. Unlike HPC work-stealing frameworks, we * cannot let workers spin indefinitely scanning for tasks when @@ -482,8 +498,7 @@ public class ForkJoinPool extends AbstractExecutorService { * available (and so is never checked in this way). When queued, * the lower 16 bits of its phase must hold its pool index. So we * place the index there upon initialization and never modify - * these bits, which also allows use as a versioned id in other - * contexts. + * these bits. * * The ctl field also serves as the basis for memory * synchronization surrounding activation. This uses a more @@ -560,49 +575,62 @@ public class ForkJoinPool extends AbstractExecutorService { * quieter spinwaits in awaitWork; currently set to small values * that only cover near-miss scenarios for deactivate vs activate * races. Because idle workers are often not yet blocked (via - * LockSupport.park), we use the WorkQueue access field to + * LockSupport.park), we use the WorkQueue source field to * advertise that a waiter actually needs unparking upon signal. * - * When idle workers are not continually woken up, the count - * fields in ctl allow efficient and accurate discovery of - * quiescent states (i.e., when all workers are idle) after - * deactivation. However, this voting mechanism alone does not - * guarantee that a pool can become dormant (or terminated), - * because external racing producers do not vote, and can - * asynchronously submit new tasks. To deal with this, the final - * unparked thread (in awaitWork) uses canStop (see below) to - * check for tasks that could have been added during a race window - * that would not be accompanied by a signal, in which case - * re-activating itself (or any other worker) to recheck. + * Quiescence. Workers scan looking for work, giving up when they + * don't find any, without being sure that none are available. + * However, some required functionality relies on consensus about + * quiescence (also termination, discussed below). The count + * fields in ctl allow efficient (modulo contention) and accurate + * discovery of states in which all workers are idle. However, + * because external (asynchronous) submitters are not part of this + * vote, these mechanisms themselves do not guarantee that the + * pool is in a quiescent state with respect to methods + * isQuiescent, shutdown (which begins termination when + * quiescent), helpQuiesce, and indirectly others including + * tryCompensate. Method checkQuiescence() is used in all of these + * contexts. It provides checks that all workers are idle and + * there are no submissions that they could poll if they were not + * idle, retrying on inconsistent reads of queues and using the + * SeqLock to retry on queue array updates. (It also reports + * quiescence if the pool is terminating.) A true report means + * only that there was a moment at which quiescence held; further + * actions based on this may be racy (requiring a seq lock upgrade + * to trigger quiescent termination). False negatives are + * inevitable (for example when queues indices lag updates, as + * described above), which is accommodated when (tentatively) idle + * by scanning for work etc, and then re-invoking. This includes + * cases in which the final unparked thread (in awaitWork) uses + * checkQuiescence() to check for tasks that could have been added + * during a race window that would not be accompanied by a signal, + * in which case re-activating itself (or any other worker) to + * rescan. This check also serves as a liveness safeguard when all + * other workers gave up prematurely and deactivated. Method + * helpQuiesce also uses checkQuiescence, but cannot rely on ctl + * counts to determine that all workers are inactive because the + * caller and any others executing helpQuiesce are not included. * * Termination. A call to shutdownNow invokes tryTerminate to - * atomically set a mode bit. However, termination is - * intrinsically non-atomic. The calling thread, as well as other - * workers thereafter terminating, and any external callers - * checking termination help cancel queued tasks and unblock - * workers, repeating until no more are found. These actions race - * with non-terminated workers. Except when using Interruptible - * tasks (see below), there are no quarantees after an abrupt - * shutdown whether remaining tasks complete normally or - * exceptionally or are cancelled, and termination may fail if - * tasks repeatedly ignore both cancellation status and - * interrupts. - * - * Quiescent shutdown. Calls to non-abrupt shutdown() use method - * canStop to trigger the "STOP" phase of termination upon - * quiescence, which is intrinsically racy, and requires multiple - * scans of queues to detect whether all workers are idle and - * there are no submissions that they could poll if they were not - * idle. If two scans agree on state, then there was at least one - * point at which quiescence held, and so further submissions may - * be rejected. To avoid false-positives, we use a linear hash. - * (A similar scheme is used in tryTerminate.) If not immediately - * triggered, whenever all workers become idle in awaitWork, - * canStop is rechecked. Method helpQuiesce also uses canStop to - * help determine quiescence. It differs in that it does not - * trigger termination (even if enabled) and cannot rely on ctl - * counts to determine that all workers are inactive (because any - * executing helpQuiesce are not included). + * atomically set a mode bit, performed under the registration + * lock to atomically disable further array updates. However, the + * process of termination is intrinsically non-atomic. The calling + * thread, as well as other workers thereafter terminating help + * cancel queued tasks and interrupt other workers. These actions + * race with unterminated workers. By default, workers check for + * termination only when accessing pool state (usually the + * "queues" array). ExecutorServicesTasks (see below) + * additionally perform runState checks before executing their + * task bodies. But otherwise, there are no quarantees after an + * abrupt shutdown that remaining tasks complete normally or + * exceptionally or are cancelled. Termination may fail to + * complete if running tasks repeatedly ignore both task status + * and interrupts and/or produce more tasks after others that + * could cancel them have exited. Parallelizing termination + * provides multiple attempts to cancel tasks and workers when + * some are only boundedly unresponsive, in addition to speeding + * termination when there are large numbers of tasks that need to + * be cancelled. * * Trimming workers. To release resources after periods of lack of * use, a worker starting to wait when the pool is quiescent will @@ -631,33 +659,32 @@ public class ForkJoinPool extends AbstractExecutorService { * method tryCompensate() may create or re-activate a spare * thread to compensate for blocked joiners until they unblock. * - * A third form (implemented via tryRemove) amounts to helping a - * hypothetical compensator: If we can readily tell that a - * possible action of a compensator is to steal and execute the + * A third form (implemented via tryRemoveAndExec) amounts to + * helping a hypothetical compensator: If we can readily tell that + * a possible action of a compensator is to steal and execute the * task being joined, the joining thread can do so directly, - * without the need for a compensation thread; although with a - * possibility of reduced parallelism because of a transient gap - * in the queue array that stalls stealers. + * without the need for a compensation thread; This may result in + * reordering a task slot in the queue, but in these cases, the + * join orders do not match (LIFO) submission orders anyway, so + * the reordering improves future processing. * * Other intermediate forms available for specific task types (for * example helpAsyncBlocker) often avoid or postpone the need for * blocking or compensation. * - * The ManagedBlocker extension API can't use helping so relies - * only on compensation in method awaitBlocker. - * * The algorithm in helpJoin entails a form of "linear helping". - * Each worker records (in field "source") a reference to the - * queue from which it last stole a task. The scan in method - * helpJoin uses these markers to try to find a worker to help - * (i.e., steal back a task from and execute it) that could hasten - * completion of the actively joined task. Thus, the joiner - * executes a task that would be on its own local deque if the - * to-be-joined task had not been stolen. This is a conservative - * variant of the approach described in Wagner & Calder - * "Leapfrogging: a portable technique for implementing efficient - * futures" SIGPLAN Notices, 1993 - * (http://portal.acm.org/citation.cfm?id=155354). It differs + * Each worker records (in field "source") an offset index to the + * queue from which it last stole a task. (Implementation note: to + * distinguish lack of source from index 0 or other sentinels, + * source fields are offset by SRC.) The scan in method helpJoin + * uses these markers to try to find a worker to help (i.e., steal + * back a task from and execute it) that could hasten completion + * of the actively joined task. Thus, the joiner executes a task + * that would be on its own local deque if the to-be-joined task + * had not been stolen. This is a conservative variant of the + * approach described in Wagner & Calder "Leapfrogging: a portable + * technique for implementing efficient futures" SIGPLAN Notices, + * 1993 (http://portal.acm.org/citation.cfm?id=155354). It differs * mainly in that we only record queues, not full dependency * links. This requires a linear scan of the queues array to * locate stealers, but isolates cost to when it is needed, rather @@ -679,12 +706,27 @@ public class ForkJoinPool extends AbstractExecutorService { * compensations for any blocked join. However, in practice, the * vast majority of blockages are transient byproducts of GC and * other JVM or OS activities that are made worse by replacement - * when they cause longer-term oversubscription. Rather than - * impose arbitrary policies, we allow users to override the - * default of only adding threads upon apparent starvation. The - * compensation mechanism may also be bounded. Bounds for the - * commonPool better enable JVMs to cope with programming errors - * and abuse before running out of resources to do so. + * when they cause longer-term oversubscription. These are + * inevitable without (unobtainably) perfect information about + * whether worker creation is actually necessary. False alarms + * are common enough to negatively impact performance, so + * compensation is by default attempted only when it is possible + * the the pool could stall due to lack of any unblocked workers. + * However, we allow users to override defaults using the long + * form of the ForkJoinPool constructor. The compensation + * mechanism may also be bounded. Bounds for the commonPool + * better enable JVMs to cope with programming errors and abuse + * before running out of resources to do so. + * + * The ManagedBlocker extension API can't use helping so relies + * only on compensation in method awaitBlocker. This API was + * designed to highlight the uncertainty of compensation decisions + * by requiring implementation of method isReleasable to abort + * compensation during attempts to obtain a stable snapshot. But + * users now rely upon the fact that if isReleasable always + * returns false, the API can be used to obtain precautionary + * compensation, which is sometimes their only reasonable option + * when running unknown code in tasks. * * Common Pool * =========== @@ -707,11 +749,10 @@ public class ForkJoinPool extends AbstractExecutorService { * sensible to set common pool parallelism level to one (or more) * less than the total number of available cores, or even zero for * pure caller-runs. For the sake of ExecutorService specs, we can - * only do this for tasks entered via fork, not submit. We track - * this using a task status bit (markPoolSubmission). In all - * other cases, external threads waiting for joins first check the - * common pool for their task, which fails quickly if the caller - * did not fork to common pool. + * only do this tasks not covered by ExecutorService APIs (see + * below). In all other cases, external threads waiting for joins + * first check the common pool for their task, which fails quickly + * if the caller did not fork to common pool. * * Guarantees for common pool parallelism zero are limited to * tasks that are joined by their callers in a tree-structured @@ -742,23 +783,18 @@ public class ForkJoinPool extends AbstractExecutorService { * LockSupport.park do not loop indefinitely (park returns * immediately if the current thread is interrupted). * - * Interruptible tasks. To conform to ExecutorService specs and - * expectations externally invoked ExecutorService methods use - * InterruptibleForkJoinTasks. External submitters do not help - * run such tasks (implemented by marking with - * ForkJoinTask.POOLSUBMIT in task status and/or as a bit flag - * argument to other methods). These tasks include a "runner" - * field (similarly to FutureTask) to support cancel(true), - * minimizing impact of "stray" interrupts and those in which an - * interrupt designed to cancel one task occurs both late and - * unnecessarily. Upon pool shutdown, the task is cancelled, and - * runners are interrupted so they can cancel. Since external - * joining callers never run these tasks, they must await - * cancellation. - * - * See the ForkJoinTask internal documentation for an account of - * how these correspond to how and when different exceptions are - * thrown. + * ExecutorServiceTasks. To conform to ExecutorService specs and + * expectations, externally invoked ExecutorService API methods + * use ExecutorServiceTasks. External submitters do not help run + * such tasks. These tasks include a "runner" field (similarly to + * FutureTask) to support cancel(true). We minimize possibilities + * of "stray" interrupts such as those in which an interrupt + * designed to cancel one task occurs late and unnecessarily. Upon + * pool shutdown, runners are interrupted so they can cancel. + * Since external joining callers never run these tasks, they must + * await cancellation by others. See the ForkJoinTask internal + * documentation for an account of how these correspond to how and + * when different exceptions are thrown. * * Memory placement * ================ @@ -821,6 +857,11 @@ public class ForkJoinPool extends AbstractExecutorService { * few unusual loop constructions encourage (with varying * effectiveness) JVMs about where (not) to place safepoints. * + * Even though the current version of this class initializes + * minimal subcomponents (queue array and lock), nearly all code + * (inefficiently) tolerates alternatives, including use of some + * further useless-looking checks. + * * There is a lot of representation-level coupling among classes * ForkJoinPool, ForkJoinWorkerThread, and ForkJoinTask. The * fields of WorkQueue maintain data structures managed by @@ -850,10 +891,14 @@ public class ForkJoinPool extends AbstractExecutorService { * * The main sources of differences from previous version are: * - * * Handling of Interruptible tasks is now consistent with - * ExecutorService specs, including uniform use of revised - * canStop for quiescence-related checks and corresponding - * revisions to termination checks + * Handling of tasks submitted under the ExecutorService API is + * now consistent with specs. Method checkQuiescence() replaces + * previous quiescence-related checks. It relies on new class + * SeqLock replacing a ReentrantLock. Plus corresponding + * revisions to termination, requiring use of CountDownLatch + * instead of a Condition for termination, and many other + * adjustments to accommodate. + * */ // static configuration constants @@ -890,20 +935,20 @@ public class ForkJoinPool extends AbstractExecutorService { static final int SMASK = 0xffff; // short bits == max index static final int MAX_CAP = 0x7fff; // max #workers - 1 - // pool.runState and workQueue.access bits and sentinels + // pool.runState bits static final int STOP = 1 << 31; // must be negative static final int SHUTDOWN = 1; static final int TERMINATED = 2; - static final int PARKED = -1; // access value when parked + static final int PARKED = -1; // negative source when parked - // {pool, workQueue}.config bits + // {pool, workQueue} bits and sentinels + static final int SRC = 1; // offset for source fields static final int FIFO = 1 << 16; // fifo queue or access mode - static final int SRC = 1 << 17; // set when stealable + static final int ALIVE = 1 << 17; // set if initialized worker static final int CLEAR_TLS = 1 << 18; // set for Innocuous workers static final int TRIMMED = 1 << 19; // timed out while idle static final int ISCOMMON = 1 << 20; // set for common pool static final int PRESET_SIZE = 1 << 21; // size was set by property - static final int UNCOMPENSATE = 1 << 16; // tryCompensate return /* @@ -1054,13 +1099,39 @@ public ForkJoinWorkerThread run() { } } + /** + * Simple AQS-based Sequence Lock, used instead of ReentrantLock + * to enable detection of possible queue array changes and + * upgrades from seq-tracking reader to writer + */ + static final class SeqLock extends AbstractQueuedSynchronizer { + public boolean isHeldExclusively() { + return (getState() & 1) != 0; // odd if locked + } + public boolean tryAcquire(int ignore) { + int s = getState(); + return ((s & 1) == 0 && compareAndSetState(s, s + 1)); + } + public boolean tryRelease(int ignore) { + setState(getState() + 1); + return true; + } + public boolean tryUpgrade(int s) { // s must be even + return compareAndSetState(s, s + 1); // from reader to writer + } + public void lock() { acquire(1); } + public void unlock() { release(1); } + public int seq() { return getState(); } + private static final long serialVersionUID = 2838392045355241008L; + } + /** * Queues supporting work-stealing as well as external task * submission. See above for descriptions and algorithms. */ static final class WorkQueue { int stackPred; // pool stack (ctl) predecessor link - int config; // index, mode, ORed with SRC after init + int config; // index, mode, ORed with ALIVE after init int base; // index of next slot for poll ForkJoinTask[] array; // the queued tasks; power of 2 size final ForkJoinWorkerThread owner; // owning thread or null if shared @@ -1069,18 +1140,17 @@ static final class WorkQueue { @jdk.internal.vm.annotation.Contended("w") int top; // index of next slot for push @jdk.internal.vm.annotation.Contended("w") - volatile int access; // values 0, 1 (locked), PARKED, STOP + volatile int access; // values 0, 1 (locked) @jdk.internal.vm.annotation.Contended("w") volatile int phase; // versioned, negative if inactive @jdk.internal.vm.annotation.Contended("w") - volatile int source; // source queue id in topLevelExec + volatile int source; // source queue id + SRC, or < 0 if parked @jdk.internal.vm.annotation.Contended("w") int nsteals; // number of steals from other queues // Support for atomic operations private static final Unsafe U; private static final long ACCESS; - private static final long PHASE; private static final long ABASE; private static final int ASHIFT; @@ -1096,12 +1166,9 @@ static boolean casSlotToNull(ForkJoinTask[] a, int i, final int getAndSetAccess(int v) { return U.getAndSetInt(this, ACCESS, v); } - final void releaseAccess() { - U.putIntRelease(this, ACCESS, 0); - } /** - * Constructor. For owned queues, most fields are initialized + * Constructor. For internal queues, most fields are initialized * upon thread start in pool.registerWorker. */ WorkQueue(ForkJoinWorkerThread owner, int config) { @@ -1151,16 +1218,17 @@ final void push(ForkJoinTask task, ForkJoinPool pool, "Queue capacity exceeded"); } if (newCap > 0) { // always true - int newMask = newCap - 1, k = s; + int newMask = newCap - 1, k = s, remaining = cap; do { // poll old, push to new newArray[k-- & newMask] = task; - } while ((task = getAndClearSlot(a, k & m)) != null); + } while (remaining-- >= 0 && // exit when lose to pollers + (task = getAndClearSlot(a, k & m)) != null); } array = newArray; } else a[m & s] = task; - getAndSetAccess(0); // for memory effects if owned + getAndSetAccess(0); // for memory effects if internal if ((resize || (a[m & (s - 1)] == null && signalIfEmpty)) && pool != null) pool.signalWork(); @@ -1201,7 +1269,7 @@ else if ((t = getAndClearSlot(a, (cap - 1) & b)) != null) { /** * Takes next task, if one exists, using configured mode. - * (Always owned, never called for Common pool.) + * (Always internal, never called for Common pool.) */ final ForkJoinTask nextLocalTask() { return nextLocalTask(config & FIFO); @@ -1210,12 +1278,12 @@ final ForkJoinTask nextLocalTask() { /** * Pops the given task only if it is at the current top. */ - final boolean tryUnpush(ForkJoinTask task, boolean owned) { + final boolean tryUnpush(ForkJoinTask task, boolean internal) { ForkJoinTask[] a = array; int p = top, s, cap, k; if (task != null && base != p && a != null && (cap = a.length) > 0 && a[k = (cap - 1) & (s = p - 1)] == task) { - if (owned || getAndSetAccess(1) == 0) { + if (internal || getAndSetAccess(1) == 0) { if (top != p || a[k] != task || getAndClearSlot(a, k) == null) access = 0; @@ -1274,7 +1342,7 @@ else if (t != null && casSlotToNull(a, k, t)) { } else if (array != a || a[k] != null) ; // stale - else if (a[nk] == null && access <= 0 && top - b <= 0) + else if (a[nk] == null && top - b <= 0 && access == 0) break; // empty } return null; @@ -1294,16 +1362,13 @@ final ForkJoinTask tryPoll() { U.loadFence(); // for re-reads if (b != (b = base)) ; // inconsistent - else if (t != null) { - if (casSlotToNull(a, k, t)) { - base = nb; - U.storeStoreFence(); - return t; - } - break; // contended + else if (t != null && casSlotToNull(a, k, t)) { + base = nb; + U.storeStoreFence(); + return t; } else if (a[k] == null) - break; // empty or stalled + break; // contended, empty or stalled } } return null; @@ -1332,34 +1397,31 @@ final void topLevelExec(ForkJoinTask task, WorkQueue src) { /** * Deep form of tryUnpush: Traverses from top and removes and - * runs task if present, shifting others to fill gap. + * runs task if present. * @return task status if removed, else 0 */ - final int tryRemoveAndExec(ForkJoinTask task, boolean owned) { + final int tryRemoveAndExec(ForkJoinTask task, boolean internal) { ForkJoinTask[] a = array; int p = top, s = p - 1, d = p - base, cap; if (task != null && d > 0 && a != null && (cap = a.length) > 0) { for (int m = cap - 1, i = s; ; --i) { ForkJoinTask t; int k; if ((t = a[k = i & m]) == task) { - if (!owned && getAndSetAccess(1) != 0) - break; // fail if locked - else if (top != p || a[k] != task || - getAndClearSlot(a, k) == null) { + if (!internal && getAndSetAccess(1) != 0) + break; // fail if locked + if (top != p || a[k] != task || + getAndClearSlot(a, k) == null) { access = 0; - break; // missed - } - else { - if (i != s && i == base) - base = i + 1; // avoid shift - else { - for (int j = i; j != s;) // shift down - a[j & m] = getAndClearSlot(a, ++j & m); - top = s; - } - releaseAccess(); - return task.doExec(); + break; // missed } + if (i == s) // act as pop + top = s; + else if (i == base) // act as poll + base = i + 1; + else // reorder by swapping top + a[k] = getAndClearSlot(a, (top = s) & m); + access = 0; + return task.doExec(); } else if (t == null || --d == 0) break; @@ -1376,7 +1438,7 @@ else if (t == null || --d == 0) * @param limit max runs, or zero for no limit * @return task status on exit */ - final int helpComplete(ForkJoinTask task, boolean owned, int limit) { + final int helpComplete(ForkJoinTask task, boolean internal, int limit) { int status = 0; if (task != null) { outer: for (;;) { @@ -1394,14 +1456,14 @@ final int helpComplete(ForkJoinTask task, boolean owned, int limit) { else if ((f = f.completer) == null) break outer; // ineligible } - if (!owned && getAndSetAccess(1) != 0) + if (!internal && getAndSetAccess(1) != 0) break; // fail if locked if (top != p || a[k] != t || getAndClearSlot(a, k) == null) { access = 0; break; // missed } top = s; - releaseAccess(); + access = 0; t.doExec(); if (limit != 0 && --limit == 0) break; @@ -1451,11 +1513,25 @@ else if (a[nk] == null) // misc /** - * Returns true if owned by a worker thread and not known to be blocked. + * Cancels given task if nonnull and any other local tasks. + */ + final void cancelRemainingTasks(ForkJoinTask t) { + do { + if (t != null) { + try { + t.cancel(false); + } catch (Throwable ignore) { + } + } + } while ((t = poll(null)) != null); + } + + /** + * Returns true if internal and not known to be blocked. */ final boolean isApparentlyUnblocked() { Thread wt; Thread.State s; - return (access != STOP && (wt = owner) != null && + return ((config & ALIVE) != 0 && (wt = owner) != null && (s = wt.getState()) != Thread.State.BLOCKED && s != Thread.State.WAITING && s != Thread.State.TIMED_WAITING); @@ -1472,7 +1548,6 @@ final void setClearThreadLocals() { U = Unsafe.getUnsafe(); Class klass = WorkQueue.class; ACCESS = U.objectFieldOffset(klass, "access"); - PHASE = U.objectFieldOffset(klass, "phase"); Class aklass = ForkJoinTask[].class; ABASE = U.arrayBaseOffset(aklass); int scale = U.arrayIndexScale(aklass); @@ -1519,8 +1594,8 @@ final void setClearThreadLocals() { final int config; // static configuration bits volatile int runState; // SHUTDOWN, STOP, TERMINATED bits WorkQueue[] queues; // main registry - final ReentrantLock registrationLock; - Condition termination; // lazily constructed + final SeqLock registrationLock; + volatile CountDownLatch termination; // lazily constructed final String workerNamePrefix; // null for common pool final ForkJoinWorkerThreadFactory factory; final UncaughtExceptionHandler ueh; // per-worker UEH @@ -1538,6 +1613,7 @@ final void setClearThreadLocals() { private static final long RUNSTATE; private static final long PARALLELISM; private static final long THREADIDS; + private static final long TERMINATION; private static final Object POOLIDS_BASE; private static final long POOLIDS; @@ -1565,6 +1641,9 @@ private int getAndSetParallelism(int v) { private int getParallelismOpaque() { return U.getIntOpaque(this, PARALLELISM); } + private boolean casTerminationSignal(CountDownLatch x) { + return U.compareAndSetReference(this, TERMINATION, null, x); + } // Creating, registering, and deregistering workers @@ -1577,12 +1656,16 @@ private int getParallelismOpaque() { */ private boolean createWorker() { ForkJoinWorkerThreadFactory fac = factory; + SharedThreadContainer ctr = container; Throwable ex = null; ForkJoinWorkerThread wt = null; try { if (runState >= 0 && // avoid construction if terminating fac != null && (wt = fac.newThread(this)) != null) { - container.start(wt); + if (ctr != null) + ctr.start(wt); + else + wt.start(); return true; } } catch (Throwable rex) { @@ -1604,32 +1687,32 @@ final String nextWorkerThreadName() { } /** - * Finishes initializing and records owned queue. + * Finishes initializing and records internal queue. * * @param w caller's WorkQueue */ final void registerWorker(WorkQueue w) { ThreadLocalRandom.localInit(); int seed = ThreadLocalRandom.getProbe(); - ReentrantLock lock = registrationLock; + SeqLock lock = registrationLock; int cfg = config & FIFO; if (w != null && lock != null) { w.array = new ForkJoinTask[INITIAL_QUEUE_CAPACITY]; - cfg |= w.config | SRC; + cfg |= w.config; w.stackPred = seed; - int seq = (seed ^ (seed >>> 16)) << 16; // initial phase seq + int seq = seed << 16; // initial phase seq int id = (seed << 1) | 1; // initial index guess lock.lock(); try { WorkQueue[] qs; int n; // find queue index - if ((qs = queues) != null && (n = qs.length) > 0) { + if (runState >= 0 && (qs = queues) != null && + (n = qs.length) > 0) { // skip if terminating int k = n, m = n - 1; for (; qs[id &= m] != null && k > 0; id -= 2, k -= 2); if (k == 0) id = n | 1; // resize below - w.config = id | cfg; + w.config = id | cfg | ALIVE; w.phase = (id | seq) & ~INACTIVE; // now publishable - if (id < n) qs[id] = w; else { // expand array @@ -1663,13 +1746,28 @@ final void registerWorker(WorkQueue w) { * @param ex the exception causing failure, or null if none */ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { - int cfg = 0; + int cfg; SeqLock lock; WorkQueue w = (wt == null) ? null : wt.workQueue; - if (w != null) { - cfg = w.config; - w.access = STOP; // mark as done - do {} while (w.phase < 0 && // ensure released - reactivate() != null); + if (w == null) + cfg = 0; + else if (((cfg = w.config) & ALIVE) != 0) { + w.config = cfg & ~ALIVE; + if (w.phase < 0) // possible during termination + reactivate(w); + if (w.top - w.base > 0) + w.cancelRemainingTasks(null); + if (runState >= 0 && (lock = registrationLock) != null) { + WorkQueue[] qs; int n, i; // remove index unless terminating + long ns = w.nsteals & 0xffffffffL; + lock.lock(); + if (runState >= 0) { // recheck + if ((qs = queues) != null && (n = qs.length) > 0 && + qs[i = cfg & (n - 1)] == w) + qs[i] = null; + stealCount += ns; // accumulate steals + } + lock.unlock(); + } } long c = ctl; if ((cfg & TRIMMED) == 0) // decrement counts @@ -1678,23 +1776,9 @@ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { (TC_MASK & (c - TC_UNIT)) | (SP_MASK & c))))); else if ((int)c == 0) // was dropped on timeout - cfg &= ~SRC; // suppress signal if last - if (tryTerminate(false, false) >= 0 && w != null) { - ReentrantLock lock; WorkQueue[] qs; int n, i; - long ns = w.nsteals & 0xffffffffL; - if ((lock = registrationLock) != null) { - lock.lock(); // remove index unless terminating - if ((qs = queues) != null && (n = qs.length) > 0 && - qs[i = cfg & (n - 1)] == w) - qs[i] = null; - stealCount += ns; // accumulate steals - lock.unlock(); - } - for (ForkJoinTask t; (t = w.nextLocalTask(0)) != null; ) - ForkJoinTask.cancelIgnoringExceptions(t); - if ((cfg & SRC) != 0) - signalWork(); // possibly replace worker - } + cfg &= ~ALIVE; // suppress replacement if last + if (tryTerminate(false, false) >= 0 && (cfg & ALIVE) != 0) + signalWork(); // replace unless terminating, uninitialized or trimmed if (ex != null) ForkJoinTask.rethrow(ex); } @@ -1727,7 +1811,7 @@ else if (deficit <= 0) else { Thread owner = v.owner; v.phase = sp; - if (v.access == PARKED) + if (v.source < 0) LockSupport.unpark(owner); } break; @@ -1737,11 +1821,10 @@ else if (deficit <= 0) } /** - * Reactivates any idle worker, if one exists. - * - * @return the signalled worker, or null if none + * Reactivates the given worker (and possibly others if not top of + * ctl stack), or any worker if null and idle. */ - private WorkQueue reactivate() { + private void reactivate(WorkQueue w) { WorkQueue[] qs; int n; long c = ctl; if ((qs = queues) != null && (n = qs.length) > 0) { @@ -1749,19 +1832,21 @@ private WorkQueue reactivate() { int sp = (int)c & ~INACTIVE; WorkQueue v = qs[sp & (n - 1)]; long ac = UC_MASK & (c + RC_UNIT); - if (sp == 0 || v == null) + if (sp == 0 || v == null || + ((w == null) ? ((c & RC_MASK) > 0L) : (w.phase >= 0))) break; if (c == (c = compareAndExchangeCtl( c, (v.stackPred & SP_MASK) | ac))) { Thread owner = v.owner; v.phase = sp; - if (v.access == PARKED) + if (v.source < 0) LockSupport.unpark(owner); - return v; + if (v == w || w == null) + break; + c = ctl; } } } - return null; } /** @@ -1783,6 +1868,60 @@ private boolean tryTrim(WorkQueue w) { return false; } + /** + * Internal version of isQuiescent and related functionality. + * Returns true if terminating or submission queues are empty and + * unlocked, and all workers are inactive. If so, and quiescent + * shutdown is enabled, starts termination. + * + * @param canStop if can start terminating if quiescent + */ + private boolean checkQuiescence(boolean canStop) { + SeqLock lock; + boolean scanned = false; + if ((lock = registrationLock) != null) { // else uninitialized + for (int seq = 0, prevSeq = 1; ; prevSeq = seq) { + WorkQueue[] qs; int n, rs; + if ((rs = runState) < 0) // terminating + break; + if (canStop) { + if ((ctl & RC_MASK) > 0L) + return false; // active workers exist + if (rs != 0 && scanned) { + if (!lock.tryUpgrade(seq)) // trigger STOP under lock + scanned = false; // retry if lock held + else { + if ((ctl & RC_MASK) > 0L) // recheck + scanned = false; + else + getAndBitwiseOrRunState(STOP); + lock.unlock(); + } + } + } + if (scanned) + break; + if ((qs = queues) != null && (n = qs.length) > 0) { + for (int i = 0;;) { + WorkQueue q, w; // alternate external, internal + if ((q = qs[i]) != null && // is external + (q.top != q.base || q.access != 0)) + return false; // possibly nonempty + if (++i == n) + break; + if ((w = qs[i]) != null && w.phase > 0) + return false; // active worker + if (++i == n) + break; + } + } + if ((seq = lock.seq()) == prevSeq && (seq & 1) == 0) + scanned = true; // same unlocked q state + } + } + return true; + } + /** * Top-level runloop for workers, called by ForkJoinWorkerThread.run. * See above for explanation. @@ -1817,7 +1956,7 @@ private int scan(WorkQueue w, int prevSrc, int r) { int j, cap; WorkQueue q; ForkJoinTask[] a; if ((q = qs[j = r & (n - 1)]) != null && (a = q.array) != null && (cap = a.length) > 0) { - int src = j | SRC, b = q.base; + int src = j + SRC, b = q.base; int k = (cap - 1) & b, nb = b + 1, nk = (cap - 1) & nb; ForkJoinTask t = a[k]; U.loadFence(); // for re-reads @@ -1826,11 +1965,11 @@ private int scan(WorkQueue w, int prevSrc, int r) { else if (t != null && WorkQueue.casSlotToNull(a, k, t)) { q.base = nb; w.source = src; - if (src + (src << SWIDTH) != prevSrc && - q.base == nb && a[nk] != null) - signalWork(); // propagate at most twice/run + int nextSrc = ((prevSrc << 16) | src) & 0x7fffffff; + if (nextSrc != prevSrc && q.base == nb && a[nk] != null) + signalWork(); // propagate at most twice/run w.topLevelExec(t, q); - return src + (prevSrc << SWIDTH); + return nextSrc; } else if (q.array != a || a[k] != null || a[nk] != null) return prevSrc; // revisit @@ -1848,36 +1987,38 @@ private int awaitWork(WorkQueue w) { if (w == null) return -1; // currently impossible int p = (w.phase + SS_SEQ) & ~INACTIVE; // advance phase - boolean idle; // true if possibly quiescent if (runState < 0) return -1; // terminating - long sp = p & SP_MASK, pc = ctl, qc; + boolean idle = false; // true if pool idle + int spins = (parallelism << 2) + 0xf; // avg #accesses in scan+signal + long pc = ctl, sp = p & SP_MASK, qc; w.phase = p | INACTIVE; do { // enqueue w.stackPred = (int)pc; // set ctl stack link } while (pc != (pc = compareAndExchangeCtl( pc, qc = ((pc - RC_UNIT) & UC_MASK) | sp))); - if ((idle = ((qc & RC_MASK) <= 0L)) && // check for stragglers - !canStop(true, true) && (w.phase >= 0 || reactivate() == w)) - return 0; // rescan or terminate - WorkQueue[] qs = queues; // spin for expected #accesses in scan+signal - int spins = ((qs == null) ? 0 : ((qs.length & SMASK) << 1)) | 0xf; + if ((qc & RC_MASK) <= 0L) { // possibly quiescent + if (!(idle = checkQuiescence(true))) + reactivate(null); // ensure live + else if (runState < 0) + return -1; // quiescently terminating + } while ((p = w.phase) < 0 && --spins > 0) - Thread.onSpinWait(); + Thread.onSpinWait(); // spin before block if (p < 0) { long deadline = idle ? keepAlive + System.currentTimeMillis() : 0L; LockSupport.setCurrentBlocker(this); for (;;) { // await signal or termination if (runState < 0) - break; - w.access = PARKED; // enable unpark + return -1; + w.source = PARKED; // enable unpark if (w.phase < 0) { if (idle) LockSupport.parkUntil(deadline); else LockSupport.park(); } - w.access = 0; // disable unpark + w.source = 0; // disable unpark if (w.phase >= 0) { LockSupport.setCurrentBlocker(null); break; @@ -1896,49 +2037,6 @@ private int awaitWork(WorkQueue w) { return 0; } - /** - * Internal version of isQuiescent and related functionality. - * Returns true if terminating or all submission queues are empty - * and unlocked, and all workers are idle. - * - * @param idle false if caller need not be counted as idle. - * @param stopIfEnabled true if should transition runState to STOP - * if quiescent, in which case returns false - */ - private boolean canStop(boolean idle, boolean stopIfEnabled) { - long prevSum = 0L; // at least 2 scans - for (boolean stable = false, rescan = true;;) { - if (runState < 0) - return true; // terminating - else if (idle && (ctl & RC_MASK) > 0L) - return false; // active workers exist - else if (stable) { - if (stopIfEnabled && (runState & SHUTDOWN) != 0) { - getAndBitwiseOrRunState(STOP); - return false; // caller cannot stop yet - } - return true; - } - long sum = 0L; - WorkQueue[] qs; WorkQueue q; - int n = ((qs = queues) == null) ? 0 : qs.length; - for (int i = 0; i < n; ++i) { - if ((q = qs[i]) != null) { - int x = q.access, p = q.phase, s = q.top, d = s - q.base; - if (x > 0 || p > 0 || d > 0 || q.access > 0) - return false; // locked, active or nonempty - if (d != 0 || q.top != s) - rescan = true; // inconsistent reads - sum = (sum << 5) - sum + s + p; // sequence hash - } - } - if (!rescan && sum == prevSum) - stable = true; - prevSum = sum; - rescan = false; - } - } - /** * Scans for and returns a polled task, if available. Used only * for untracked polls. Begins scan at a random index to avoid @@ -1976,14 +2074,16 @@ private ForkJoinTask pollScan(boolean submissionsOnly) { */ private int tryCompensate(long c, boolean canSaturate) { Predicate sat; - long b = bounds; // unpack fields + long b = bounds; // unpack fields int pc = parallelism; int minActive = (short)(b & SMASK), maxTotal = (short)(b >>> SWIDTH) + pc, active = (short)(c >>> RC_SHIFT), total = (short)(c >>> TC_SHIFT), sp = (int)c & ~INACTIVE; - if (sp != 0 && active <= pc) { // activate idle worker + if (runState < 0) // stopping + return 0; + else if (sp != 0 && active <= pc) { // activate idle worker WorkQueue[] qs; WorkQueue v; int i; if (ctl == c && (qs = queues) != null && qs.length > (i = sp & SMASK) && (v = qs[i]) != null) { @@ -2035,35 +2135,39 @@ final void uncompensate() { final int helpJoin(ForkJoinTask task, WorkQueue w, boolean timed) { if (w == null || task == null) return 0; - int wsrc = w.source, wid = (w.config & SMASK) | SRC, r = wid + 2; + int wsrc = w.source, cfg = (w.config & SMASK); + int wid = cfg + SRC, r = cfg + 2; long sctl = 0L; // track stability for (boolean rescan = true;;) { int s; WorkQueue[] qs; if ((s = task.status) < 0) return s; if (!rescan && sctl == (sctl = ctl)) { - if (runState < 0) - return 0; if ((s = tryCompensate(sctl, timed)) >= 0) - return s; // block + return s; // block } - rescan = false; + else if (runState < 0) + return 0; int n = ((qs = queues) == null) ? 0 : qs.length, m = n - 1; + rescan = false; scan: for (int i = n >>> 1; i > 0; --i, r += 2) { int j, cap; WorkQueue q; ForkJoinTask[] a; if ((q = qs[j = r & m]) != null && (a = q.array) != null && (cap = a.length) > 0) { - for (int src = j | SRC;;) { + for (int src = j + SRC;;) { int sq = q.source, b = q.base; int k = (cap - 1) & b, nb = b + 1; ForkJoinTask t = a[k]; U.loadFence(); // for re-reads - boolean eligible = true; // check steal chain + boolean eligible; // check steal chain for (int d = n, v = sq;;) { // may be cyclic; bound WorkQueue p; - if (v == wid) + if (v == wid) { + eligible = true; break; - if (v == 0 || --d == 0 || (p = qs[v & m]) == null) { + } + if (v <= 0 || --d == 0 || + (p = qs[(v - SRC) & m]) == null) { eligible = false; break; } @@ -2102,33 +2206,35 @@ else if (WorkQueue.casSlotToNull(a, k, t)) { * * @param task the task * @param w caller's WorkQueue - * @param owned true if w is owned by a ForkJoinWorkerThread + * @param internal true if w is owned by a ForkJoinWorkerThread * @param timed true if this is a timed join * @return task status on exit, or UNCOMPENSATE for compensated blocking */ - final int helpComplete(ForkJoinTask task, WorkQueue w, boolean owned, + final int helpComplete(ForkJoinTask task, WorkQueue w, boolean internal, boolean timed) { if (w == null || task == null) return 0; - int wsrc = w.source, r = w.config; + int wsrc = w.source, r = w.config + 1; long sctl = 0L; // track stability for (boolean rescan = true;;) { int s; WorkQueue[] qs; - if ((s = w.helpComplete(task, owned, 0)) < 0) + if ((s = w.helpComplete(task, internal, 0)) < 0) return s; if (!rescan && sctl == (sctl = ctl)) { - if (!owned || runState < 0) + if (!internal) return 0; if ((s = tryCompensate(sctl, timed)) >= 0) return s; } - rescan = false; + else if (runState < 0) + return 0; int n = ((qs = queues) == null) ? 0 : qs.length, m = n - 1; + rescan = false; scan: for (int i = n; i > 0; --i, ++r) { int j, cap; WorkQueue q; ForkJoinTask[] a; if ((q = qs[j = r & m]) != null && (a = q.array) != null && (cap = a.length) > 0) { - poll: for (int src = j | SRC, b = q.base;;) { + poll: for (int src = j + SRC, b = q.base;;) { int k = (cap - 1) & b, nb = b + 1; ForkJoinTask t = a[k]; U.loadFence(); // for re-reads @@ -2139,7 +2245,8 @@ else if ((s = task.status) < 0) else if (t == null) { if (a[k] == null) { if (!rescan && // resized or stalled - (q.array != a || q.top != b)) + (q.array != a || + q.top != b || q.access != 0)) rescan = true; break; } @@ -2170,22 +2277,21 @@ else if ((f = f.completer) == null) } /** - * Runs tasks until {@code isQuiescent()}. Rather than blocking - * when tasks cannot be found, rescans until all others cannot - * find tasks either. + * Runs tasks until all workerss are inactive and no tasks are + * found. Rather than blocking when tasks cannot be found, rescans + * until all others cannot find tasks either. * * @param nanos max wait time (Long.MAX_VALUE if effectively untimed) * @param interruptible true if return on interrupt * @return positive if quiescent, negative if interrupted, else 0 */ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { - long startTime = System.nanoTime(); + long startTime = System.nanoTime(), parkTime = 0L; int phase; // w.phase set negative when temporarily quiescent if (w == null || (phase = w.phase) < 0) return 0; int activePhase = phase, inactivePhase = phase | INACTIVE; - int wsrc = w.source, r = 0, returnStatus = 1; - long initialSleep = -parallelism, parkTime = 0L; + int wsrc = w.source, r = w.config + 1, returnStatus = 1; for (boolean locals = true;;) { WorkQueue[] qs; WorkQueue q; if (runState < 0) @@ -2202,8 +2308,8 @@ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { boolean rescan = false; int n = ((qs = queues) == null) ? 0 : qs.length, m = n - 1; scan: for (int i = n, j; i > 0; --i, ++r) { - if ((q = qs[j = m & r]) != null && q != w) { - for (int src = j | SRC;;) { + if ((q = qs[j = m & r]) != null) { + for (int src = j + SRC;;) { ForkJoinTask[] a = q.array; int b = q.base, cap; if (a == null || (cap = a.length) <= 0) @@ -2214,7 +2320,7 @@ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { if (q.base != b || q.array != a || a[k] != t) ; else if (t == null) { - if (!rescan && (q.access > 0 || q.top - b > 0)) + if (!rescan && (q.top != b || q.access != 0)) rescan = true; break; } @@ -2234,16 +2340,18 @@ else if (WorkQueue.casSlotToNull(a, k, t)) { if (!rescan) { if (phase >= 0) { // tentatively inactivate w.phase = phase = inactivePhase; - parkTime = initialSleep; + parkTime = -2L; // to recheck, then yield, then sleep } - if (canStop(false, false)) - break; // quiescent + if (checkQuiescence(false)) + break; else if (System.nanoTime() - startTime > nanos) { returnStatus = 0; // timed out break; } - else if (parkTime <= 0L) // spin before sleep - ++parkTime; + else if (parkTime < 0L) { // recheck or yield + if (++parkTime == 0L) + Thread.yield(); + } else { // initial sleep approx 1 usec parkTime = Math.max(parkTime, 1L << 10); LockSupport.parkNanos(this, parkTime); @@ -2264,24 +2372,25 @@ else if (parkTime <= 0L) // spin before sleep * @return positive if quiescent, negative if interrupted, else 0 */ private int externalHelpQuiesce(long nanos, boolean interruptible) { - long startTime = System.nanoTime(); - long initialSleep = -parallelism, parkTime = 0L; - for (;;) { // same structure as helpQuiesce + long startTime = System.nanoTime(), parkTime = 0L; + for (;;) { // same structure as helpQuiesce, using poll vs scanning ForkJoinTask t; if (runState < 0) return 1; else if (interruptible && Thread.interrupted()) return -1; else if ((t = pollScan(false)) != null) { - parkTime = initialSleep; + parkTime = -2L; t.doExec(); } - else if (canStop(true, false)) + else if (checkQuiescence(true)) return 1; else if (System.nanoTime() - startTime > nanos) return 0; - else if (parkTime <= 0L) - ++parkTime; + else if (parkTime < 0L) { + if (++parkTime == 0L) + Thread.yield(); + } else { parkTime = Math.max(parkTime, 1L << 10); LockSupport.parkNanos(this, parkTime); @@ -2333,24 +2442,25 @@ final ForkJoinTask nextTaskFor(WorkQueue w) { */ final WorkQueue submissionQueue(boolean isSubmit) { int r; - ReentrantLock lock = registrationLock; + SeqLock lock = registrationLock; if ((r = ThreadLocalRandom.getProbe()) == 0) { ThreadLocalRandom.localInit(); // initialize caller's probe r = ThreadLocalRandom.getProbe(); } if (lock != null) { // else init error for (int id = r << 1;;) { // even indices only - int n, i; WorkQueue[] qs; WorkQueue q; + int n, i, rs; WorkQueue[] qs; WorkQueue q; if ((qs = queues) == null || (n = qs.length) <= 0) break; else if ((q = qs[i = (n - 1) & id]) == null) { - WorkQueue w = new WorkQueue(null, id | SRC); - w.phase = (id >>> 1) | INACTIVE; // for use as unique id + WorkQueue w = new WorkQueue(null, id); w.array = new ForkJoinTask[INITIAL_QUEUE_CAPACITY]; lock.lock(); // install under lock - if (queues == qs && qs[i] == null) + if ((rs = runState) >= 0 && queues == qs && qs[i] == null) qs[i] = w; // else lost race; discard lock.unlock(); + if (rs < 0) + break; // terminating } else if (q.getAndSetAccess(1) != 0) // move and restart id = (r = ThreadLocalRandom.advanceProbe(r)) << 1; @@ -2378,7 +2488,6 @@ private ForkJoinTask poolSubmit(boolean signalIfEmpty, (wt = (ForkJoinWorkerThread)t).pool == this) q = wt.workQueue; else { - task.markPoolSubmission(); q = submissionQueue(true); } q.push(task, this, signalIfEmpty); @@ -2500,64 +2609,62 @@ static int getSurplusQueuedTaskCount() { */ private int tryTerminate(boolean now, boolean enable) { int rs = runState; - if ((config & ISCOMMON) == 0 && rs >= 0) { - if (now) - rs = getAndBitwiseOrRunState(SHUTDOWN | STOP) | STOP; + SeqLock lock = registrationLock; + if ((config & ISCOMMON) == 0 && lock != null && rs >= 0) { + if (now) { + lock.lock(); // transition under lock + rs = getAndBitwiseOrRunState(STOP | SHUTDOWN) | STOP; + lock.unlock(); + } else { if (enable && (rs & SHUTDOWN) == 0) rs = getAndBitwiseOrRunState(SHUTDOWN) | SHUTDOWN; if ((rs & SHUTDOWN) != 0) { - canStop(true, true); // sets STOP if quiescent + checkQuiescence(true); // sets runState rs = runState; } } } - if ((rs & (STOP | TERMINATED)) != STOP) // else help terminate - return rs; - reactivate(); // try to fan out - Thread current = Thread.currentThread(); - WorkQueue w = ((current instanceof ForkJoinWorkerThread) ? - ((ForkJoinWorkerThread)current).workQueue : null); - int r = (w != null) ? w.config : 0; // stagger traversals - // loop until all queues empty and all workers stopped or interrupted - long prevSum = 0L; // similar to canStop - for (boolean stable = false, rescan = true;;) { - if (((rs = runState) & TERMINATED) != 0) - return rs; - if (stable) { - ReentrantLock lock; Condition cond; // transition if no workers - if ((short)(ctl >>> TC_SHIFT) <= 0 && - (getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0 && - (lock = registrationLock) != null) { - lock.lock(); - if ((cond = termination) != null) - cond.signalAll(); - lock.unlock(); - container.close(); + if ((rs & (STOP | TERMINATED)) == STOP) { // help terminate + WorkQueue[] qs; + Thread current = Thread.currentThread(); + WorkQueue w = ((current instanceof ForkJoinWorkerThread) ? + ((ForkJoinWorkerThread)current).workQueue : null); + int r = (w != null) ? w.config : 0; // stagger traversals + int n = ((qs = queues) == null) ? 0 : qs.length, m = n - 1; + for (int i = n; i > 0; --i, ++r) { + WorkQueue q; Thread t; + if ((q = qs[m & r]) != null) { + if ((t = q.owner) != null && !t.isInterrupted() && + ((q.config) & ALIVE) != 0) // try to unblock + ForkJoinTask.interruptIgnoringExceptions(t); + q.cancelRemainingTasks(null); } - return runState; } - long sum = 0L; - WorkQueue[] qs = queues; - int n = (qs == null) ? 0 : qs.length; - for (int i = 0; i < n; ++i) { - WorkQueue q; Thread qt; - if ((q = qs[(r + i) & (n - 1)]) != null) { - for (ForkJoinTask f; (f = q.poll(null)) != null; ) - ForkJoinTask.cancelIgnoringExceptions(f); - if ((qt = q.owner) != null && q.access != STOP && - !qt.isInterrupted()) { - rescan = true; - ForkJoinTask.interruptIgnoringExceptions(qt); - } - sum = (sum << 5) - sum + (q.phase & ~INACTIVE); + if (((rs = runState) & TERMINATED) == 0 && ctl == 0L) { + rs |= TERMINATED; // transition + if ((getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0) { + CountDownLatch done; SharedThreadContainer ctr; + if ((done = termination) != null) + done.countDown(); + if ((ctr = container) != null) + ctr.close(); } } - if (!rescan && sum == prevSum) - stable = true; - prevSum = sum; - rescan = false; } + return rs; + } + + /** + * Lazily constructs termination signal + */ + private CountDownLatch terminationSignal() { + CountDownLatch x; + do { + x = termination; + } while (x == null && // OK to throw away in rare cases of CAS failures + !casTerminationSignal(x = new CountDownLatch(1))); + return x; } // Exported methods @@ -2743,7 +2850,7 @@ public ForkJoinPool(int parallelism, this.bounds = (long)(minAvail & SMASK) | (long)(maxSpares << SWIDTH) | ((long)corep << 32); int size = 1 << (33 - Integer.numberOfLeadingZeros(p - 1)); - this.registrationLock = new ReentrantLock(); + this.registrationLock = new SeqLock(); this.queues = new WorkQueue[size]; String pid = Integer.toString(getAndAddPoolIds(1) + 1); String name = "ForkJoinPool-" + pid; @@ -2798,7 +2905,7 @@ private ForkJoinPool(byte forCommonPoolOnly) { this.keepAlive = DEFAULT_KEEPALIVE; this.saturate = null; this.workerNamePrefix = null; - this.registrationLock = new ReentrantLock(); + this.registrationLock = new SeqLock(); this.queues = new WorkQueue[size]; this.container = SharedThreadContainer.create("ForkJoinPool.commonPool"); } @@ -2944,7 +3051,6 @@ public ForkJoinTask submit(Runnable task) { */ public ForkJoinTask externalSubmit(ForkJoinTask task) { U.storeStoreFence(); // ensure safely publishable - task.markPoolSubmission(); WorkQueue q = submissionQueue(true); q.push(task, this, true); return task; @@ -3073,7 +3179,7 @@ public List> invokeAll(Collection> tasks, * multiple calls to tryComplete by the same task under async * cancellation. */ - static final class InvokeAnyRoot extends ForkJoinTask { + static final class InvokeAnyRoot extends ForkJoinTask.ExecutorServiceTask { private static final long serialVersionUID = 2838392045355241008L; @SuppressWarnings("serial") // Conditionally serializable volatile E result; @@ -3102,17 +3208,13 @@ else if (count.getAndDecrement() <= 1 || } } } - public final boolean exec() { return false; } // never forked + public final E compute() { return null; } // never forked public final E getRawResult() { return result; } public final void setRawResult(E v) { result = v; } // Common support for timed and untimed versions of invokeAny final E invokeAny(Collection> tasks, ForkJoinPool pool, boolean timed, long nanos) throws InterruptedException, ExecutionException, TimeoutException { - Thread t = Thread.currentThread(); - if (!(t instanceof ForkJoinWorkerThread) || - ((ForkJoinWorkerThread)t).pool != pool) - markPoolSubmission(); // for exception reporting ArrayList> fs = new ArrayList<>(count.get()); try { for (Callable c : tasks) { @@ -3134,19 +3236,21 @@ final E invokeAny(Collection> tasks, } /** - * AdaptedInterruptibleCallable with results in InvokeAnyRoot (and - * never independently joined). + * ExecutorServiceTask with results in InvokeAnyRoot (and never + * independently joined). */ - static final class InvokeAnyTask extends ForkJoinTask.AdaptedInterruptibleCallable { - private static final long serialVersionUID = 2838392045355241008L; + static final class InvokeAnyTask extends ForkJoinTask.ExecutorServiceTask { final InvokeAnyRoot root; + @SuppressWarnings("serial") // Conditionally serializable + final Callable callable; InvokeAnyTask(InvokeAnyRoot root, Callable callable) { - super(callable); + if (callable == null) throw new NullPointerException(); + this.callable = callable; this.root = root; } - final E compute() throws Exception { - InvokeAnyRoot r; - if ((r = root) != null && !r.isDone()) { + final Void compute() throws Exception { + InvokeAnyRoot r = root; + if (!r.isDone()) { E v = null; Throwable ex = null; boolean completed = false; try { v = callable.call(); @@ -3165,6 +3269,12 @@ public boolean cancel(boolean mayInterruptIfRunning) { r.tryComplete(this, null, null, false); return super.cancel(mayInterruptIfRunning); } + public final Void getRawResult() { return null; } + public final void setRawResult(Void v) { } + public String toString() { + return super.toString() + "[Wrapped task = " + callable + "]"; + } + private static final long serialVersionUID = 2838392045355241008L; } @Override @@ -3290,7 +3400,7 @@ public int getActiveThreadCount() { * @return {@code true} if all threads are currently idle */ public boolean isQuiescent() { - return canStop(true, false); + return checkQuiescence(true); } /** @@ -3512,7 +3622,6 @@ public List shutdownNow() { * @return {@code true} if all tasks have completed following shut down */ public boolean isTerminated() { - // reduce premature negatives during termination by helping return (tryTerminate(false, false) & TERMINATED) != 0; } @@ -3558,30 +3667,19 @@ public boolean isShutdown() { */ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { - ReentrantLock lock; long nanos = unit.toNanos(timeout); + CountDownLatch done; if ((config & ISCOMMON) != 0) { if (helpQuiescePool(this, nanos, true) < 0) throw new InterruptedException(); return false; } - else if ((lock = registrationLock) != null) { - Condition cond = null; - while ((tryTerminate(false, false) & TERMINATED) == 0) { - if (nanos <= 0L) - return false; - lock.lock(); - try { - if (cond == null && (cond = termination) == null) - termination = cond = lock.newCondition(); - if ((runState & TERMINATED) == 0) - nanos = cond.awaitNanos(nanos); - } finally { - lock.unlock(); - } - } - } - return true; + else if ((tryTerminate(false, false) & TERMINATED) != 0 || + (done = terminationSignal()) == null || + (runState & TERMINATED) != 0) + return true; + else + return done.await(nanos, TimeUnit.NANOSECONDS); } /** @@ -3625,23 +3723,20 @@ public boolean awaitQuiescence(long timeout, TimeUnit unit) { */ @Override public void close() { - ReentrantLock lock; - if ((runState & TERMINATED) == 0 && (config & ISCOMMON) == 0 && - (lock = registrationLock) != null) { + if ((runState & TERMINATED) == 0 && (config & ISCOMMON) == 0) { checkPermission(); - Condition cond = null; + CountDownLatch done = null; boolean interrupted = Thread.interrupted(); while ((tryTerminate(interrupted, true) & TERMINATED) == 0) { - lock.lock(); - try { - if (cond == null && (cond = termination) == null) - termination = cond = lock.newCondition(); - if ((runState & TERMINATED) == 0) - cond.await(); - } catch (InterruptedException ex) { - interrupted = true; - } finally { - lock.unlock(); + if (done == null) + done = terminationSignal(); + else { + try { + done.await(); + break; + } catch (InterruptedException ex) { + interrupted = true; + } } } if (interrupted) @@ -3770,8 +3865,6 @@ private void compensatedBlock(ManagedBlocker blocker) long c = ctl; if (blocker.isReleasable()) break; - if (runState < 0) // will be interrupted on cancellation - throw new InterruptedException(); if ((comp = tryCompensate(c, false)) >= 0) { long post = (comp == 0) ? 0L : RC_UNIT; try { @@ -3792,14 +3885,9 @@ private void compensatedBlock(ManagedBlocker blocker) * with the value returned by this method to re-adjust the parallelism. */ private long beginCompensatedBlock() { - for (;;) { - int comp; - if ((comp = tryCompensate(ctl, false)) >= 0) { - return (comp == 0) ? 0L : RC_UNIT; - } else { - Thread.onSpinWait(); - } - } + int comp; + do {} while ((comp = tryCompensate(ctl, false)) < 0); + return (comp == 0) ? 0L : RC_UNIT; } /** @@ -3818,10 +3906,6 @@ private static void unmanagedBlock(ManagedBlocker blocker) do {} while (!blocker.isReleasable() && !blocker.block()); } - // AbstractExecutorService.newTaskFor overrides rely on - // undocumented fact that ForkJoinTask.adapt returns ForkJoinTasks - // that also implement RunnableFuture. - @Override protected RunnableFuture newTaskFor(Runnable runnable, T value) { return new ForkJoinTask.AdaptedInterruptibleRunnable(runnable, value); @@ -3844,8 +3928,9 @@ protected RunnableFuture newTaskFor(Callable callable) { } CTL = U.objectFieldOffset(klass, "ctl"); RUNSTATE = U.objectFieldOffset(klass, "runState"); - PARALLELISM = U.objectFieldOffset(klass, "parallelism"); + PARALLELISM = U.objectFieldOffset(klass, "parallelism"); THREADIDS = U.objectFieldOffset(klass, "threadIds"); + TERMINATION = U.objectFieldOffset(klass, "termination"); defaultForkJoinWorkerThreadFactory = new DefaultForkJoinWorkerThreadFactory(); diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index aff45d931a4e5..0db19c93fc163 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -213,31 +213,29 @@ public abstract class ForkJoinTask implements Future, Serializable { * holds bits recording completion status. There is no bit * representing "running", to recording whether incomplete tasks * are queued vs executing (although these cases can be - * distinguished in InterruptibleForkJoinTasks). Cancellation is + * distinguished in ExecutorServiceTasks). Cancellation is * recorded in status bits (ABNORMAL but not THROWN), but reported * in joining methods by throwing an exception. Other exceptions * of completed (THROWN) tasks are recorded in the "aux" field, * but may be reconstructed (in getThrowableException) to produce - * more useful stack traces when reported. Sentinels for being - * interrupted or timing out while waiting for completion are not + * more useful stack traces when reported. Sentinels for + * interruptions or timeouts while waiting for completion are not * recorded as status bits but are included in return values of - * methods in which they occur. - * - * The rules for how these are managed and issued in public - * methods have evolved across versions of this class. Those for - * tasks with results accessed via join() differ from those via - * Future.get(), which may also vary when invoked using pool - * submit methods by non-workers. In particular, internal usages - * of ForkJoinTasks ignore interrupt status when executing or - * awaiting completion. In all other cases, reporting task status - * (results or exceptions while executing) is always preferred to - * interrupts (also timeouts), restoring caller interrupt status - * in case of races. Similarly, completion status is preferred to - * reporting cancellation (ensured by atomically setting status), - * including cases where tasks are cancelled during termination. - * Cancellation is reported as an unchecked exception by join(), - * and by worker calls to get(), but is otherwise wrapped by a - * (checked) ExecutionException. + * methods in which they occur. Their representation and + * management have evolved across versions of this class (while + * remaining compatible with original conventions for + * computation-intensive tasks). Rules for reporting exceptions + * for tasks with results accessed via join() differ from those + * via get(), which differ from those invoked using pool submit + * methods by non-workers (which comply with Future.get() + * specs). Internal usages of ForkJoinTasks ignore interrupt + * status when executing or awaiting completion. Otherwise, + * reporting task results or exceptions is preferred to throwing + * InterruptedExecptions, which are in turn preferred to + * timeouts. Similarly, completion status is preferred to + * reporting cancellation. Cancellation is reported as an + * unchecked exception by join(), and by worker calls to get(), + * but is otherwise wrapped in a (checked) ExecutionException. * * The methods of this class are more-or-less layered into * (1) basic status maintenance @@ -287,9 +285,8 @@ final boolean casNext(Aux c, Aux v) { // used only in cancellation static final int THROWN = 1 << 17; static final int SMASK = 0xffff; // short bits for tags static final int UNCOMPENSATE = 1 << 16; // helpJoin return sentinel - static final int POOLSUBMIT = 1 << 18; // for pool.submit vs fork - // flags for awaitDone (in addition to above) + // flags for "how" arguments in awaitDone and elsewhere static final int RAN = 1; static final int INTERRUPTIBLE = 2; static final int TIMED = 4; @@ -312,13 +309,6 @@ private boolean casAux(Aux c, Aux v) { return U.compareAndSetReference(this, AUX, c, v); } - /** - * Marks this task as an external pool submission. - */ - final void markPoolSubmission() { - getAndBitwiseOrStatus(POOLSUBMIT); - } - /** Removes and unparks waiters */ private void signalWaiters() { for (Aux a; (a = aux) != null && a.ex == null; ) { @@ -379,6 +369,30 @@ final int trySetThrown(Throwable ex) { return s; } + /** + * Clears out a node after a wait was cancelled by interruption or + * timeout. Similar to AbstractQueuedSynchronizer (which see for + * further description). + */ + private void cancelWait(Aux node) { + outer: for (Aux a; (a = aux) != null && a.ex == null; ) { + for (Aux trail = null;;) { + Aux next = a.next; + if (a == node) { + if (trail != null) + trail.casNext(trail, next); + else if (casAux(a, next)) + break outer; // cannot be re-encountered + break; // restart + } else { + trail = a; + if ((a = next) == null) + break outer; + } + } + } + } + /** * Records exception unless already done. Overridable in subclasses. * @@ -422,97 +436,83 @@ final int doExec() { * Helps and/or waits for completion from join, get, or invoke; * called from either internal or external threads. * - * @param how flags for POOLSUBMIT, RAN, INTERRUPTIBLE, TIMED + * @param how flags for RAN, INTERRUPTIBLE, TIMED * @param deadline if timed, timeout deadline * @return ABNORMAL if interrupted, else status on exit */ private int awaitDone(int how, long deadline) { - int s; Thread t; ForkJoinWorkerThread wt; ForkJoinPool p; - ForkJoinPool.WorkQueue q = null; - boolean timed = (how & TIMED) != 0; - boolean owned = false, uncompensate = false; - if ((t = Thread.currentThread()) instanceof ForkJoinWorkerThread) { - owned = true; + int s; ForkJoinWorkerThread wt; ForkJoinPool p; ForkJoinPool.WorkQueue q; + boolean internal; + Thread t = Thread.currentThread(); + if (internal = (t instanceof ForkJoinWorkerThread)) { q = (wt = (ForkJoinWorkerThread)t).workQueue; p = wt.pool; } - else if ((p = ForkJoinPool.common) != null && (how & POOLSUBMIT) == 0) - q = p.externalQueue(); - if (q != null && p != null) { // try helping - if (this instanceof CountedCompleter) - s = p.helpComplete(this, q, owned, timed); - else if ((how & RAN) != 0 || - (s = q.tryRemoveAndExec(this, owned)) >= 0) - s = (owned) ? p.helpJoin(this, q, timed) : 0; - if (s < 0) - return s; - if (s == UNCOMPENSATE) - uncompensate = true; - if (p.runState < 0) // recheck below if interrupted - cancelIgnoringExceptions(this); - } + else + q = ((p = ForkJoinPool.common) != null) ? p.externalQueue() : null; + if ((s = tryHelpJoin(p, q, internal, how)) < 0) + return s; + int compensation = s; // used below after waiting + int interruptStatus = 0; // < 0 for throw, > 0 for reinterrupt + boolean queued = false; Aux node = null; - long ns = 0L; - boolean interrupted = false, queued = false; for (;;) { // install node and await signal - Aux a; + Aux a; long ns; if ((s = status) < 0) break; - else if (node == null) - node = new Aux(Thread.currentThread(), null); + else if (interruptStatus < 0) { + s = ABNORMAL; // interrupted and not done + break; + } + else if (Thread.interrupted()) { + interruptStatus = (how & INTERRUPTIBLE) != 0 ? -1 : 1; + if (p != null && q != null && p.runState < 0) + q.cancelRemainingTasks(this); // terminating + } else if (!queued) { + if (node == null) + node = new Aux(Thread.currentThread(), null); if (((a = aux) == null || a.ex == null) && (queued = casAux(node.next = a, node))) LockSupport.setCurrentBlocker(this); } - else if (timed && (ns = deadline - System.nanoTime()) <= 0) { - s = 0; - break; - } - else if (Thread.interrupted()) { - interrupted = true; - if (p != null && p.runState < 0) - cancelIgnoringExceptions(this); - if ((how & INTERRUPTIBLE) != 0) { - if ((s = status) >= 0) - s = ABNORMAL; // prefer reporting result to IE + else if ((how & TIMED) != 0) { + if ((ns = deadline - System.nanoTime()) <= 0L) { + s = 0; break; } - } - else if (timed) LockSupport.parkNanos(ns); + } else LockSupport.park(); } - if (uncompensate) - p.uncompensate(); - if (queued) { LockSupport.setCurrentBlocker(null); - if (s >= 0) { // cancellation similar to AbstractQueuedSynchronizer - outer: for (Aux a; (a = aux) != null && a.ex == null; ) { - for (Aux trail = null;;) { - Aux next = a.next; - if (a == node) { - if (trail != null) - trail.casNext(trail, next); - else if (casAux(a, next)) - break outer; // cannot be re-encountered - break; // restart - } else { - trail = a; - if ((a = next) == null) - break outer; - } - } - } - } - else { + if (s >= 0) + cancelWait(node); + else signalWaiters(); // help clean or signal - if (interrupted) - Thread.currentThread().interrupt(); - } } + if (compensation == UNCOMPENSATE && p != null) + p.uncompensate(); + if (interruptStatus > 0) + Thread.currentThread().interrupt(); + return s; + } + + /** + * Tries to locate and run this task or its subtasks. By default, + * applicable to possibly recursively structured tasks. Overridden + * in jdk subclasses that use different rules or mechanics. + * @return status if done or UNCOMPENSATE to uncompensate after waiting + */ + int tryHelpJoin(ForkJoinPool p, ForkJoinPool.WorkQueue q, + boolean internal, int how) { + int s = 0; + if (q != null && p != null && + ((how & RAN) != 0 || (s = q.tryRemoveAndExec(this, internal)) >= 0) && + internal) + s = p.helpJoin(this, q, (how & TIMED) != 0); return s; } @@ -601,6 +601,14 @@ private void reportException(int s) { ForkJoinTask.uncheckedThrow(getThrowableException()); } + /** + * Optional wrapping for cancellation in reportExecutionException. + * (Overridden by ExecutorServiceTasks.) + */ + Throwable wrapCancellationException(CancellationException cx) { + return cx; + } + /** * Throws exception for (timed or untimed) get, wrapping if * necessary in an ExecutionException. @@ -613,13 +621,8 @@ else if (s >= 0) ex = new TimeoutException(); else if ((rx = getThrowableException()) != null) ex = new ExecutionException(rx); - else { - rx = new CancellationException(); - if ((status & POOLSUBMIT) != 0) // wrap if external - ex = new ExecutionException(rx); - else - ex = rx; - } + else + ex = wrapCancellationException(new CancellationException()); ForkJoinTask.uncheckedThrow(ex); } @@ -689,7 +692,7 @@ public final ForkJoinTask fork() { public final V join() { int s; if ((s = status) >= 0) - s = awaitDone(s & POOLSUBMIT, 0L); + s = awaitDone(0, 0L); if ((s & ABNORMAL) != 0) reportException(s); return getRawResult(); @@ -1027,10 +1030,12 @@ public final void quietlyComplete() { */ public final V get() throws InterruptedException, ExecutionException { int s; - if (Thread.interrupted()) - s = ABNORMAL; - else if ((s = status) >= 0) - s = awaitDone((s & POOLSUBMIT) | INTERRUPTIBLE, 0L); + if ((s = status) >= 0) { + if (Thread.interrupted()) + s = ABNORMAL; + else + s = awaitDone(INTERRUPTIBLE, 0L); + } if ((s & ABNORMAL) != 0) reportExecutionException(s); return getRawResult(); @@ -1054,11 +1059,12 @@ public final V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { long nanos = unit.toNanos(timeout); int s; - if (Thread.interrupted()) - s = ABNORMAL; - else if ((s = status) >= 0 && nanos > 0L) - s = awaitDone((s & POOLSUBMIT) | INTERRUPTIBLE | TIMED, - nanos + System.nanoTime()); + if ((s = status) >= 0) { + if (Thread.interrupted()) + s = ABNORMAL; + else if (nanos > 0L) + s = awaitDone(INTERRUPTIBLE | TIMED, nanos + System.nanoTime()); + } if (s >= 0 || (s & ABNORMAL) != 0) reportExecutionException(s); return getRawResult(); @@ -1073,7 +1079,7 @@ else if ((s = status) >= 0 && nanos > 0L) public final void quietlyJoin() { int s; if ((s = status) >= 0) - awaitDone(s & POOLSUBMIT, 0L); + awaitDone(0, 0L); } /** @@ -1103,11 +1109,12 @@ public final boolean quietlyJoin(long timeout, TimeUnit unit) throws InterruptedException { int s; long nanos = unit.toNanos(timeout); - if (Thread.interrupted()) - s = ABNORMAL; - else if ((s = status) >= 0 && nanos > 0L) - s = awaitDone((s & POOLSUBMIT) | INTERRUPTIBLE | TIMED, - nanos + System.nanoTime()); + if ((s = status) >= 0) { + if (Thread.interrupted()) + s = ABNORMAL; + else if (nanos > 0L) + s = awaitDone(INTERRUPTIBLE | TIMED, nanos + System.nanoTime()); + } if (s == ABNORMAL) throw new InterruptedException(); else @@ -1128,7 +1135,7 @@ public final boolean quietlyJoinUninterruptibly(long timeout, int s; long nanos = unit.toNanos(timeout); if ((s = status) >= 0 && nanos > 0L) - s = awaitDone((s & POOLSUBMIT) | TIMED, nanos + System.nanoTime()); + s = awaitDone(TIMED, nanos + System.nanoTime()); return (s < 0); } @@ -1202,12 +1209,12 @@ public static boolean inForkJoinPool() { * @return {@code true} if unforked */ public boolean tryUnfork() { - Thread t; ForkJoinPool.WorkQueue q; boolean owned; - if (owned = (t = Thread.currentThread()) instanceof ForkJoinWorkerThread) + Thread t; ForkJoinPool.WorkQueue q; boolean internal; + if (internal = (t = Thread.currentThread()) instanceof ForkJoinWorkerThread) q = ((ForkJoinWorkerThread)t).workQueue; else q = ForkJoinPool.commonQueue(); - return (q != null && q.tryUnpush(this, owned)); + return (q != null && q.tryUnpush(this, internal)); } /** @@ -1403,6 +1410,8 @@ public final boolean compareAndSetForkJoinTaskTag(short expect, short update) { } } + // Classes wrapping Runnables and Callables + /** * Adapter for Runnables. This implements RunnableFuture * to be compliant with AbstractExecutorService constraints @@ -1426,6 +1435,10 @@ static final class AdaptedRunnable extends ForkJoinTask public String toString() { return super.toString() + "[Wrapped task = " + runnable + "]"; } + final int tryHelpJoin(ForkJoinPool p, ForkJoinPool.WorkQueue q, + boolean internal, int how) { + return internal ? super.tryHelpJoin(p, q, true, how) : 0; + } private static final long serialVersionUID = 5232453952276885070L; } @@ -1447,33 +1460,9 @@ public final void setRawResult(Void v) { } public String toString() { return super.toString() + "[Wrapped task = " + runnable + "]"; } - private static final long serialVersionUID = 5232453952276885070L; - } - - /** - * Adapter for Runnables in which failure forces worker exception. - */ - static final class RunnableExecuteAction extends ForkJoinTask { - @SuppressWarnings("serial") // Conditionally serializable - final Runnable runnable; - RunnableExecuteAction(Runnable runnable) { - if (runnable == null) throw new NullPointerException(); - this.runnable = runnable; - } - public final Void getRawResult() { return null; } - public final void setRawResult(Void v) { } - public final boolean exec() { runnable.run(); return true; } - int trySetException(Throwable ex) { // if a handler, invoke it - int s; Thread t; java.lang.Thread.UncaughtExceptionHandler h; - if (isExceptionalStatus(s = trySetThrown(ex)) && - (h = ((t = Thread.currentThread()). - getUncaughtExceptionHandler())) != null) { - try { - h.uncaughtException(t, ex); - } catch (Throwable ignore) { - } - } - return s; + final int tryHelpJoin(ForkJoinPool p, ForkJoinPool.WorkQueue q, + boolean internal, int how) { + return internal ? super.tryHelpJoin(p, q, true, how) : 0; } private static final long serialVersionUID = 5232453952276885070L; } @@ -1511,15 +1500,17 @@ public String toString() { } /** - * Tasks that are designed to be interruptible when cancelled, - * including cases of cancellation upon pool termination. In - * addition to recording the running thread to enable interrupt in + * Tasks with semantics conforming to ExecutorService + * conventions. Tasks are interruptible when cancelled, including + * cases of cancellation upon pool termination. In addition to + * recording the running thread to enable interrupt in * cancel(true), the task checks for termination before executing * the compute method, to cover shutdown races in which the task - * has not yet been cancelled on entry but might not otherwise be - * re-interrupted by others. + * has not yet been cancelled on entry and might not otherwise be + * cancelled by others. */ - static abstract class InterruptibleForkJoinTask extends ForkJoinTask { + static abstract class ExecutorServiceTask extends ForkJoinTask + implements RunnableFuture { transient volatile Thread runner; abstract T compute() throws Exception; static boolean isTerminating(Thread t) { @@ -1536,10 +1527,10 @@ public final boolean exec() { Thread.interrupted(); runner = t; try { - if (!isDone()) + if (status >= 0) setRawResult(compute()); } catch (Exception ex) { - trySetThrown(ex); + trySetException(ex); } finally { runner = null; } @@ -1549,17 +1540,25 @@ public final boolean exec() { public boolean cancel(boolean mayInterruptIfRunning) { int s = trySetCancelled(); if (s >= 0 && mayInterruptIfRunning) - ForkJoinTask.interruptIgnoringExceptions(runner); + interruptIgnoringExceptions(runner); return (s >= 0 || (s & (ABNORMAL | THROWN)) == ABNORMAL); } + final int tryHelpJoin(ForkJoinPool p, ForkJoinPool.WorkQueue q, + boolean internal, int how) { + return internal ? super.tryHelpJoin(p, q, true, how) : 0; + } + Throwable wrapCancellationException(CancellationException cx) { + return new ExecutionException(cx); + } + public final void run() { invoke(); } private static final long serialVersionUID = 2838392045355241008L; } /** * Adapter for Callable-based interruptible tasks. */ - static class AdaptedInterruptibleCallable extends InterruptibleForkJoinTask - implements RunnableFuture { + static final class AdaptedInterruptibleCallable + extends ExecutorServiceTask { @SuppressWarnings("serial") // Conditionally serializable final Callable callable; @SuppressWarnings("serial") // Conditionally serializable @@ -1570,8 +1569,7 @@ static class AdaptedInterruptibleCallable extends InterruptibleForkJoinTask extends InterruptibleForkJoinTask - implements RunnableFuture { + static final class AdaptedInterruptibleRunnable + extends ExecutorServiceTask { @SuppressWarnings("serial") // Conditionally serializable final Runnable runnable; @SuppressWarnings("serial") // Conditionally serializable @@ -1595,13 +1593,44 @@ static final class AdaptedInterruptibleRunnable extends InterruptibleForkJoin public final T getRawResult() { return result; } public final void setRawResult(T v) { } final T compute() { runnable.run(); return result; } - public final void run() { invoke(); } public String toString() { return super.toString() + "[Wrapped task = " + runnable + "]"; } private static final long serialVersionUID = 2838392045355241008L; } + /** + * Adapter for Runnables in which failure forces worker exception. + */ + static final class RunnableExecuteAction + extends ExecutorServiceTask { + @SuppressWarnings("serial") // Conditionally serializable + final Runnable runnable; + RunnableExecuteAction(Runnable runnable) { + if (runnable == null) throw new NullPointerException(); + this.runnable = runnable; + } + public final Void getRawResult() { return null; } + public final void setRawResult(Void v) { } + final Void compute() { runnable.run(); return null; } + int trySetException(Throwable ex) { // if a handler, invoke it + int s; Thread t; java.lang.Thread.UncaughtExceptionHandler h; + if (isExceptionalStatus(s = trySetThrown(ex)) && + (h = ((t = Thread.currentThread()). + getUncaughtExceptionHandler())) != null) { + try { + h.uncaughtException(t, ex); + } catch (Throwable ignore) { + } + } + return s; + } + public String toString() { + return super.toString() + "[Wrapped task = " + runnable + "]"; + } + private static final long serialVersionUID = 5232453952276885070L; + } + /** * Returns a new {@code ForkJoinTask} that performs the {@code run} * method of the given {@code Runnable} as its action, and returns diff --git a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java index 320f908cd3dc0..3a402c46d4c22 100644 --- a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java +++ b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java @@ -389,6 +389,7 @@ protected void realCompute() { } f.quietlyJoin(); checkCancelled(f); + Thread.interrupted(); }}; checkInvoke(a); a.reinitialize(); From c286c1e795d01e410ac4a5d73a26caae8df9b8c2 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sat, 18 Mar 2023 14:12:29 -0400 Subject: [PATCH 12/61] Recover performance --- .../util/concurrent/CountedCompleter.java | 44 +- .../java/util/concurrent/ForkJoinPool.java | 1855 +++++++++-------- .../java/util/concurrent/ForkJoinTask.java | 712 ++++--- 3 files changed, 1387 insertions(+), 1224 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java b/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java index 76d9378b7db7e..74ada2239f380 100644 --- a/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java +++ b/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java @@ -524,6 +524,10 @@ public final int getPendingCount() { return pending; } + final void initPending(int count) { + U.putInt(this, PENDING, count); + } + /** * Sets the pending count to the given value. * @@ -732,18 +736,37 @@ public final void helpComplete(int maxTasks) { if (q != null && maxTasks > 0) q.helpComplete(this, internal, maxTasks); } + // ForkJoinTask overrides + @Override + final boolean canHelpCompleteFJT(ForkJoinTask t) { + CountedCompleter f = null; + if (t instanceof CountedCompleter) { + f = (CountedCompleter)t; + do {} while (f != this && (f = f.completer) != null); + } + return f != null; + } + /** * Supports ForkJoinTask exception propagation. */ @Override - final int trySetException(Throwable ex) { + final void onFJTExceptionSet(Throwable ex) { CountedCompleter a = this, p = a; - do {} while (isExceptionalStatus(a.trySetThrown(ex)) && - a.onExceptionalCompletion(ex, p) && - (a = (p = a).completer) != null && a.status >= 0); - return status; + do {} while (a.onExceptionalCompletion(ex, p) && + (a = (p = a).completer) != null && + a.trySetThrown(ex)); + } + + /** + * Helps complete subtasks while joining + */ + @Override + final int tryHelpJoinFJT(ForkJoinPool p, ForkJoinPool.WorkQueue q, + boolean internal) { + return (p == null) ? 0 : p.helpComplete(this, q, internal); } /** @@ -777,17 +800,6 @@ protected final boolean exec() { @Override protected void setRawResult(T t) { } - /** - * Overrride default recursive helping to instead help along - * completion chains. - */ - @Override - final int tryHelpJoin(ForkJoinPool p, ForkJoinPool.WorkQueue q, - boolean internal, int how) { - return ((q == null || p == null) ? 0 : - p.helpComplete(this, q, internal, (how & TIMED) != 0)); - } - /* * This class uses jdk-internal Unsafe for atomics and special * memory modes, rather than VarHandles, to avoid initialization diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 9b3dc437cf3bd..2329ca44438d5 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -187,27 +187,39 @@ public class ForkJoinPool extends AbstractExecutorService { * Implementation Overview * * This class and its nested classes provide the main - * functionality and control for a set of worker threads: - * Submissions from non-FJ threads enter into submission queues. - * Workers take these tasks and typically split them into subtasks - * that may be stolen by other workers. Work-stealing based on - * randomized scans generally leads to better throughput than - * "work dealing" in which producers assign tasks to idle threads, - * in part because threads that have finished other tasks before - * the signalled thread wakes up (which can be a long time) can - * take the task instead. Preference rules give first priority to - * processing tasks from their own queues (LIFO or FIFO, depending - * on mode), then to randomized FIFO steals of tasks in other - * queues. This framework began as vehicle for supporting - * tree-structured parallelism using work-stealing. Over time, - * its scalability advantages led to extensions and changes to - * better support more diverse usage contexts. Because most - * internal methods and nested classes are interrelated, their - * main rationale and descriptions are presented here; individual - * methods and nested classes contain only brief comments about - * details. There are a fair number of odd code constructions and - * design decisions for components that reside at the edge of Java - * vs JVM functionality. + * functionality and control for a set of worker threads. Because + * most internal methods and nested classes are interrelated, + * their main rationale and descriptions are presented here; + * individual methods and nested classes contain only brief + * comments about details. Broadly: submissions from non-FJ + * threads enter into submission queues. Workers take these tasks + * and typically split them into subtasks that may be stolen by + * other workers. Work-stealing based on randomized scans + * generally leads to better throughput than "work dealing" in + * which producers assign tasks to idle threads, in part because + * threads that have finished other tasks before the signalled + * thread wakes up (which can be a long time) can take the task + * instead. Preference rules give first priority to processing + * tasks from their own queues (LIFO or FIFO, depending on mode), + * then to randomized FIFO steals of tasks in other queues. This + * framework began as vehicle for supporting tree-structured + * parallelism using work-stealing. Over time, its scalability + * advantages led to extensions and changes to better support more + * diverse usage contexts. Here's a brief history of major + * revisions, each also with other minor features and changes. + * + * 1. Only handle recursively structured computational tasks + * 2. Async (FIFO) mode and striped submission queues + * 3. Completion-based tasks (mainly CountedCompleters) + * 4. CommonPool and parallelStream support + * 5. ExternalTasks for externally submitted tasks + * + * Most changes involve adaptions of base algorithms using + * combinations of static and dynamic bitwise mode settings (both + * here and in ForkJoinTask), and subclassing of ForkJoinTask. + * There are a fair number of odd code constructions and design + * decisions for components that reside at the edge of Java vs JVM + * functionality. * * WorkQueues * ========== @@ -267,8 +279,8 @@ public class ForkJoinPool extends AbstractExecutorService { * uses masking, not mod, for indexing a power-of-two-sized array, * enforces memory ordering, supports resizing, and possibly * signals waiting workers to start scanning (described below), - * which requires even internal usages to strictly order accesses - * (using a form of lock release). + * which requires stronger forms of order accesses (using a form + * of lock release). * * The pop operation (always performed by owner) is of the form: * if ((task = getAndSet(q.array, (q.top-1) % length, null)) != null) @@ -289,7 +301,10 @@ public class ForkJoinPool extends AbstractExecutorService { * * Slot k must be read with an acquiring read, which it must * anyway to dereference and run the task if the (acquiring) * CAS succeeds, but uses an explicit acquire fence to support - * the following rechecks even if the CAS is not attempted. + * the following rechecks even if the CAS is not attempted. To + * more easily distinguish among kinds of CAS failures, we use + * the compareAndExchange version, and usually handle null + * returns (emptiness at that slot) separately from others. * * * q.base may change between reading and using its value to * index the slot. To avoid trying to use the wrong t, the @@ -312,8 +327,9 @@ public class ForkJoinPool extends AbstractExecutorService { * reduce producing these kinds of stalls by other stealers (as * well as quiescence checks etc described below), we encourage * timely writes to indices using otherwise unnecessarily - * strong writes or store fences when memory ordering is not - * already constrained by context. + * strong writes (including releasing the access lock even in + * cases where it need not be held) or fences when memory + * ordering is not already constrained by context. * * * The CAS may fail, in which case we may want to retry unless * there is too much contention. One goal is to balance and @@ -336,7 +352,10 @@ public class ForkJoinPool extends AbstractExecutorService { * one-ahead reads to check whether to repoll, relying on the * fact that a non-empty queue does not have two null slots in * a row, except in cases (resizes and shifts) that can be - * detected with a secondary recheck. + * detected with a secondary recheck. Otherwise, we read the + * access lock (see below) to conservatively (for submission + * queues), or more accurately (for worker queues) determine + * local quiescence. * * The poll operations in q.poll(), scan(), helpJoin(), and * elsewhere differ with respect to whether other queues are @@ -370,10 +389,7 @@ public class ForkJoinPool extends AbstractExecutorService { * elsewhere (by revisiting later. The lock needed for external * queues is generalized (as field "access") for operations on * internal queues that require a fully-fenced write to deal with - * Dekker-like signalling constructs described below. Because of - * the lock, checking non-quiescence of submission queues can be - * easier than worker queues because of the added check that if - * the lock is held, it is not quiescent. + * Dekker-like signalling constructs described below. * * Management * ========== @@ -408,26 +424,31 @@ public class ForkJoinPool extends AbstractExecutorService { * terminate when idle. * * Field "runState" holds lifetime status, atomically and - * monotonically setting SHUTDOWN, STOP, and finally TERMINATED - * bits. It is updated only via bitwise atomics (getAndBitwiseOr), - * although triggering STOP is performed under the registration - * lock to disable updates. The bits are arranged such that if - * runState is 0, the pool is not in a shutdown-able state, and if - * negative, it is stopping or terminated. + * monotonically setting SHUTDOWN, STOP, and TERMINATED as low + * bits, and otherwise acts as a 29-bit spin-based sequence lock + * for queue array updates (incrementing by LOCKED units per + * operation, rather than the usual even/odd scheme). The seqLock + * properties detect changes and conditionally upgrade to + * coordinate with some runState updates. It is typically held for + * less than a dozen instructions unless the queue array is being + * resized, during which contention is rare. Still, to be + * conservative, acquireRunStateLock is implemented as a + * spin/yield/sleep loop. In addition to checking pool status, + * reads of runState serve as acquire fences before reading other + * non-volatile fields. * * Array "queues" holds references to WorkQueues. It is updated * (only during worker creation and termination) under the - * registrationLock, but is otherwise concurrently readable but - * reads are always prefaced by a volatile read of runState to - * check termination, that also serves as an acquire fence. The - * registration lock is a simple AQS-based Sequence Lock, to - * enable retries if held during queiscence checks. To simplify - * index-based operations, the array size is always a power of - * two, and all readers must tolerate null slots. Worker queues - * are at odd indices. Worker ids masked with SMASK match their - * index. Shared (submission) queues are at even indices. Grouping - * them together in this way simplifies and speeds up task - * scanning. + * runState lock. It is otherwise concurrently readable but reads + * for use in scans (see below) are always prefaced by a volatile + * read of runState (or equivalent constructions), ensuring that + * its state is current at the point it is used (which is all we + * require). To simplify index-based operations, the array size is + * always a power of two, and all readers must tolerate null + * slots. Worker queues are at odd indices. Worker ids masked + * with SMASK match their index. Shared (submission) queues are at + * even indices. Grouping them together in this way simplifies and + * speeds up task scanning. * * All worker thread creation is on-demand, triggered by task * submissions, replacement of terminated workers, and/or @@ -454,7 +475,8 @@ public class ForkJoinPool extends AbstractExecutorService { * other hand, we must quickly prod them into action when new * tasks are submitted or generated. These latencies are mainly a * function of JVM park/unpark (and underlying OS) performance, - * which can be slow and variable. In many usages, ramp-up time + * which can be slow and variable (even though usages are + * streamlined as much as possible). In many usages, ramp-up time * is the main limiting factor in overall performance, which is * compounded at program start-up by JIT compilation and * allocation. On the other hand, throughput degrades when too @@ -464,18 +486,19 @@ public class ForkJoinPool extends AbstractExecutorService { * worker counts, plus the head of the available worker queue * (actually stack, represented by the lower 32bit subfield of * ctl). Released workers are those known to be scanning for - * and/or running tasks. Unreleased ("available") workers are - * recorded in the ctl stack. These workers are made eligible for - * signalling by enqueuing in ctl (see method awaitWork). The - * "queue" is a form of Treiber stack. This is ideal for - * activating threads in most-recently used order, and improves - * performance and locality, outweighing the disadvantages of - * being prone to contention and inability to release a worker - * unless it is topmost on stack. The top stack state holds the - * value of the "phase" field of the worker: its index and status, - * plus a version counter that, in addition to the count subfields - * (also serving as version stamps) provide protection against - * Treiber stack ABA effects. + * and/or running tasks (we cannot accurately determine + * which). Unreleased ("available") workers are recorded in the + * ctl stack. These workers are made eligible for signalling by + * enqueuing in ctl (see method awaitWork). The "queue" is a form + * of Treiber stack. This is ideal for activating threads in + * most-recently used order, and improves performance and + * locality, outweighing the disadvantages of being prone to + * contention and inability to release a worker unless it is + * topmost on stack. The top stack state holds the value of the + * "phase" field of the worker: its index and status, plus a + * version counter that, in addition to the count subfields (also + * serving as version stamps) provide protection against Treiber + * stack ABA effects. * * Creating workers. To create a worker, we pre-increment counts * (serving as a reservation), and attempt to construct a @@ -514,51 +537,82 @@ public class ForkJoinPool extends AbstractExecutorService { * workers to scan for tasks. Method signalWork and its callers * try to approximate the unattainable goal of having the right * number of workers activated for the tasks at hand, but must err - * on the side of too many workers vs too few to avoid stalls. If - * computations are purely tree structured, it suffices for every - * worker to activate another when it pushes a task into an empty - * queue, resulting in O(log(#threads)) steps to full activation. - * (To reduce resource usages in some cases, at the expense of - * slower startup in others, activation of an idle thread is - * preferred over creating a new one, here and elsewhere.) If - * instead, tasks come in serially from only a single producer, - * each worker taking its first (since the last activation) task - * from a queue should signal another if there are more tasks in - * that queue. This is equivalent to, but generally faster than, - * arranging the stealer take two tasks, re-pushing one on its own - * queue, and signalling (because its queue is empty), also - * resulting in logarithmic full activation time. Because we don't - * know about usage patterns (or most commonly, mixtures), we use - * both approaches. Together these are minimally necessary for - * maintaining liveness. However, they do not account for the fact - * that when tasks are short-lived, signals are unnecessary - * because workers will already be scanning for new tasks without - * the need of new signals. We track these cases (variable - * "prevSrc" in scan() and related methods) to avoid some - * unnecessary signals and scans. However, signal contention and - * overhead effects may still occur during ramp-up, ramp-down, and - * small computations involving only a few workers. + * on the side of too many workers vs too few to avoid stalls: + * + * * If computations are purely tree structured, it suffices for + * every worker to activate another when it pushes a task into + * an empty queue, resulting in O(log(#threads)) steps to full + * activation. As discussed above, "empty" must be + * conservatively approximated, sometimes resulting in + * unnecessary signals. Also, to reduce resource usages in + * some cases, at the expense of slower startup in others, + * activation of an idle thread is preferred over creating a + * new one, here and elsewhere. + * + * * If instead, tasks come in serially from only a single + * producer, each worker taking its first (since the last + * activation) task from a queue should signal another if there + * are more tasks in that queue. This is equivalent to, but + * generally faster than, arranging the stealer take multiple + * tasks, re-pushing one or more on its own queue, and + * signalling (because its queue is empty), also resulting in + * logarithmic full activation time + * + * * Because we don't know about usage patterns (or most commonly, + * mixtures), we use both approaches, which present even more + * opportunities to over-signal. Note that in either of these + * contexts, signals may be (and often are) unnecessary because + * active workers continue scanning after running tasks without + * the need to be signalled (which is one reason work stealing + * is often faster than alternatives), so additional workers + * aren't needed. But there is no efficient way to detect this + * without adding expensive and unscalable centralized control. + * + * * For rapidly branching tasks that require full pool resources, + * oversignalling is OK, because signalWork will soon have no + * more workers to create or reactivate. But for others (mainly + * externally submitted tasks), overprovisioning may cause very + * noticeable slowdowns due to contention and resource + * wastage. All remedies are intrinsically heuristic. We use a + * strategy that works well in most cases: We track "sources" + * (queue ids offset by constant 1 to avoid zero) of non-empty + * (usually polled) queues while scanning. These are maintained + * in the "source" field of WorkQueues for use in method + * helpJoin and elsewhere (see below). We also maintain them as + * arguments/results of top-level polls (argument "srcs" in + * method scan) as an encoded sliding window of the last two + * sources, and stop signalling when the last two were from the + * same source. And similarly not retry when two CAS failures + * were from the same source. This may result in transiently too + * few workers, but once they poll from the new source, they + * rapidly reactivate others. (An implementation / encoding + * quirk: Because of the offset, unnecessary signals for the + * maximum-numbered 32767th worker, which is unlikely to ever be + * produced, are not suppressed.) + * + * * Despite these, signal contention and overhead effects still + * occur during ramp-up and ramp-down of small computations. * * Scanning. Method scan performs top-level scanning for (and * execution of) tasks by polling a pseudo-random permutation of * the array (by starting at a random index, and using a constant - * cyclically exhaustive stride.) It uses the same basic polling - * method as WorkQueue.poll(), but restarts with a different - * permutation on each invocation. (Non-top-level scans; for - * example in helpJoin, use simpler and faster linear probes - * because they do not systematically contend with top-level - * scans.) The pseudorandom generator need not have high-quality - * statistical properties in the long term. We use Marsaglia - * XorShifts, seeded with the Weyl sequence from ThreadLocalRandom - * probes, which are cheap and suffice. Scans do not otherwise - * explicitly take into account core affinities, loads, cache - * localities, etc, However, they do exploit temporal locality - * (which usually approximates these) by preferring to re-poll - * from the same queue (using method tryPoll()) after a successful - * poll before trying others (see method topLevelExec), which also - * reduces bookkeeping and scanning overhead. This also reduces - * fairness, which is partially counteracted by giving up on - * contention. + * cyclically exhaustive stride.) This maintains top-level + * randomized fairness at the cost of poor locaility (other forms + * of scanning while helping etc described below have better + * locality.) It uses the same basic polling method as + * WorkQueue.poll(), but restarts with a different permutation on + * each invocation. The pseudorandom generator need not have + * high-quality statistical properties in the long term. We use + * Marsaglia XorShifts, seeded with the Weyl sequence from + * ThreadLocalRandom probes, which are cheap and suffice. Scans do + * not otherwise explicitly take into account core affinities, + * loads, cache localities, etc, However, they do exploit temporal + * locality (which usually approximates these) by preferring to + * re-poll from the same queue (using method tryPoll()) after a + * successful poll before trying others (see method topLevelExec), + * which also reduces bookkeeping and scanning overhead. This + * also reduces fairness, which is partially counteracted by + * giving up on contention. * * Deactivation. When method scan indicates that no tasks are * found by a worker, it deactivates (see awaitWork). Note that @@ -596,8 +650,8 @@ public class ForkJoinPool extends AbstractExecutorService { * SeqLock to retry on queue array updates. (It also reports * quiescence if the pool is terminating.) A true report means * only that there was a moment at which quiescence held; further - * actions based on this may be racy (requiring a seq lock upgrade - * to trigger quiescent termination). False negatives are + * actions based on this may be racy (except when using seq lock + * upgrade to trigger quiescent termination). False negatives are * inevitable (for example when queues indices lag updates, as * described above), which is accommodated when (tentatively) idle * by scanning for work etc, and then re-invoking. This includes @@ -605,25 +659,31 @@ public class ForkJoinPool extends AbstractExecutorService { * checkQuiescence() to check for tasks that could have been added * during a race window that would not be accompanied by a signal, * in which case re-activating itself (or any other worker) to - * rescan. This check also serves as a liveness safeguard when all - * other workers gave up prematurely and deactivated. Method - * helpQuiesce also uses checkQuiescence, but cannot rely on ctl - * counts to determine that all workers are inactive because the - * caller and any others executing helpQuiesce are not included. + * rescan. Method helpQuiesce acts similarly but cannot rely on + * ctl counts to determine that all workers are inactive because + * the caller and any others executing helpQuiesce are not + * included. + * + * The quiescence check in awaitWork also serves as a safeguard + * when all other workers gave up prematurely and inactivated (due + * to excessive contention or array resizing) which will cause + * this thread to rescan and wake up others. * * Termination. A call to shutdownNow invokes tryTerminate to - * atomically set a mode bit, performed under the registration - * lock to atomically disable further array updates. However, the - * process of termination is intrinsically non-atomic. The calling - * thread, as well as other workers thereafter terminating help - * cancel queued tasks and interrupt other workers. These actions - * race with unterminated workers. By default, workers check for + * atomically set a mode bit, performed on the runState lock to + * also disable further queue array updates. However, the process + * of termination is intrinsically non-atomic. The calling thread, + * as well as other workers thereafter terminating help cancel + * queued tasks and interrupt other workers. These actions race + * with unterminated workers. By default, workers check for * termination only when accessing pool state (usually the - * "queues" array). ExecutorServicesTasks (see below) - * additionally perform runState checks before executing their - * task bodies. But otherwise, there are no quarantees after an - * abrupt shutdown that remaining tasks complete normally or - * exceptionally or are cancelled. Termination may fail to + * "queues" array). This may take a while but suffices for + * structured computational tasks. But not necessarily for + * others. Class ExecutorServicesTask (see below) further arranges + * runState checks before executing task bodies, and ensures + * interrupts while terminating. Even so, there are no guarantees + * after an abrupt shutdown that remaining tasks complete normally + * or exceptionally or are cancelled. Termination may fail to * complete if running tasks repeatedly ignore both task status * and interrupts and/or produce more tasks after others that * could cancel them have exited. Parallelizing termination @@ -640,65 +700,82 @@ public class ForkJoinPool extends AbstractExecutorService { * Joining Tasks * ============= * + * The "Join" part of ForkJoinPools consists of a set of + * mechanisms that sometimes or always (depending on the kind of + * task) avoid context switching or adding worker threads when one + * task would otherwise be blocked waiting for completion of + * another, basically, just by running that task or one of its + * subtasks if not already stolen. As described below, these + * mechanics are disabled for ExternalTasks, that guarantee + * that callers do not executed submitted tasks. + * + * The basic structure of joining is an extended spin/block scheme + * in which workers check for task completions status between + * steps to find other work, until relevant pool state stabilizes + * enough to believe that no such tasks are available, at which + * point blocking. This is usually a good choice of when to block + * that would otherwise be harder to approximate. + * + * These forms of helping may increase stack space usage, but that + * space is bounded in tree/dag structured procedurally parallel + * designs to be no more than that if a task were executed only by + * the joining thread. This is arranged by associated task + * subclasses and internal tags that also help detect and control + * the ways in which this may occur. + * * Normally, the first option when joining a task that is not done - * is to try to take it from local queue and run it. Otherwise, - * any of several actions may be taken when one worker is waiting - * to join a task stolen (or always held) by another. Because we - * are multiplexing many tasks on to a pool of workers, we can't - * always just let them block (as in Thread.join). We also cannot - * just reassign the joiner's run-time stack with another and - * replace it later, which would be a form of "continuation", that - * even if possible is not necessarily a good idea since we may - * need both an unblocked task and its continuation to progress. - * Instead we combine two tactics: - * - * Helping: Arranging for the joiner to execute some task that it - * could be running if the steal had not occurred. - * - * Compensating: Unless there are already enough live threads, - * method tryCompensate() may create or re-activate a spare - * thread to compensate for blocked joiners until they unblock. - * - * A third form (implemented via tryRemoveAndExec) amounts to - * helping a hypothetical compensator: If we can readily tell that - * a possible action of a compensator is to steal and execute the - * task being joined, the joining thread can do so directly, - * without the need for a compensation thread; This may result in - * reordering a task slot in the queue, but in these cases, the - * join orders do not match (LIFO) submission orders anyway, so - * the reordering improves future processing. - * - * Other intermediate forms available for specific task types (for - * example helpAsyncBlocker) often avoid or postpone the need for - * blocking or compensation. - * - * The algorithm in helpJoin entails a form of "linear helping". - * Each worker records (in field "source") an offset index to the - * queue from which it last stole a task. (Implementation note: to - * distinguish lack of source from index 0 or other sentinels, - * source fields are offset by SRC.) The scan in method helpJoin - * uses these markers to try to find a worker to help (i.e., steal - * back a task from and execute it) that could hasten completion - * of the actively joined task. Thus, the joiner executes a task + * is to try to take it from the local queue and run it. Method + * tryRemoveAndExec tries to do so. For tasks with any form of + * subtasks that must be completed first, we try to locate these + * subtasks and run them as well. This is easy when local, but + * when stolen, steal-backs are restricted to the same rules as + * stealing (polling), which requires additional bookkeeping and + * scanning. This cost is still very much worthwhile because of + * its impact on task scheduling. + * + * The two methods for finding and executing subtasks vary in + * details. The algorithm in helpJoin entails a form of "linear + * helping". Each worker records (in field "source") an offset + * index to the queue from which it last stole a + * task. (Implementation note: to distinguish lack of source from + * index 0 or other sentinels, source fields are offset by + * constant 1.) The scan in method helpJoin uses these markers + * to try to find a worker to help (i.e., steal back a task from + * and execute it) that could makje progress toward completion of + * the actively joined task. Thus, the joiner executes a task * that would be on its own local deque if the to-be-joined task * had not been stolen. This is a conservative variant of the * approach described in Wagner & Calder "Leapfrogging: a portable * technique for implementing efficient futures" SIGPLAN Notices, * 1993 (http://portal.acm.org/citation.cfm?id=155354). It differs * mainly in that we only record queues, not full dependency - * links. This requires a linear scan of the queues array to - * locate stealers, but isolates cost to when it is needed, rather - * than adding to per-task overhead. For CountedCompleters, the + * links. This requires a linear scan of the queues to locate + * stealers, but isolates cost to when it is needed, rather than + * adding to per-task overhead. For CountedCompleters, the * analogous method helpComplete doesn't need stealer-tracking, - * but requires a similar check of completion chains. + * but requires a similar (but simpler) check of completion + * chains. * * In either case, searches can fail to locate stealers when - * stalls delay recording sources. We avoid some of these cases by - * using snapshotted values of ctl as a check that the numbers of - * workers are not changing. But even when accurately identified, - * stealers might not ever produce a task that the joiner can in - * turn help with. So, compensation is tried upon failure to find - * tasks to run. + * stalls delay recording sources or issuing subtasks. We avoid + * some of these cases by using snapshotted values of ctl as a + * check that the numbers of workers are not changing, along with + * rescans to deal with contention and stalls. But even when + * accurately identified, stealers might not ever produce a task + * that the joiner can in turn help with. + * + * Related method helpAsyncBlocker does not directly rely on + * subtask structure, but instead avoids or postpones blocking of + * tagged tasks (CompletableFuture.AsynchronousCompletionTask) by + * executing other asyncs that can be processed in any order. + * This is currently invoked only in non-join-based blocking + * contexts from classes CompletableFuture and + * SubmissionPublisher, that could be further generalized. + * + * When any of the above fail to avoid blocking, we rely on + * "compensation" -- an indirect form of context switching that + * either activates an existing worker to take the place of the + * blocked one, or expands the number of workers. * * Compensation does not by default aim to keep exactly the target * parallelism number of unblocked threads running at any given @@ -706,17 +783,17 @@ public class ForkJoinPool extends AbstractExecutorService { * compensations for any blocked join. However, in practice, the * vast majority of blockages are transient byproducts of GC and * other JVM or OS activities that are made worse by replacement - * when they cause longer-term oversubscription. These are - * inevitable without (unobtainably) perfect information about - * whether worker creation is actually necessary. False alarms - * are common enough to negatively impact performance, so - * compensation is by default attempted only when it is possible - * the the pool could stall due to lack of any unblocked workers. - * However, we allow users to override defaults using the long - * form of the ForkJoinPool constructor. The compensation - * mechanism may also be bounded. Bounds for the commonPool - * better enable JVMs to cope with programming errors and abuse - * before running out of resources to do so. + * by causing longer-term oversubscription. These are inevitable + * without (unobtainably) perfect information about whether worker + * creation is actually necessary. False alarms are common enough + * to negatively impact performance, so compensation is by default + * attempted only when it appears possible the the pool could + * stall due to lack of any unblocked workers. However, we allow + * users to override defaults using the long form of the + * ForkJoinPool constructor. The compensation mechanism may also + * be bounded. Bounds for the commonPool better enable JVMs to + * cope with programming errors and abuse before running out of + * resources to do so. * * The ManagedBlocker extension API can't use helping so relies * only on compensation in method awaitBlocker. This API was @@ -726,7 +803,8 @@ public class ForkJoinPool extends AbstractExecutorService { * users now rely upon the fact that if isReleasable always * returns false, the API can be used to obtain precautionary * compensation, which is sometimes their only reasonable option - * when running unknown code in tasks. + * when running unknown code in tasks; which is now supported more + * simply (see method beginCompensatedBlock). * * Common Pool * =========== @@ -743,23 +821,23 @@ public class ForkJoinPool extends AbstractExecutorService { * PRESET_SIZE set if parallelism was configured by system * property. * - * When external threads use ForkJoinTask.fork for the common - * pool, they can perform subtask processing (see helpComplete and - * related methods) upon joins. This caller-helps policy makes it - * sensible to set common pool parallelism level to one (or more) - * less than the total number of available cores, or even zero for - * pure caller-runs. For the sake of ExecutorService specs, we can - * only do this tasks not covered by ExecutorService APIs (see - * below). In all other cases, external threads waiting for joins - * first check the common pool for their task, which fails quickly - * if the caller did not fork to common pool. + * When external threads use the common pool, they can perform + * subtask processing (see helpComplete and related methods) upon + * joins, unless they are submitted using ExecutorService + * submission methods, which implicitly disallow this. This + * caller-helps policy makes it sensible to set common pool + * parallelism level to one (or more) less than the total number + * of available cores, or even zero for pure caller-runs. External + * threads waiting for joins first check the common pool for their + * task, which fails quickly if the caller did not fork to common + * pool. * * Guarantees for common pool parallelism zero are limited to * tasks that are joined by their callers in a tree-structured * fashion or use CountedCompleters (as is true for jdk * parallelStreams). Support infiltrates several methods, - * including those that retry helping steps or spin until we are - * sure that none apply if there are no workers. + * including those that retry helping steps until we are sure that + * none apply if there are no workers. * * As a more appropriate default in managed environments, unless * overridden by system properties, we use workers of subclass @@ -770,31 +848,46 @@ public class ForkJoinPool extends AbstractExecutorService { * may be JVM-dependent and must access particular Thread class * fields to achieve this effect. * - * Interrupt handling and Cancellation - * =================================== - * - * The framework is primarily designed to manage task cancellation - * (ForkJoinTask.cancel) independently from the interrupt status - * of threads running tasks. (See the public ForkJoinTask - * documentation for rationale.) Interrupts are issued internally - * only in tryTerminate, when workers should be terminating and - * tasks should be cancelled anyway. By default, interrupts are - * cleared only when necessary to ensure that calls to - * LockSupport.park do not loop indefinitely (park returns - * immediately if the current thread is interrupted). - * - * ExecutorServiceTasks. To conform to ExecutorService specs and - * expectations, externally invoked ExecutorService API methods - * use ExecutorServiceTasks. External submitters do not help run - * such tasks. These tasks include a "runner" field (similarly to - * FutureTask) to support cancel(true). We minimize possibilities - * of "stray" interrupts such as those in which an interrupt - * designed to cancel one task occurs late and unnecessarily. Upon - * pool shutdown, runners are interrupted so they can cancel. - * Since external joining callers never run these tasks, they must - * await cancellation by others. See the ForkJoinTask internal - * documentation for an account of how these correspond to how and - * when different exceptions are thrown. + * ExternalTasks + * ==================== + * + * Regular ForkJoinTasks manage task cancellation (method cancel) + * independently from the interrupt status of threads running + * tasks. Interrupts are issued internally only while + * terminating, to wake up workers and cancel queued tasks. By + * default, interrupts are cleared only when necessary to ensure + * that calls to LockSupport.park do not loop indefinitely (park + * returns immediately if the current thread is interrupted). + * + * To comply with ExecutorService specs, we use subclasses of + * abstract class ExternalTask for tasks that require + * stronger interruption and cancellation guarantees. External + * submitters never run these tasks, even if in the common pool. + * ExternalTasks include a "runner" field (implemented + * similarly to FutureTask) to support cancel(true). Upon pool + * shutdown, runners are interrupted so they can cancel. Since + * external joining callers never run these tasks, they must await + * cancellation by others, which can occur along several different + * paths. + * + * Across these APIs, rules for reporting exceptions for tasks + * with results accessed via join() differ from those via get(), + * which differ from those invoked using pool submit methods by + * non-workers (which comply with Future.get() specs). Internal + * usages of ForkJoinTasks ignore interrupt status when executing + * or awaiting completion. Otherwise, reporting task results or + * exceptions is preferred to throwing InterruptedExecptions, + * which are in turn preferred to timeouts. Similarly, completion + * status is preferred to reporting cancellation. Cancellation is + * reported as an unchecked exception by join(), and by worker + * calls to get(), but is otherwise wrapped in a (checked) + * ExecutionException. + * + * Worker Threads cannot be VirtualThreads, as enforced by + * requiring ForkJoinWorkerThreads in factories. There are + * several constructions relying on this. However as of this + * writing, virtual thread bodies are by default run as some form + * of ExternalTask. * * Memory placement * ================ @@ -857,11 +950,6 @@ public class ForkJoinPool extends AbstractExecutorService { * few unusual loop constructions encourage (with varying * effectiveness) JVMs about where (not) to place safepoints. * - * Even though the current version of this class initializes - * minimal subcomponents (queue array and lock), nearly all code - * (inefficiently) tolerates alternatives, including use of some - * further useless-looking checks. - * * There is a lot of representation-level coupling among classes * ForkJoinPool, ForkJoinWorkerThread, and ForkJoinTask. The * fields of WorkQueue maintain data structures managed by @@ -891,14 +979,24 @@ public class ForkJoinPool extends AbstractExecutorService { * * The main sources of differences from previous version are: * - * Handling of tasks submitted under the ExecutorService API is - * now consistent with specs. Method checkQuiescence() replaces - * previous quiescence-related checks. It relies on new class - * SeqLock replacing a ReentrantLock. Plus corresponding - * revisions to termination, requiring use of CountDownLatch - * instead of a Condition for termination, and many other - * adjustments to accommodate. - * + * * New abstract class ForkJoinTask.ExternalTask ensures + * handling of tasks submitted under the ExecutorService + * API consistent with specs. + * * Method checkQuiescence() replaces previous quiescence-related + * checks, relying on a sequence lock instead of ReentrantLock. + * * Termination processing now ensures that internal data + * structures are maintained consistently enough while stopping + * to interrupt all workers and cancel all tasks. It also uses a + * CountDownLatch instead of a Condition for termination because + * of lock change. + * * Refactorings to accommodate the increasing ways that tasks + * can be submitted, executed, joined. Includes reliance on new + * ForkJoinTask methods that can be overridden only by jdk + * subclasses (and with names including "FJT" to be + * extra-cautious about potential name clashes with external + * subclasses.) + * * Many other adjustments to avoid performance regressions due + * to the above. */ // static configuration constants @@ -936,19 +1034,19 @@ public class ForkJoinPool extends AbstractExecutorService { static final int MAX_CAP = 0x7fff; // max #workers - 1 // pool.runState bits - static final int STOP = 1 << 31; // must be negative - static final int SHUTDOWN = 1; - static final int TERMINATED = 2; - static final int PARKED = -1; // negative source when parked + static final int STOP = 1 << 0; + static final int SHUTDOWN = 1 << 1; + static final int TERMINATED = 1 << 2; + static final int LOCKED = 1 << 3; - // {pool, workQueue} bits and sentinels - static final int SRC = 1; // offset for source fields + // {pool, workQueue}.config bits and sentinels static final int FIFO = 1 << 16; // fifo queue or access mode - static final int ALIVE = 1 << 17; // set if initialized worker + static final int ALIVE = 1 << 17; // initialized, not deregistered static final int CLEAR_TLS = 1 << 18; // set for Innocuous workers static final int TRIMMED = 1 << 19; // timed out while idle static final int ISCOMMON = 1 << 20; // set for common pool static final int PRESET_SIZE = 1 << 21; // size was set by property + static final int UNCOMPENSATE = 1 << 16; // tryCompensate return /* @@ -982,7 +1080,13 @@ public class ForkJoinPool extends AbstractExecutorService { static final long TC_MASK = 0xffffL << TC_SHIFT; // sp bits static final int SS_SEQ = 1 << 16; // version count - static final int INACTIVE = 1 << 31; // phase bit when idle + static final int INACTIVE = 1 << 31; // phase or source bit when idle + static final int RESCAN = ~INACTIVE; // initial value for scan srcs + + // spin/yield/sleep control for runState locking and helpQuiesce + static final long MAX_SPINS = 1L << 6; // max calls to onSpinWait + static final long MIN_SLEEP = 1L << 10; // approx 1 usec as nanos + static final long MAX_SLEEP = 1L << 20; // approx 1 sec as nanos // Static utilities @@ -1099,32 +1203,6 @@ public ForkJoinWorkerThread run() { } } - /** - * Simple AQS-based Sequence Lock, used instead of ReentrantLock - * to enable detection of possible queue array changes and - * upgrades from seq-tracking reader to writer - */ - static final class SeqLock extends AbstractQueuedSynchronizer { - public boolean isHeldExclusively() { - return (getState() & 1) != 0; // odd if locked - } - public boolean tryAcquire(int ignore) { - int s = getState(); - return ((s & 1) == 0 && compareAndSetState(s, s + 1)); - } - public boolean tryRelease(int ignore) { - setState(getState() + 1); - return true; - } - public boolean tryUpgrade(int s) { // s must be even - return compareAndSetState(s, s + 1); // from reader to writer - } - public void lock() { acquire(1); } - public void unlock() { release(1); } - public int seq() { return getState(); } - private static final long serialVersionUID = 2838392045355241008L; - } - /** * Queues supporting work-stealing as well as external task * submission. See above for descriptions and algorithms. @@ -1144,7 +1222,7 @@ static final class WorkQueue { @jdk.internal.vm.annotation.Contended("w") volatile int phase; // versioned, negative if inactive @jdk.internal.vm.annotation.Contended("w") - volatile int source; // source queue id + SRC, or < 0 if parked + volatile int source; // source queue id + 1, or < 0 if parked @jdk.internal.vm.annotation.Contended("w") int nsteals; // number of steals from other queues @@ -1158,14 +1236,18 @@ static ForkJoinTask getAndClearSlot(ForkJoinTask[] a, int i) { return (ForkJoinTask) U.getAndSetReference(a, ((long)i << ASHIFT) + ABASE, null); } - static boolean casSlotToNull(ForkJoinTask[] a, int i, - ForkJoinTask c) { - return U.compareAndSetReference(a, ((long)i << ASHIFT) + ABASE, - c, null); + static ForkJoinTask cmpExSlotToNull(ForkJoinTask[] a, int i, + ForkJoinTask c) { + return (ForkJoinTask) + U.compareAndExchangeReference(a, ((long)i << ASHIFT) + ABASE, + c, null); } final int getAndSetAccess(int v) { return U.getAndSetInt(this, ACCESS, v); } + final void releaseAccess() { + U.putIntRelease(this, ACCESS, 0); + } /** * Constructor. For internal queues, most fields are initialized @@ -1197,53 +1279,65 @@ final int queueSize() { * * @param task the task. Caller must ensure non-null. * @param pool the pool. Must be non-null unless terminating. - * @param signalIfEmpty true if signal when pushing to empty queue + * @param signal true if signal when queue may have appeared empty * @throws RejectedExecutionException if array cannot be resized */ - final void push(ForkJoinTask task, ForkJoinPool pool, - boolean signalIfEmpty) { - boolean resize = false; - int s = top++, b = base, cap, m; ForkJoinTask[] a; - if ((a = array) != null && (cap = a.length) > 0) { - if ((m = (cap - 1)) == s - b) { - resize = true; // rapidly grow until large - int newCap = (cap < 1 << 24) ? cap << 2 : cap << 1; - ForkJoinTask[] newArray; - try { - newArray = new ForkJoinTask[newCap]; - } catch (Throwable ex) { - top = s; - access = 0; - throw new RejectedExecutionException( - "Queue capacity exceeded"); - } - if (newCap > 0) { // always true - int newMask = newCap - 1, k = s, remaining = cap; - do { // poll old, push to new - newArray[k-- & newMask] = task; - } while (remaining-- >= 0 && // exit when lose to pollers - (task = getAndClearSlot(a, k & m)) != null); - } - array = newArray; - } - else + final void push(ForkJoinTask task, ForkJoinPool pool, boolean signal) { + U.storeStoreFence(); // ensure safe publication + ForkJoinTask[] a = array; + int b = base, s = top++, cap, m; + if (a != null && (cap = a.length) > 0) { + if ((m = (cap - 1)) == s - b) + growAndPush(task, a, s); // may have appeared empty + else { a[m & s] = task; - getAndSetAccess(0); // for memory effects if internal - if ((resize || (a[m & (s - 1)] == null && signalIfEmpty)) && - pool != null) + getAndSetAccess(0); // for memory effects if internal + if (a[m & (s - 1)] != null) + signal = false; // not empty + } + if (signal && pool != null) pool.signalWork(); } } + /** + * Grows the task array if possible and adds the task. + */ + private void growAndPush(ForkJoinTask task, ForkJoinTask[] a, int s) { + int cap; + if (a != null && (cap = a.length) > 0) { // always true here + int newCap = (cap < 1 << 24) ? cap << 2 : cap << 1; + ForkJoinTask[] newArray; // rapidly grow until large + try { + newArray = new ForkJoinTask[newCap]; + } catch (Throwable ex) { + top = s; + access = 0; + throw new RejectedExecutionException( + "Queue capacity exceeded"); + } + if (newCap > 0) { // always true here + int newMask = newCap - 1, k = s, n = cap, m = cap - 1; + do { // poll old, push to new; exit if lose to other pollers + newArray[k-- & newMask] = task; + } while (n-- >= 0 && + (task = getAndClearSlot(a, k & m)) != null); + U.storeFence(); + array = newArray; + access = 0; + } + } + } + /** * Takes next task, if one exists, in order specified by mode, * so acts as either local-pop or local-poll. Called only by owner. * @param fifo nonzero if FIFO mode */ - final ForkJoinTask nextLocalTask(int fifo) { - ForkJoinTask t = null; + private ForkJoinTask nextLocalTask(int fifo) { ForkJoinTask[] a = array; - int p = top, s = p - 1, b = base, nb, cap; + int b = base, p = top, s = p - 1, nb, cap; + ForkJoinTask t = null; if (p - b > 0 && a != null && (cap = a.length) > 0) { do { if (fifo == 0 || (nb = b + 1) == p) { @@ -1262,7 +1356,7 @@ else if ((t = getAndClearSlot(a, (cap - 1) & b)) != null) { } } } while (p - b > 0); - U.storeStoreFence(); // for timely index updates + U.storeFence(); // for timely index updates } return t; } @@ -1280,21 +1374,23 @@ final ForkJoinTask nextLocalTask() { */ final boolean tryUnpush(ForkJoinTask task, boolean internal) { ForkJoinTask[] a = array; - int p = top, s, cap, k; - if (task != null && base != p && a != null && (cap = a.length) > 0 && - a[k = (cap - 1) & (s = p - 1)] == task) { - if (internal || getAndSetAccess(1) == 0) { - if (top != p || a[k] != task || - getAndClearSlot(a, k) == null) - access = 0; - else { - top = s; - access = 0; - return true; - } + int b = base, p = top, s, cap, k; + if (task == null || b == p || a == null || (cap = a.length) <= 0 || + a[k = (cap - 1) & (s = p - 1)] != task) + return false; + if (!internal) { + if (getAndSetAccess(1) != 0) + return false; + else if (top != p || cmpExSlotToNull(a, k, task) != task) { + releaseAccess(); + return false; } } - return false; + else if (getAndClearSlot(a, k) == null) + return false; + top = s; + releaseAccess(); + return true; } /** @@ -1302,7 +1398,7 @@ final boolean tryUnpush(ForkJoinTask task, boolean internal) { */ final ForkJoinTask peek() { ForkJoinTask[] a = array; - int cfg = config, p = top, b = base, cap; + int b = base, cfg = config, p = top, cap; if (p != b && a != null && (cap = a.length) > 0) { if ((cfg & FIFO) == 0) return a[(cap - 1) & (p - 1)]; @@ -1318,32 +1414,32 @@ final ForkJoinTask peek() { } /** - * Polls for a task. Used only by non-owners in usually - * uncontended contexts. + * Polls for a task. Used only by non-owners. * * @param pool if nonnull, pool to signal if more tasks exist */ final ForkJoinTask poll(ForkJoinPool pool) { for (int b = base;;) { - int cap; ForkJoinTask[] a; + int cap, k; ForkJoinTask[] a; if ((a = array) == null || (cap = a.length) <= 0) break; // currently impossible - int k = (cap - 1) & b, nb = b + 1, nk = (cap - 1) & nb; - ForkJoinTask t = a[k]; - U.loadFence(); // for re-reads - if (b != (b = base)) // inconsistent - ; - else if (t != null && casSlotToNull(a, k, t)) { - base = nb; - U.storeFence(); - if (pool != null && a[nk] != null) - pool.signalWork(); // propagate - return t; + ForkJoinTask t = a[k = (cap - 1) & b], u; + U.loadFence(); + int nb = b + 1, nk = (cap - 1) & nb; + if (b == (b = base)) { + if (t == null) + u = a[k]; // reread or CAS + else if ((u = cmpExSlotToNull(a, k, t)) == t) { + base = nb; + U.storeFence(); + if (pool != null && a[nk] != null) + pool.signalWork(); // propagate + return t; + } + if (u == null && access == 0 && top == b) + break; // empty + b = base; } - else if (array != a || a[k] != null) - ; // stale - else if (a[nk] == null && top - b <= 0 && access == 0) - break; // empty } return null; } @@ -1353,22 +1449,28 @@ else if (a[nk] == null && top - b <= 0 && access == 0) * contention or stalls. Used only by topLevelExec to repoll * from the queue obtained from pool.scan. */ - final ForkJoinTask tryPoll() { - int b = base, cap; ForkJoinTask[] a; + private ForkJoinTask tryPoll() { + ForkJoinTask[] a; int cap; if ((a = array) != null && (cap = a.length) > 0) { - for (;;) { - int k = (cap - 1) & b, nb = b + 1; - ForkJoinTask t = a[k]; - U.loadFence(); // for re-reads + for (int b = base, k;;) { + ForkJoinTask t = a[k = (cap - 1) & b], u; + U.loadFence(); + int nb = b + 1; if (b != (b = base)) - ; // inconsistent - else if (t != null && casSlotToNull(a, k, t)) { + ; + else if (t == null) { + if (a[k] == null) + break; + } + else if ((u = cmpExSlotToNull(a, k, t)) == t) { base = nb; - U.storeStoreFence(); + U.storeFence(); return t; } - else if (a[k] == null) - break; // contended, empty or stalled + else if (u == null) + break; + else + b = base; } } return null; @@ -1378,15 +1480,15 @@ else if (a[k] == null) /** * Runs the given (stolen) task if nonnull, as well as - * remaining local tasks and/or others available from its - * source queue, if any. + * remaining local tasks and/or others available from the + * given queue, if any. */ - final void topLevelExec(ForkJoinTask task, WorkQueue src) { + final void topLevelExec(ForkJoinTask task, WorkQueue q) { int cfg = config, fifo = cfg & FIFO, nstolen = 1; while (task != null) { task.doExec(); if ((task = nextLocalTask(fifo)) == null && - src != null && (task = src.tryPoll()) != null) + q != null && (task = q.tryPoll()) != null) ++nstolen; } nsteals += nstolen; @@ -1398,79 +1500,83 @@ final void topLevelExec(ForkJoinTask task, WorkQueue src) { /** * Deep form of tryUnpush: Traverses from top and removes and * runs task if present. - * @return task status if removed, else 0 + * @return true if successful */ - final int tryRemoveAndExec(ForkJoinTask task, boolean internal) { + final boolean tryRemoveAndExec(ForkJoinTask task, boolean internal) { ForkJoinTask[] a = array; - int p = top, s = p - 1, d = p - base, cap; + int b = base, p = top, s = p - 1, d = p - b, cap; if (task != null && d > 0 && a != null && (cap = a.length) > 0) { for (int m = cap - 1, i = s; ; --i) { ForkJoinTask t; int k; if ((t = a[k = i & m]) == task) { - if (!internal && getAndSetAccess(1) != 0) - break; // fail if locked - if (top != p || a[k] != task || - getAndClearSlot(a, k) == null) { - access = 0; - break; // missed + if (!internal) { + if (getAndSetAccess(1) != 0) + break; // fail if locked + if (top != p || cmpExSlotToNull(a, k, t) != t) { + releaseAccess(); + break; + } } + else if (getAndClearSlot(a, k) == null) + break; // missed if (i == s) // act as pop top = s; else if (i == base) // act as poll base = i + 1; - else // reorder by swapping top + else // swap top a[k] = getAndClearSlot(a, (top = s) & m); - access = 0; - return task.doExec(); + releaseAccess(); + task.doExec(); + return true; } else if (t == null || --d == 0) break; } } - return 0; + return false; } /** * Tries to pop and run tasks within the target's computation - * until done, not found, or limit exceeded. + * until done, not found, or limit exceeded. Note: The nominal + * param type here could just be ForkJoinTask, because it is + * only called in these contexts, but using CountedCompleter + * base type has been found to reduce warmup times. * * @param task root of computation * @param limit max runs, or zero for no limit - * @return task status on exit + * @return task status if known done; else 0 */ - final int helpComplete(ForkJoinTask task, boolean internal, int limit) { - int status = 0; + final int helpComplete(CountedCompleter task, boolean internal, + int limit) { if (task != null) { - outer: for (;;) { + for (;;) { ForkJoinTask[] a; ForkJoinTask t; - int p, s, cap, k; + int status, p, s, cap, k; if ((status = task.status) < 0) return status; if ((a = array) == null || (cap = a.length) <= 0 || (t = a[k = (cap - 1) & (s = (p = top) - 1)]) == null || - !(t instanceof CountedCompleter)) + !task.canHelpCompleteFJT(t)) // screen tasks break; - for (CountedCompleter f = (CountedCompleter)t;;) { - if (f == task) - break; - else if ((f = f.completer) == null) - break outer; // ineligible - } - if (!internal && getAndSetAccess(1) != 0) - break; // fail if locked - if (top != p || a[k] != t || getAndClearSlot(a, k) == null) { - access = 0; - break; // missed + if (!internal) { + if (getAndSetAccess(1) != 0) + break; // fail if locked + if (top != p || cmpExSlotToNull(a, k, t) != t) { + releaseAccess(); + break; // missed + } } + else if (getAndClearSlot(a, k) == null) + break; top = s; - access = 0; + releaseAccess(); t.doExec(); if (limit != 0 && --limit == 0) break; } - status = task.status; } - return status; + return 0; } /** @@ -1482,12 +1588,12 @@ else if ((f = f.completer) == null) final void helpAsyncBlocker(ManagedBlocker blocker) { if (blocker != null) { for (;;) { - int b = base, cap; ForkJoinTask[] a; - if ((a = array) == null || (cap = a.length) <= 0 || b == top) + int b = base, cap, k; ForkJoinTask[] a; + if ((a = array) == null || (cap = a.length) <= 0 || top - b <= 0) break; - int k = (cap - 1) & b, nb = b + 1, nk = (cap - 1) & nb; - ForkJoinTask t = a[k]; - U.loadFence(); // for re-reads + ForkJoinTask t = a[k = (cap - 1) & b]; + U.loadFence(); + int nb = b + 1, nk = (cap - 1) & nb; if (base != b) ; else if (blocker.isReleasable()) @@ -1498,9 +1604,9 @@ else if (t != null) { if (!(t instanceof CompletableFuture .AsynchronousCompletionTask)) break; - else if (casSlotToNull(a, k, t)) { + else if (cmpExSlotToNull(a, k, t) == t) { base = nb; - U.storeStoreFence(); + U.storeFence(); t.doExec(); } } @@ -1512,26 +1618,12 @@ else if (a[nk] == null) // misc - /** - * Cancels given task if nonnull and any other local tasks. - */ - final void cancelRemainingTasks(ForkJoinTask t) { - do { - if (t != null) { - try { - t.cancel(false); - } catch (Throwable ignore) { - } - } - } while ((t = poll(null)) != null); - } - /** * Returns true if internal and not known to be blocked. */ final boolean isApparentlyUnblocked() { Thread wt; Thread.State s; - return ((config & ALIVE) != 0 && (wt = owner) != null && + return ((wt = owner) != null && phase >= 0 && (config & ALIVE) != 0 && (s = wt.getState()) != Thread.State.BLOCKED && s != Thread.State.WAITING && s != Thread.State.TIMED_WAITING); @@ -1592,9 +1684,8 @@ final void setClearThreadLocals() { final long keepAlive; // milliseconds before dropping if idle final long bounds; // min, max threads packed as shorts final int config; // static configuration bits - volatile int runState; // SHUTDOWN, STOP, TERMINATED bits + volatile int runState; // SHUTDOWN, STOP, TERMINATED, seq bits WorkQueue[] queues; // main registry - final SeqLock registrationLock; volatile CountDownLatch termination; // lazily constructed final String workerNamePrefix; // null for common pool final ForkJoinWorkerThreadFactory factory; @@ -1626,9 +1717,6 @@ private long compareAndExchangeCtl(long c, long v) { private long getAndAddCtl(long v) { return U.getAndAddLong(this, CTL, v); } - private int getAndBitwiseOrRunState(int v) { - return U.getAndBitwiseOrInt(this, RUNSTATE, v); - } private long incrementThreadIds() { return U.getAndAddLong(this, THREADIDS, 1L); } @@ -1645,6 +1733,48 @@ private boolean casTerminationSignal(CountDownLatch x) { return U.compareAndSetReference(this, TERMINATION, null, x); } + // runState operations + + private int getAndBitwiseOrRunState(int v) { // for status bits + return U.getAndBitwiseOrInt(this, RUNSTATE, v); + } + private boolean casRunState(int c, int v) { + return U.compareAndSetInt(this, RUNSTATE, c, v); + } + private void releaseRunStateLock() { // increment lock bit + U.getAndAddInt(this, RUNSTATE, LOCKED); + } + private void acquireRunStateLock() { + int s; // locked when LOCKED set + if (((s = runState) & LOCKED) != 0 || !casRunState(s, s + LOCKED)) + spinLockRunState(); + } + private void spinLockRunState() { // spin/yield/sleep + for (long waits = 0L;;) { + int s; + if (((s = runState) & LOCKED) == 0) { + if (casRunState(s, s + LOCKED)) + break; + waits = 0L; + } + else if (waits < MAX_SPINS) { + Thread.onSpinWait(); + ++waits; + } + else if (waits == MAX_SPINS) { + Thread.yield(); + waits = MIN_SLEEP; + } + else { + LockSupport.parkNanos(this, waits); + if (waits < MAX_SLEEP) + waits <<= 1; + } + } + } + + final boolean stopping() { return (runState & STOP) != 0; } // for ForkJoinTask + // Creating, registering, and deregistering workers /** @@ -1655,12 +1785,13 @@ private boolean casTerminationSignal(CountDownLatch x) { * @return true if successful */ private boolean createWorker() { + int rs = runState; ForkJoinWorkerThreadFactory fac = factory; SharedThreadContainer ctr = container; Throwable ex = null; ForkJoinWorkerThread wt = null; try { - if (runState >= 0 && // avoid construction if terminating + if ((rs & STOP) == 0 && // avoid construction if terminating fac != null && (wt = fac.newThread(this)) != null) { if (ctr != null) ctr.start(wt); @@ -1692,30 +1823,40 @@ final String nextWorkerThreadName() { * @param w caller's WorkQueue */ final void registerWorker(WorkQueue w) { + WorkQueue[] qs; int n; ThreadLocalRandom.localInit(); int seed = ThreadLocalRandom.getProbe(); - SeqLock lock = registrationLock; - int cfg = config & FIFO; - if (w != null && lock != null) { + int cfg = (config & FIFO) | ALIVE; + if (w != null && (runState & STOP) == 0 && (qs = queues) != null && + (n = qs.length) > 0) { w.array = new ForkJoinTask[INITIAL_QUEUE_CAPACITY]; cfg |= w.config; w.stackPred = seed; - int seq = seed << 16; // initial phase seq int id = (seed << 1) | 1; // initial index guess - lock.lock(); + { // try to find slot without lock (nearly same code below if fail) + int k = n, m = n - 1; + for (; qs[id &= m] != null && k > 0; id -= 2, k -= 2); + if (k == 0) + id = -1; + } + acquireRunStateLock(); try { - WorkQueue[] qs; int n; // find queue index - if (runState >= 0 && (qs = queues) != null && - (n = qs.length) > 0) { // skip if terminating - int k = n, m = n - 1; - for (; qs[id &= m] != null && k > 0; id -= 2, k -= 2); - if (k == 0) - id = n | 1; // resize below - w.config = id | cfg | ALIVE; - w.phase = (id | seq) & ~INACTIVE; // now publishable - if (id < n) - qs[id] = w; - else { // expand array + if ((runState & STOP) == 0) { + boolean resize = false; + if (qs == (qs = queues) && id >= 0 && id < n && qs[id] == null) + qs[id] = w; + else if (qs != null && (n = qs.length) > 0) { + id = (seed << 1) | 1; // restart + int k = n, m = n - 1; + for (; qs[id &= m] != null && k > 0; id -= 2, k -= 2); + if (k == 0) { + id = n | 1; + resize = true; + } + } + w.config = id | cfg; + w.phase = id; // now publishable + if (resize) { // expand array int an = n << 1, am = an - 1; WorkQueue[] as = new WorkQueue[an]; as[id & am] = w; @@ -1726,12 +1867,12 @@ final void registerWorker(WorkQueue w) { if ((q = qs[j]) != null) // shared queues may move as[q.config & am] = q; } - U.storeFence(); // fill before publish + U.storeFence(); // fill before publish queues = as; } } } finally { - lock.unlock(); + releaseRunStateLock(); } } } @@ -1746,27 +1887,16 @@ final void registerWorker(WorkQueue w) { * @param ex the exception causing failure, or null if none */ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { - int cfg; SeqLock lock; - WorkQueue w = (wt == null) ? null : wt.workQueue; - if (w == null) + WorkQueue w; int cfg; + if ((w = (wt == null) ? null : wt.workQueue) == null) cfg = 0; else if (((cfg = w.config) & ALIVE) != 0) { w.config = cfg & ~ALIVE; - if (w.phase < 0) // possible during termination + if (w.phase < 0) // possible if stopped before release reactivate(w); - if (w.top - w.base > 0) - w.cancelRemainingTasks(null); - if (runState >= 0 && (lock = registrationLock) != null) { - WorkQueue[] qs; int n, i; // remove index unless terminating - long ns = w.nsteals & 0xffffffffL; - lock.lock(); - if (runState >= 0) { // recheck - if ((qs = queues) != null && (n = qs.length) > 0 && - qs[i = cfg & (n - 1)] == w) - qs[i] = null; - stealCount += ns; // accumulate steals - } - lock.unlock(); + if (w.top - w.base > 0) { + for (ForkJoinTask t; (t = w.nextLocalTask()) != null; ) + ForkJoinTask.cancelIgnoringExceptions(t); } } long c = ctl; @@ -1777,45 +1907,61 @@ else if (((cfg = w.config) & ALIVE) != 0) { (SP_MASK & c))))); else if ((int)c == 0) // was dropped on timeout cfg &= ~ALIVE; // suppress replacement if last - if (tryTerminate(false, false) >= 0 && (cfg & ALIVE) != 0) - signalWork(); // replace unless terminating, uninitialized or trimmed + + if ((tryTerminate(false, false) & STOP) == 0 && w != null) { + WorkQueue[] qs; int n, i; // remove index unless terminating + long ns = w.nsteals & 0xffffffffL; + acquireRunStateLock(); + if ((runState & STOP) == 0 && (qs = queues) != null && + (n = qs.length) > 0 && + qs[i = cfg & (n - 1)] == w) { + qs[i] = null; + stealCount += ns; // accumulate steals + } + releaseRunStateLock(); + if ((cfg & ALIVE) != 0) + signalWork(); // may replace unless trimmed or uninitialized + } if (ex != null) ForkJoinTask.rethrow(ex); } - /* + /** * Releases an idle worker, or creates one if not enough exist. */ final void signalWork() { - int pc = parallelism, n; - long c = ctl; - WorkQueue[] qs = queues; - if ((short)(c >>> RC_SHIFT) < pc && qs != null && (n = qs.length) > 0) { - for (;;) { - boolean create = false; - int sp = (int)c & ~INACTIVE; - WorkQueue v = qs[sp & (n - 1)]; - int deficit = pc - (short)(c >>> TC_SHIFT); - long ac = (c + RC_UNIT) & RC_MASK, nc; - if (sp != 0 && v != null) - nc = (v.stackPred & SP_MASK) | (c & TC_MASK); - else if (deficit <= 0) - break; + int pc = parallelism; + for (long c = ctl;;) { + Thread owner = null; + boolean create = false; + WorkQueue[] qs = queues; + long ac = (c + RC_UNIT) & RC_MASK, nc; + int deficit = pc - (short)(c >>> TC_SHIFT); + int sp = (int)c & ~INACTIVE, n; + if ((short)(c >>> RC_SHIFT) >= pc) + break; + if (qs == null || (n = qs.length) <= 0) + break; + WorkQueue v = qs[sp & (n - 1)]; + if (sp != 0 && v != null) { + owner = v.owner; + nc = (v.stackPred & SP_MASK) | (c & TC_MASK); + } + else if (deficit <= 0) + break; + else { + create = true; + nc = ((c + TC_UNIT) & TC_MASK); + } + if (c == (c = compareAndExchangeCtl(c, nc | ac))) { + if (create) + createWorker(); else { - create = true; - nc = ((c + TC_UNIT) & TC_MASK); - } - if (c == (c = compareAndExchangeCtl(c, nc | ac))) { - if (create) - createWorker(); - else { - Thread owner = v.owner; - v.phase = sp; - if (v.source < 0) - LockSupport.unpark(owner); - } - break; + v.phase = sp; + if (v.source < 0) + U.unpark(owner); } + break; } } } @@ -1825,26 +1971,26 @@ else if (deficit <= 0) * ctl stack), or any worker if null and idle. */ private void reactivate(WorkQueue w) { - WorkQueue[] qs; int n; - long c = ctl; - if ((qs = queues) != null && (n = qs.length) > 0) { - for (;;) { - int sp = (int)c & ~INACTIVE; - WorkQueue v = qs[sp & (n - 1)]; - long ac = UC_MASK & (c + RC_UNIT); - if (sp == 0 || v == null || - ((w == null) ? ((c & RC_MASK) > 0L) : (w.phase >= 0))) + for (long c = ctl;;) { // mostly, the no-create case of signalWork + WorkQueue v; + WorkQueue[] qs = queues; + int sp = (int)c & ~INACTIVE, n; + long pc = (UC_MASK & (c + RC_UNIT)) | (c & TC_MASK); + if (qs == null || (n = qs.length) <= 0 || sp == 0 || + (w == null && (c & RC_MASK) > 0L) || + (v = qs[sp & (n - 1)]) == null) + break; + Thread owner = v.owner; + long nc = (v.stackPred & SP_MASK) | pc; + if (w != null && w != v && w.phase >= 0) + break; + if (c == (c = compareAndExchangeCtl(c, nc))) { + v.phase = sp; + if (v.source < 0) + U.unpark(owner); + if (v == w || w == null) break; - if (c == (c = compareAndExchangeCtl( - c, (v.stackPred & SP_MASK) | ac))) { - Thread owner = v.owner; - v.phase = sp; - if (v.source < 0) - LockSupport.unpark(owner); - if (v == w || w == null) - break; - c = ctl; - } + c = ctl; } } } @@ -1857,7 +2003,7 @@ private boolean tryTrim(WorkQueue w) { int pred = w.stackPred, cfg = w.config | TRIMMED; long c = ctl; int sp = (int)c & ~INACTIVE; - if ((sp & SMASK) == (cfg & SMASK) && runState >= 0 && + if ((sp & SMASK) == (cfg & SMASK) && compareAndSetCtl(c, ((pred & SP_MASK) | (UC_MASK & (c - TC_UNIT))))) { w.config = cfg; // add sentinel for deregisterWorker @@ -1873,51 +2019,36 @@ private boolean tryTrim(WorkQueue w) { * Returns true if terminating or submission queues are empty and * unlocked, and all workers are inactive. If so, and quiescent * shutdown is enabled, starts termination. - * - * @param canStop if can start terminating if quiescent */ - private boolean checkQuiescence(boolean canStop) { - SeqLock lock; + private boolean checkQuiescence() { boolean scanned = false; - if ((lock = registrationLock) != null) { // else uninitialized - for (int seq = 0, prevSeq = 1; ; prevSeq = seq) { - WorkQueue[] qs; int n, rs; - if ((rs = runState) < 0) // terminating + for (int prevState = -1, rs = runState; ; prevState = rs) { + WorkQueue[] qs; int n; + if ((rs & STOP) != 0) // terminating + break; + if ((ctl & RC_MASK) > 0L) + return false; // active workers exist + if (scanned) { // try to trigger termination + if ((rs & SHUTDOWN) != 0 && !casRunState(rs, rs | STOP)) + scanned = false; // inconsistent; retry + else break; - if (canStop) { - if ((ctl & RC_MASK) > 0L) - return false; // active workers exist - if (rs != 0 && scanned) { - if (!lock.tryUpgrade(seq)) // trigger STOP under lock - scanned = false; // retry if lock held - else { - if ((ctl & RC_MASK) > 0L) // recheck - scanned = false; - else - getAndBitwiseOrRunState(STOP); - lock.unlock(); - } - } - } - if (scanned) + } + if ((qs = queues) == null || (n = qs.length) <= 0) + break; // currently impossible + for (int i = 0;;) { // alternates even for external, odd internal + WorkQueue q, w; + if ((q = qs[i]) != null && (q.access != 0 || q.top - q.base > 0)) + return false; // possibly nonempty + if (++i == n) + break; + if ((w = qs[i]) != null && w.phase > 0) + return false; // active worker + if (++i == n) break; - if ((qs = queues) != null && (n = qs.length) > 0) { - for (int i = 0;;) { - WorkQueue q, w; // alternate external, internal - if ((q = qs[i]) != null && // is external - (q.top != q.base || q.access != 0)) - return false; // possibly nonempty - if (++i == n) - break; - if ((w = qs[i]) != null && w.phase > 0) - return false; // active worker - if (++i == n) - break; - } - } - if ((seq = lock.seq()) == prevSeq && (seq & 1) == 0) - scanned = true; // same unlocked q state } + if ((rs = runState) == prevState && (rs & LOCKED) == 0) + scanned = true; // same unlocked q state } return true; } @@ -1930,111 +2061,117 @@ private boolean checkQuiescence(boolean canStop) { */ final void runWorker(WorkQueue w) { if (w != null) { // skip on failed init - int r = w.stackPred, src = 0; // use seed from registerWorker - do { + int r = w.stackPred; // use seed from registerWorker + for (int rs = 0, srcs = RESCAN;;) { r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift - } while (runState >= 0 && ((src = scan(w, src, r)) >= 0 || - (src = awaitWork(w)) == 0)); + if (rs != (rs = runState)) { // stop or rescan + if ((rs & STOP) != 0) + break; + srcs &= RESCAN; + } + if (srcs >= 0) + srcs = scan(w, srcs, r); + else if (awaitWork(w)) + srcs = RESCAN; + else + break; + } } } /** * Scans for and if found executes top-level tasks: Tries to poll * each queue starting at a random index with random stride, - * returning source id or retry indicator. + * returning scan window and retry indicator. * * @param w caller's WorkQueue - * @param prevSrc the two previous queues (if nonzero) stolen from - * in current phase, packed as int + * @param srcs encoding for the two previously non-empty scanned queues * @param r random seed - * @return the next prevSrc value to use, or negative if none found + * @return the next srcs value to use, with negative (INACTIVE) set if + * none found */ - private int scan(WorkQueue w, int prevSrc, int r) { + private int scan(WorkQueue w, int srcs, int r) { WorkQueue[] qs = queues; int n = (w == null || qs == null) ? 0 : qs.length; for (int step = (r >>> 16) | 1, i = n; i > 0; --i, r += step) { - int j, cap; WorkQueue q; ForkJoinTask[] a; + int j, cap, b, k; WorkQueue q; ForkJoinTask[] a; if ((q = qs[j = r & (n - 1)]) != null && (a = q.array) != null && (cap = a.length) > 0) { - int src = j + SRC, b = q.base; - int k = (cap - 1) & b, nb = b + 1, nk = (cap - 1) & nb; - ForkJoinTask t = a[k]; - U.loadFence(); // for re-reads - if (q.base != b) // inconsistent - return prevSrc; - else if (t != null && WorkQueue.casSlotToNull(a, k, t)) { + ForkJoinTask t = a[k = (cap - 1) & (b = q.base)], u; + U.loadFence(); // to re-read b and t + int nb = b + 1, nk = (cap - 1) & nb; + int qsrcs = ((j + 1) | (srcs << 16)) & RESCAN; + if (q.base != b) + return srcs; // inconsistent + if (t == null) { + if (a[k] != null || a[nk] != null) + return qsrcs; + } + else if ((u = WorkQueue.cmpExSlotToNull(a, k, t)) == t) { q.base = nb; - w.source = src; - int nextSrc = ((prevSrc << 16) | src) & 0x7fffffff; - if (nextSrc != prevSrc && q.base == nb && a[nk] != null) - signalWork(); // propagate at most twice/run + w.source = qsrcs; + if (qsrcs != srcs && a[nk] != null) + signalWork(); // propagate at most twice/run w.topLevelExec(t, q); - return nextSrc; + return qsrcs; } - else if (q.array != a || a[k] != null || a[nk] != null) - return prevSrc; // revisit + else if (u != null || (qsrcs != srcs && a[nk] != null)) + return qsrcs; // limit retries under contention } } - return -1; + return srcs | INACTIVE; } /** * Advances phase, enqueues, and awaits signal or termination. * - * @return negative if terminated, else 0 + * @return false for exit */ - private int awaitWork(WorkQueue w) { + private boolean awaitWork(WorkQueue w) { if (w == null) - return -1; // currently impossible - int p = (w.phase + SS_SEQ) & ~INACTIVE; // advance phase - if (runState < 0) - return -1; // terminating - boolean idle = false; // true if pool idle - int spins = (parallelism << 2) + 0xf; // avg #accesses in scan+signal + return false; // currently impossible + int p = (w.phase + SS_SEQ) & ~INACTIVE; // next phase number + int par = parallelism; long pc = ctl, sp = p & SP_MASK, qc; w.phase = p | INACTIVE; do { // enqueue w.stackPred = (int)pc; // set ctl stack link } while (pc != (pc = compareAndExchangeCtl( pc, qc = ((pc - RC_UNIT) & UC_MASK) | sp))); - if ((qc & RC_MASK) <= 0L) { // possibly quiescent - if (!(idle = checkQuiescence(true))) - reactivate(null); // ensure live - else if (runState < 0) - return -1; // quiescently terminating - } - while ((p = w.phase) < 0 && --spins > 0) - Thread.onSpinWait(); // spin before block - if (p < 0) { - long deadline = idle ? keepAlive + System.currentTimeMillis() : 0L; - LockSupport.setCurrentBlocker(this); - for (;;) { // await signal or termination - if (runState < 0) - return -1; - w.source = PARKED; // enable unpark - if (w.phase < 0) { - if (idle) - LockSupport.parkUntil(deadline); - else - LockSupport.park(); - } - w.source = 0; // disable unpark - if (w.phase >= 0) { - LockSupport.setCurrentBlocker(null); - break; - } - Thread.interrupted(); // clear status for next park - if (idle) { // check for idle timeout - if (deadline - System.currentTimeMillis() < TIMEOUT_SLOP) { - if (tryTrim(w)) - return -1; - else // not at head; restart timer - deadline += keepAlive; - } + boolean idle = false; // true if pool idle + if ((qc & RC_MASK) <= 0L && !(idle = checkQuiescence())) + reactivate(null); // ensure live if may be tasks + if ((runState & STOP) != 0) + return false; + for (int spins = (par << 2) + 0xf;;) { // avg #accesses in scan+signal + if (w.phase >= 0) // spin before block + return true; + if (--spins < 0) + break; + Thread.onSpinWait(); + } + long deadline = idle ? keepAlive + System.currentTimeMillis() : 0L; + LockSupport.setCurrentBlocker(this); // emulate LockSupport.park + for (;;) { // await signal or termination + w.source = INACTIVE; // enable unpark + if (w.phase < 0) + U.park(idle, deadline); + w.source = 0; // disable unpark + if (w.phase >= 0) { + LockSupport.setCurrentBlocker(null); + return true; + } + if (idle) { // check for idle timeout + if (deadline - System.currentTimeMillis() < TIMEOUT_SLOP) { + if (tryTrim(w)) + return false; + deadline += keepAlive; // not at head; restart timer } } + Thread.interrupted(); // clear status for next park + if ((runState & STOP) != 0) + return false; } - return 0; } /** @@ -2050,7 +2187,7 @@ private ForkJoinTask pollScan(boolean submissionsOnly) { r &= ~1; int step = (submissionsOnly) ? 2 : 1; WorkQueue[] qs; int n; WorkQueue q; ForkJoinTask t; - if (runState >= 0 && (qs = queues) != null && (n = qs.length) > 0) { + if ((runState & STOP) == 0 && (qs = queues) != null && (n = qs.length) > 0) { for (int i = n; i > 0; i -= step, r += step) { if ((q = qs[r & (n - 1)]) != null && (t = q.poll(this)) != null) @@ -2075,22 +2212,26 @@ private ForkJoinTask pollScan(boolean submissionsOnly) { private int tryCompensate(long c, boolean canSaturate) { Predicate sat; long b = bounds; // unpack fields - int pc = parallelism; + int pc = parallelism, rs; int minActive = (short)(b & SMASK), maxTotal = (short)(b >>> SWIDTH) + pc, active = (short)(c >>> RC_SHIFT), total = (short)(c >>> TC_SHIFT), sp = (int)c & ~INACTIVE; - if (runState < 0) // stopping + if (((rs = runState) & STOP) != 0) // terminating return 0; + else if ((rs & LOCKED) != 0) // unstable + return -1; else if (sp != 0 && active <= pc) { // activate idle worker WorkQueue[] qs; WorkQueue v; int i; if (ctl == c && (qs = queues) != null && qs.length > (i = sp & SMASK) && (v = qs[i]) != null) { + Thread owner = v.owner; long nc = (v.stackPred & SP_MASK) | (UC_MASK & c); if (compareAndSetCtl(c, nc)) { v.phase = sp; - LockSupport.unpark(v.owner); + if (v.source < 0) + U.unpark(owner); return UNCOMPENSATE; } } @@ -2102,7 +2243,7 @@ else if (active > minActive && total >= pc) { // reduce active workers } else if (total < maxTotal && total < MAX_CAP) { // expand pool long nc = ((c + TC_UNIT) & TC_MASK) | (c & ~TC_MASK); - return (!compareAndSetCtl(c, nc) ? -1 : + return (runState != rs || (!compareAndSetCtl(c, nc)) ? -1 : !createWorker() ? 0 : UNCOMPENSATE); } else if (!compareAndSetCtl(c, c)) // validate @@ -2129,70 +2270,70 @@ final void uncompensate() { * * @param task the task * @param w caller's WorkQueue - * @param timed true if this is a timed join + * @param internal true if w is owned by a ForkJoinWorkerThread * @return task status on exit, or UNCOMPENSATE for compensated blocking */ - final int helpJoin(ForkJoinTask task, WorkQueue w, boolean timed) { + final int helpJoin(ForkJoinTask task, WorkQueue w, boolean internal) { + int s; if (w == null || task == null) return 0; - int wsrc = w.source, cfg = (w.config & SMASK); - int wid = cfg + SRC, r = cfg + 2; + if (w.tryRemoveAndExec(task, internal) && (s = task.status) < 0) + return s; + if (!internal) + return 0; + int wsrc = w.source & SMASK, wid = (w.config & SMASK) + 1, r = wid + 1; long sctl = 0L; // track stability for (boolean rescan = true;;) { - int s; WorkQueue[] qs; + WorkQueue[] qs; if ((s = task.status) < 0) return s; if (!rescan && sctl == (sctl = ctl)) { - if ((s = tryCompensate(sctl, timed)) >= 0) + if ((s = tryCompensate(sctl, false)) >= 0) return s; // block } - else if (runState < 0) + else if ((runState & STOP) != 0) return 0; - int n = ((qs = queues) == null) ? 0 : qs.length, m = n - 1; rescan = false; + int n = ((qs = queues) == null) ? 0 : qs.length, m = n - 1; scan: for (int i = n >>> 1; i > 0; --i, r += 2) { int j, cap; WorkQueue q; ForkJoinTask[] a; - if ((q = qs[j = r & m]) != null && (a = q.array) != null && - (cap = a.length) > 0) { - for (int src = j + SRC;;) { - int sq = q.source, b = q.base; - int k = (cap - 1) & b, nb = b + 1; + if ((q = qs[j = r & m]) != null && + (a = q.array) != null && (cap = a.length) > 0) { + for (int src = j + 1;;) { + int sq = q.source, b = q.base, k = (cap - 1) & b; ForkJoinTask t = a[k]; - U.loadFence(); // for re-reads - boolean eligible; // check steal chain - for (int d = n, v = sq;;) { // may be cyclic; bound + U.loadFence(); + boolean eligible; // check steal chain + for (int d = n, v = sq;;) { // may be cyclic; bound WorkQueue p; - if (v == wid) { + if ((v &= SMASK) == wid) { eligible = true; break; } - if (v <= 0 || --d == 0 || - (p = qs[(v - SRC) & m]) == null) { + if (v == 0 || --d == 0 || + (p = qs[(v - 1) & m]) == null) { eligible = false; break; } v = p.source; } - if (q.source != sq || q.base != b) - ; // stale - else if ((s = task.status) < 0) - return s; // recheck before taking - else if (t == null) { - if (a[k] == null) { - if (!rescan && eligible && - (q.array != a || q.top != b)) - rescan = true; // resized or stalled + if ((s = task.status) < 0) + return s; // validate + int nb = b + 1, qdepth = q.top - b; + if (q.source == sq && q.base == b && a[k] == t) { + if (t == null) { + rescan |= (qdepth > 0); + break; // revisit if nonempty + } + if (t != task && !eligible) break; + if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { + q.base = nb; + w.source = src; + t.doExec(); + w.source = wsrc; } - } - else if (t != task && !eligible) - break; - else if (WorkQueue.casSlotToNull(a, k, t)) { - q.base = nb; - w.source = src; - t.doExec(); - w.source = wsrc; - rescan = true; + rescan = true; // restart at same index break scan; } } @@ -2202,74 +2343,62 @@ else if (WorkQueue.casSlotToNull(a, k, t)) { } /** - * Version of helpJoin for CountedCompleters. + * Helps join tasks with completers. * - * @param task the task + * @param task root of computation (only called when a CountedCompleter) * @param w caller's WorkQueue * @param internal true if w is owned by a ForkJoinWorkerThread * @param timed true if this is a timed join * @return task status on exit, or UNCOMPENSATE for compensated blocking */ - final int helpComplete(ForkJoinTask task, WorkQueue w, boolean internal, - boolean timed) { + final int helpComplete(CountedCompleter task, WorkQueue w, + boolean internal) { if (w == null || task == null) return 0; - int wsrc = w.source, r = w.config + 1; + int r = w.config + 1; // for indexing long sctl = 0L; // track stability - for (boolean rescan = true;;) { + for (boolean rescan = true, locals = true;;) { int s; WorkQueue[] qs; - if ((s = w.helpComplete(task, internal, 0)) < 0) + if ((locals && (s = w.helpComplete(task, internal, 0)) < 0) || + (s = task.status) < 0) return s; if (!rescan && sctl == (sctl = ctl)) { if (!internal) return 0; - if ((s = tryCompensate(sctl, timed)) >= 0) + if ((s = tryCompensate(sctl, false)) >= 0) return s; } - else if (runState < 0) + else if ((runState & STOP) != 0) return 0; + rescan = locals = false; int n = ((qs = queues) == null) ? 0 : qs.length, m = n - 1; - rescan = false; scan: for (int i = n; i > 0; --i, ++r) { int j, cap; WorkQueue q; ForkJoinTask[] a; - if ((q = qs[j = r & m]) != null && (a = q.array) != null && - (cap = a.length) > 0) { - poll: for (int src = j + SRC, b = q.base;;) { - int k = (cap - 1) & b, nb = b + 1; - ForkJoinTask t = a[k]; - U.loadFence(); // for re-reads - if (b != (b = q.base)) - ; // stale - else if ((s = task.status) < 0) - return s; // recheck before taking - else if (t == null) { - if (a[k] == null) { - if (!rescan && // resized or stalled - (q.array != a || - q.top != b || q.access != 0)) - rescan = true; + if ((q = qs[j = r & m]) != null && + (a = q.array) != null && (cap = a.length) > 0) { + for (int b = q.base, k;;) { + ForkJoinTask t = a[k = (cap - 1) & b]; + U.loadFence(); + int nb = b + 1, nk = (cap - 1) & nb; + boolean eligible = task.canHelpCompleteFJT(t); + if ((s = task.status) < 0) + return s; // validate + if (b == (b = q.base) && a[k] == t) { + if (t == null) { + rescan |= (a[nk] != null); // revisit break; } - } - else if (t instanceof CountedCompleter) { - CountedCompleter f; - for (f = (CountedCompleter)t;;) { - if (f == task) - break; - else if ((f = f.completer) == null) - break poll; // ineligible - } - if (WorkQueue.casSlotToNull(a, k, t)) { + if (!eligible) + break; + if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { q.base = nb; - w.source = src; + U.storeFence(); t.doExec(); - w.source = wsrc; - rescan = true; - break scan; + locals = true; // process local queue } + rescan = true; + break scan; } - else - break; } } } @@ -2286,15 +2415,17 @@ else if ((f = f.completer) == null) * @return positive if quiescent, negative if interrupted, else 0 */ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { - long startTime = System.nanoTime(), parkTime = 0L; int phase; // w.phase set negative when temporarily quiescent if (w == null || (phase = w.phase) < 0) return 0; - int activePhase = phase, inactivePhase = phase | INACTIVE; int wsrc = w.source, r = w.config + 1, returnStatus = 1; - for (boolean locals = true;;) { + long maxSleep = Math.min(nanos >>> 8, MAX_SLEEP); // approx 1% nanos + long startTime = System.nanoTime(), waits = 0L; + int activePhase = phase, inactivePhase = phase | INACTIVE; + boolean locals = true; + for (int prevState = -1, rs = runState; ; prevState = rs) { WorkQueue[] qs; WorkQueue q; - if (runState < 0) + if ((rs & STOP) != 0) break; // terminating if (interruptible && Thread.interrupted()) { returnStatus = -1; @@ -2305,28 +2436,32 @@ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { for (ForkJoinTask u; (u = w.nextLocalTask()) != null;) u.doExec(); } - boolean rescan = false; + boolean rescan = false, busy = false; int n = ((qs = queues) == null) ? 0 : qs.length, m = n - 1; scan: for (int i = n, j; i > 0; --i, ++r) { - if ((q = qs[j = m & r]) != null) { - for (int src = j + SRC;;) { + if ((q = qs[j = m & r]) != null && q != w) { + for (int src = j + 1;;) { ForkJoinTask[] a = q.array; - int b = q.base, cap; + int b = q.base, cap, k; if (a == null || (cap = a.length) <= 0) break; - int k = (cap - 1) & b, nb = b + 1; - ForkJoinTask t = a[k]; - U.loadFence(); // for re-reads - if (q.base != b || q.array != a || a[k] != t) + ForkJoinTask t = a[k = (cap - 1) & b]; + U.loadFence(); + int nb = b + 1; + if (t != null && phase < 0) // reactivate before taking + w.phase = phase = activePhase; + if (q.base != b || a[k] != t) ; else if (t == null) { - if (!rescan && (q.top != b || q.access != 0)) - rescan = true; + if (!rescan) { + if (q.access != 0 || q.top - b > 0) + rescan = true; + else if (q.phase > 0) + busy = true; + } break; } - else if (phase < 0) // reactivate before taking - w.phase = phase = activePhase; - else if (WorkQueue.casSlotToNull(a, k, t)) { + else if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { q.base = nb; w.source = src; t.doExec(); @@ -2337,27 +2472,29 @@ else if (WorkQueue.casSlotToNull(a, k, t)) { } } } - if (!rescan) { - if (phase >= 0) { // tentatively inactivate - w.phase = phase = inactivePhase; - parkTime = -2L; // to recheck, then yield, then sleep - } - if (checkQuiescence(false)) - break; - else if (System.nanoTime() - startTime > nanos) { - returnStatus = 0; // timed out - break; - } - else if (parkTime < 0L) { // recheck or yield - if (++parkTime == 0L) - Thread.yield(); - } - else { // initial sleep approx 1 usec - parkTime = Math.max(parkTime, 1L << 10); - LockSupport.parkNanos(this, parkTime); - if (parkTime < nanos >>> 8 && parkTime < 1L << 20) - parkTime <<= 1; // max sleep approx 1 sec or 1% nanos - } + rs = runState; + if (rescan || (rs & LOCKED) != 0 || rs != prevState) + ; // retry + else if (!busy) + break; + else if (phase >= 0) { + w.phase = phase = inactivePhase; + waits = 0L; // to recheck, then yield/sleep + } + else if (System.nanoTime() - startTime > nanos) { + returnStatus = 0; // timed out + break; + } + else if (waits == 0L) // same as spinLockRunState except + waits = 1L; // with rescan instead of onSpinWait + else if (waits == 1L) { + waits = MIN_SLEEP; + Thread.yield(); + } + else { + LockSupport.parkNanos(this, waits); + if (waits < maxSleep) + waits <<= 1; } } w.phase = activePhase; @@ -2372,30 +2509,32 @@ else if (parkTime < 0L) { // recheck or yield * @return positive if quiescent, negative if interrupted, else 0 */ private int externalHelpQuiesce(long nanos, boolean interruptible) { - long startTime = System.nanoTime(), parkTime = 0L; + long maxSleep = Math.min(nanos >>> 8, MAX_SLEEP); + long startTime = System.nanoTime(), waits = 0L; for (;;) { // same structure as helpQuiesce, using poll vs scanning ForkJoinTask t; - if (runState < 0) + if ((runState & STOP) != 0) return 1; else if (interruptible && Thread.interrupted()) return -1; else if ((t = pollScan(false)) != null) { - parkTime = -2L; t.doExec(); + waits = 0L; } - else if (checkQuiescence(true)) + else if (checkQuiescence()) return 1; else if (System.nanoTime() - startTime > nanos) return 0; - else if (parkTime < 0L) { - if (++parkTime == 0L) - Thread.yield(); + else if (waits == 0L) + waits = 1L; + else if (waits == 1L) { + waits = MIN_SLEEP; + Thread.yield(); } else { - parkTime = Math.max(parkTime, 1L << 10); - LockSupport.parkNanos(this, parkTime); - if (parkTime < nanos >>> 8 && parkTime < 1L << 20) - parkTime <<= 1; + LockSupport.parkNanos(this, waits); + if (waits < maxSleep) + waits <<= 1; } } } @@ -2442,35 +2581,31 @@ final ForkJoinTask nextTaskFor(WorkQueue w) { */ final WorkQueue submissionQueue(boolean isSubmit) { int r; - SeqLock lock = registrationLock; if ((r = ThreadLocalRandom.getProbe()) == 0) { ThreadLocalRandom.localInit(); // initialize caller's probe r = ThreadLocalRandom.getProbe(); } - if (lock != null) { // else init error - for (int id = r << 1;;) { // even indices only - int n, i, rs; WorkQueue[] qs; WorkQueue q; - if ((qs = queues) == null || (n = qs.length) <= 0) - break; - else if ((q = qs[i = (n - 1) & id]) == null) { - WorkQueue w = new WorkQueue(null, id); - w.array = new ForkJoinTask[INITIAL_QUEUE_CAPACITY]; - lock.lock(); // install under lock - if ((rs = runState) >= 0 && queues == qs && qs[i] == null) - qs[i] = w; // else lost race; discard - lock.unlock(); - if (rs < 0) - break; // terminating - } - else if (q.getAndSetAccess(1) != 0) // move and restart - id = (r = ThreadLocalRandom.advanceProbe(r)) << 1; - else if (isSubmit && runState != 0) { - q.access = 0; // check while lock held - break; - } - else - return q; + for (int id = r << 1;;) { // even indices only + int n, i; WorkQueue[] qs; WorkQueue q; + if ((qs = queues) == null || (n = qs.length) <= 0) + break; + else if ((q = qs[i = (n - 1) & id]) == null) { + WorkQueue w = new WorkQueue(null, id | ALIVE); + w.array = new ForkJoinTask[INITIAL_QUEUE_CAPACITY]; + acquireRunStateLock(); // install under lock + if (queues == qs && qs[i] == null) // OK even if stopping + qs[i] = w; // else lost race; discard + releaseRunStateLock(); + } + else if (q.getAndSetAccess(1) != 0) // move and restart + id = (r = ThreadLocalRandom.advanceProbe(r)) << 1; + else if (isSubmit && (runState & SHUTDOWN) != 0) { + q.access = 0; // check while q lock held + tryTerminate(false, false); // may trigger termination + break; } + else + return q; } throw new RejectedExecutionException(); } @@ -2480,16 +2615,14 @@ else if (isSubmit && runState != 0) { * from ForkJoinWorkerThread, else external queue. */ private ForkJoinTask poolSubmit(boolean signalIfEmpty, + boolean external, ForkJoinTask task) { - WorkQueue q; Thread t; ForkJoinWorkerThread wt; - U.storeStoreFence(); // ensure safely publishable if (task == null) throw new NullPointerException(); - if (((t = Thread.currentThread()) instanceof ForkJoinWorkerThread) && - (wt = (ForkJoinWorkerThread)t).pool == this) - q = wt.workQueue; - else { - q = submissionQueue(true); - } + Thread t = Thread.currentThread(); + ForkJoinWorkerThread wt = ((t instanceof ForkJoinWorkerThread) ? + (ForkJoinWorkerThread)t : null); + WorkQueue q = ((!external && wt != null && wt.pool == this) ? + wt.workQueue : submissionQueue(true)); q.push(task, this, signalIfEmpty); return task; } @@ -2499,7 +2632,7 @@ private ForkJoinTask poolSubmit(boolean signalIfEmpty, * possibly ever submitted to the given pool (nonzero probe), or * null if none. */ - private static WorkQueue externalQueue(ForkJoinPool p) { + static WorkQueue externalQueue(ForkJoinPool p) { WorkQueue[] qs; int r = ThreadLocalRandom.getProbe(), n; return (p != null && (qs = p.queues) != null && @@ -2605,44 +2738,36 @@ static int getSurplusQueuedTaskCount() { * @param now if true, unconditionally terminate, else only * if no work and no active workers * @param enable if true, terminate when next possible - * @return runState on exit + * @return runState (possibly only its status bits) on exit */ private int tryTerminate(boolean now, boolean enable) { - int rs = runState; - SeqLock lock = registrationLock; - if ((config & ISCOMMON) == 0 && lock != null && rs >= 0) { - if (now) { - lock.lock(); // transition under lock - rs = getAndBitwiseOrRunState(STOP | SHUTDOWN) | STOP; - lock.unlock(); - } + int rs = runState, cfg = config, isShutdown; + if ((rs & STOP) == 0 && (cfg & ISCOMMON) == 0) { + if (now) + getAndBitwiseOrRunState(rs = STOP | SHUTDOWN); else { - if (enable && (rs & SHUTDOWN) == 0) - rs = getAndBitwiseOrRunState(SHUTDOWN) | SHUTDOWN; - if ((rs & SHUTDOWN) != 0) { - checkQuiescence(true); // sets runState - rs = runState; - } + if ((isShutdown = (rs & SHUTDOWN)) == 0 && enable) + getAndBitwiseOrRunState(isShutdown = SHUTDOWN); + if (isShutdown != 0 && checkQuiescence()) + rs = STOP | SHUTDOWN; } } - if ((rs & (STOP | TERMINATED)) == STOP) { // help terminate - WorkQueue[] qs; + if ((rs & (STOP | TERMINATED)) == STOP) { + WorkQueue[] qs; WorkQueue q; // cancel tasks and interrupt workers Thread current = Thread.currentThread(); WorkQueue w = ((current instanceof ForkJoinWorkerThread) ? ((ForkJoinWorkerThread)current).workQueue : null); int r = (w != null) ? w.config : 0; // stagger traversals int n = ((qs = queues) == null) ? 0 : qs.length, m = n - 1; - for (int i = n; i > 0; --i, ++r) { - WorkQueue q; Thread t; - if ((q = qs[m & r]) != null) { - if ((t = q.owner) != null && !t.isInterrupted() && - ((q.config) & ALIVE) != 0) // try to unblock - ForkJoinTask.interruptIgnoringExceptions(t); - q.cancelRemainingTasks(null); + for (int i = n, j; i > 0; --i, ++r) { + if ((q = qs[j = m & r]) != null && (q.config & ALIVE) != 0) { + if ((j & 1) != 0) + ForkJoinTask.interruptIgnoringExceptions(q.owner); + for (ForkJoinTask t; (t = q.poll(null)) != null; ) + ForkJoinTask.cancelIgnoringExceptions(t); } } if (((rs = runState) & TERMINATED) == 0 && ctl == 0L) { - rs |= TERMINATED; // transition if ((getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0) { CountDownLatch done; SharedThreadContainer ctr; if ((done = termination) != null) @@ -2650,6 +2775,7 @@ private int tryTerminate(boolean now, boolean enable) { if ((ctr = container) != null) ctr.close(); } + rs = runState; } } return rs; @@ -2659,12 +2785,12 @@ private int tryTerminate(boolean now, boolean enable) { * Lazily constructs termination signal */ private CountDownLatch terminationSignal() { - CountDownLatch x; + CountDownLatch signal; do { - x = termination; - } while (x == null && // OK to throw away in rare cases of CAS failures - !casTerminationSignal(x = new CountDownLatch(1))); - return x; + signal = termination; + } while (signal == null && // OK to throw away if CAS failure + !casTerminationSignal(signal = new CountDownLatch(1))); + return signal; } // Exported methods @@ -2850,7 +2976,6 @@ public ForkJoinPool(int parallelism, this.bounds = (long)(minAvail & SMASK) | (long)(maxSpares << SWIDTH) | ((long)corep << 32); int size = 1 << (33 - Integer.numberOfLeadingZeros(p - 1)); - this.registrationLock = new SeqLock(); this.queues = new WorkQueue[size]; String pid = Integer.toString(getAndAddPoolIds(1) + 1); String name = "ForkJoinPool-" + pid; @@ -2905,7 +3030,6 @@ private ForkJoinPool(byte forCommonPoolOnly) { this.keepAlive = DEFAULT_KEEPALIVE; this.saturate = null; this.workerNamePrefix = null; - this.registrationLock = new SeqLock(); this.queues = new WorkQueue[size]; this.container = SharedThreadContainer.create("ForkJoinPool.commonPool"); } @@ -2948,7 +3072,7 @@ public static ForkJoinPool commonPool() { * scheduled for execution */ public T invoke(ForkJoinTask task) { - poolSubmit(true, task); + poolSubmit(true, false, task); return task.join(); } @@ -2961,7 +3085,7 @@ public T invoke(ForkJoinTask task) { * scheduled for execution */ public void execute(ForkJoinTask task) { - poolSubmit(true, task); + poolSubmit(true, false, task); } // AbstractExecutorService methods @@ -2974,7 +3098,7 @@ public void execute(ForkJoinTask task) { @Override @SuppressWarnings("unchecked") public void execute(Runnable task) { - poolSubmit(true, (task instanceof ForkJoinTask) + poolSubmit(true, false, (task instanceof ForkJoinTask) ? (ForkJoinTask) task // avoid re-wrap : new ForkJoinTask.RunnableExecuteAction(task)); } @@ -2994,7 +3118,7 @@ public void execute(Runnable task) { * scheduled for execution */ public ForkJoinTask submit(ForkJoinTask task) { - return poolSubmit(true, task); + return poolSubmit(true, false, task); } /** @@ -3004,7 +3128,8 @@ public ForkJoinTask submit(ForkJoinTask task) { */ @Override public ForkJoinTask submit(Callable task) { - return poolSubmit(true, new ForkJoinTask.AdaptedInterruptibleCallable(task)); + return poolSubmit(true, false, + new ForkJoinTask.AdaptedInterruptibleCallable(task)); } /** @@ -3014,7 +3139,8 @@ public ForkJoinTask submit(Callable task) { */ @Override public ForkJoinTask submit(Runnable task, T result) { - return poolSubmit(true, new ForkJoinTask.AdaptedInterruptibleRunnable(task, result)); + return poolSubmit(true, false, + new ForkJoinTask.AdaptedInterruptibleRunnable(task, result)); } /** @@ -3025,7 +3151,7 @@ public ForkJoinTask submit(Runnable task, T result) { @Override @SuppressWarnings("unchecked") public ForkJoinTask submit(Runnable task) { - return poolSubmit(true, (task instanceof ForkJoinTask) + return poolSubmit(true, false, (task instanceof ForkJoinTask) ? (ForkJoinTask) task // avoid re-wrap : new ForkJoinTask.AdaptedInterruptibleRunnable(task, null)); } @@ -3050,10 +3176,7 @@ public ForkJoinTask submit(Runnable task) { * @since 20 */ public ForkJoinTask externalSubmit(ForkJoinTask task) { - U.storeStoreFence(); // ensure safely publishable - WorkQueue q = submissionQueue(true); - q.push(task, this, true); - return task; + return poolSubmit(true, true, task); } /** @@ -3073,7 +3196,7 @@ public ForkJoinTask externalSubmit(ForkJoinTask task) { * @since 19 */ public ForkJoinTask lazySubmit(ForkJoinTask task) { - return poolSubmit(false, task); + return poolSubmit(false, false, task); } /** @@ -3120,7 +3243,7 @@ public List> invokeAll(Collection> tasks) { ForkJoinTask.AdaptedInterruptibleCallable f = new ForkJoinTask.AdaptedInterruptibleCallable(t); futures.add(f); - poolSubmit(true, f); + poolSubmit(true, false, f); } for (int i = futures.size() - 1; i >= 0; --i) { ForkJoinTask.AdaptedInterruptibleCallable f = @@ -3129,9 +3252,10 @@ public List> invokeAll(Collection> tasks) { } return futures; } catch (Throwable t) { - for (Future e : futures) - ForkJoinTask.cancelIgnoringExceptions( - (ForkJoinTask.AdaptedInterruptibleCallable)e); + for (Future e : futures) { + if (e != null) + e.cancel(true); + } throw t; } } @@ -3147,39 +3271,38 @@ public List> invokeAll(Collection> tasks, ForkJoinTask.AdaptedInterruptibleCallable f = new ForkJoinTask.AdaptedInterruptibleCallable(t); futures.add(f); - poolSubmit(true, f); + poolSubmit(true, false, f); } long startTime = System.nanoTime(), ns = nanos; boolean timedOut = (ns < 0L); for (int i = futures.size() - 1; i >= 0; --i) { ForkJoinTask.AdaptedInterruptibleCallable f = (ForkJoinTask.AdaptedInterruptibleCallable)futures.get(i); - if (!f.isDone()) { + while (!f.isDone()) { if (!timedOut) timedOut = !f.quietlyJoin(ns, TimeUnit.NANOSECONDS); if (timedOut) - ForkJoinTask.cancelIgnoringExceptions(f); + f.cancel(true); else ns = nanos - (System.nanoTime() - startTime); } } return futures; } catch (Throwable t) { - for (Future e : futures) - ForkJoinTask.cancelIgnoringExceptions( - (ForkJoinTask.AdaptedInterruptibleCallable)e); + for (Future e : futures) { + if (e != null) + e.cancel(true); + } throw t; } } - /** + /** * Task (that is never forked) to hold results for invokeAny, or * to report exception if all subtasks fail or are cancelled or - * the pool is terminating. ForkJoinTask tags are used to avoid - * multiple calls to tryComplete by the same task under async - * cancellation. + * the pool is terminating. */ - static final class InvokeAnyRoot extends ForkJoinTask.ExecutorServiceTask { + static final class InvokeAnyRoot extends ForkJoinTask.ExternalTask { private static final long serialVersionUID = 2838392045355241008L; @SuppressWarnings("serial") // Conditionally serializable volatile E result; @@ -3193,18 +3316,20 @@ static final class InvokeAnyRoot extends ForkJoinTask.ExecutorServiceTask final void tryComplete(InvokeAnyTask f, E v, Throwable ex, boolean completed) { ForkJoinPool p; - if (!isDone() && f != null && - f.compareAndSetForkJoinTaskTag((short)0, (short)1)) { - if (completed) { - result = v; - quietlyComplete(); - } - else if (count.getAndDecrement() <= 1 || - ((p = getPool()) != null && p.runState < 0)) { - if (ex == null) - trySetCancelled(); - else - trySetThrown(ex); + if (f != null && !isDone()) { + if ((p = getPool()) != null && p.stopping()) + trySetCancelled(); + else if (f.setForkJoinTaskStatusMarkerBit() == 0) { + if (completed) { + result = v; + quietlyComplete(); + } + else if (count.getAndDecrement() <= 1) { + if (ex == null) + trySetCancelled(); + else + trySetException(ex); + } } } } @@ -3220,7 +3345,7 @@ final E invokeAny(Collection> tasks, for (Callable c : tasks) { InvokeAnyTask f = new InvokeAnyTask(this, c); fs.add(f); - pool.poolSubmit(true, f); + pool.poolSubmit(true, false, f); if (isDone()) break; } @@ -3236,10 +3361,10 @@ final E invokeAny(Collection> tasks, } /** - * ExecutorServiceTask with results in InvokeAnyRoot (and never + * ExternalTask with results in InvokeAnyRoot (and never * independently joined). */ - static final class InvokeAnyTask extends ForkJoinTask.ExecutorServiceTask { + static final class InvokeAnyTask extends ForkJoinTask.ExternalTask { final InvokeAnyRoot root; @SuppressWarnings("serial") // Conditionally serializable final Callable callable; @@ -3250,12 +3375,12 @@ static final class InvokeAnyTask extends ForkJoinTask.ExecutorServiceTask r = root; - if (!r.isDone()) { - E v = null; Throwable ex = null; boolean completed = false; + E v = null; Throwable ex = null; boolean completed = false; + if (r != null && !r.isDone()) { try { v = callable.call(); completed = true; - } catch (Throwable rex) { + } catch (Exception rex) { ex = rex; } finally { r.tryComplete(this, v, ex, completed); @@ -3263,11 +3388,12 @@ final Void compute() throws Exception { } return null; } - public boolean cancel(boolean mayInterruptIfRunning) { - InvokeAnyRoot r = root; - if (r != null) + public final boolean cancel(boolean mayInterruptIfRunning) { + InvokeAnyRoot r; + boolean stat = super.cancel(mayInterruptIfRunning); + if ((r = root) != null) r.tryComplete(this, null, null, false); - return super.cancel(mayInterruptIfRunning); + return stat; } public final Void getRawResult() { return null; } public final void setRawResult(Void v) { } @@ -3400,7 +3526,7 @@ public int getActiveThreadCount() { * @return {@code true} if all threads are currently idle */ public boolean isQuiescent() { - return checkQuiescence(true); + return checkQuiescence(); } /** @@ -3476,7 +3602,7 @@ public int getQueuedSubmissionCount() { */ public boolean hasQueuedSubmissions() { WorkQueue[] qs; WorkQueue q; - if (runState >= 0 && (qs = queues) != null) { + if ((runState & STOP) == 0 && (qs = queues) != null) { for (int i = 0; i < qs.length; i += 2) { if ((q = qs[i]) != null && q.queueSize() > 0) return true; @@ -3556,7 +3682,7 @@ public String toString() { int ac = (short)(c >>> RC_SHIFT); if (ac < 0) // ignore transient negative ac = 0; - int rs = runState; + long rs = runState; String level = ((rs & TERMINATED) != 0 ? "Terminated" : (rs & STOP) != 0 ? "Terminating" : (rs & SHUTDOWN) != 0 ? "Shutting down" : @@ -3648,7 +3774,7 @@ public boolean isTerminating() { * @return {@code true} if this pool has been shut down */ public boolean isShutdown() { - return runState != 0; + return (runState & SHUTDOWN) != 0; } /** @@ -3884,17 +4010,18 @@ private void compensatedBlock(ManagedBlocker blocker) * blocking operation is done then endCompensatedBlock must be invoked * with the value returned by this method to re-adjust the parallelism. */ - private long beginCompensatedBlock() { - int comp; - do {} while ((comp = tryCompensate(ctl, false)) < 0); - return (comp == 0) ? 0L : RC_UNIT; + final long beginCompensatedBlock() { + int c; + while ((c = tryCompensate(ctl, false)) < 0) + Thread.onSpinWait(); + return (c == 0) ? 0L : RC_UNIT; } /** * Re-adjusts parallelism after a blocking operation completes. */ void endCompensatedBlock(long post) { - if (post > 0) { + if (post > 0L) { getAndAddCtl(post); } } diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index 0db19c93fc163..96a3dca4a81d7 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -15,7 +15,8 @@ * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * Inc., 51 Franklin St, Fifth Floor +, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any @@ -210,37 +211,25 @@ public abstract class ForkJoinTask implements Future, Serializable { * responsible for maintaining their "status" field amidst relays * to methods in ForkJoinWorkerThread and ForkJoinPool, along with * recording and reporting exceptions. The status field mainly - * holds bits recording completion status. There is no bit - * representing "running", to recording whether incomplete tasks - * are queued vs executing (although these cases can be - * distinguished in ExecutorServiceTasks). Cancellation is - * recorded in status bits (ABNORMAL but not THROWN), but reported - * in joining methods by throwing an exception. Other exceptions - * of completed (THROWN) tasks are recorded in the "aux" field, - * but may be reconstructed (in getThrowableException) to produce - * more useful stack traces when reported. Sentinels for - * interruptions or timeouts while waiting for completion are not - * recorded as status bits but are included in return values of - * methods in which they occur. Their representation and - * management have evolved across versions of this class (while - * remaining compatible with original conventions for - * computation-intensive tasks). Rules for reporting exceptions - * for tasks with results accessed via join() differ from those - * via get(), which differ from those invoked using pool submit - * methods by non-workers (which comply with Future.get() - * specs). Internal usages of ForkJoinTasks ignore interrupt - * status when executing or awaiting completion. Otherwise, - * reporting task results or exceptions is preferred to throwing - * InterruptedExecptions, which are in turn preferred to - * timeouts. Similarly, completion status is preferred to - * reporting cancellation. Cancellation is reported as an - * unchecked exception by join(), and by worker calls to get(), - * but is otherwise wrapped in a (checked) ExecutionException. + * holds bits recording completion status. Note that there is no + * status bit representing "running", recording whether incomplete + * tasks are queued vs executing. However these cases can be + * distinguished in subclasses of ExternalTask that adds + * this capability by recording the running thread. Cancellation + * is recorded in status bits (ABNORMAL but not THROWN), but + * reported in joining methods by throwing an exception. Other + * exceptions of completed (THROWN) tasks are recorded in the + * "aux" field, but are reconstructed (see getThrowableException) + * to produce more useful stack traces when reported. Sentinels + * for interruptions or timeouts while waiting for completion are + * not recorded as status bits but are included in return values + * of methods in which they occur. * * The methods of this class are more-or-less layered into * (1) basic status maintenance * (2) execution and awaiting completion * (3) user-level methods that additionally report results. + * (4) Subclasses for adaptors and internal usages * This is sometimes hard to see because this file orders exported * methods in a way that flows well in javadocs. */ @@ -283,14 +272,19 @@ final boolean casNext(Aux c, Aux v) { // used only in cancellation static final int DONE = 1 << 31; // must be negative static final int ABNORMAL = 1 << 16; static final int THROWN = 1 << 17; - static final int SMASK = 0xffff; // short bits for tags + static final int MARKER = 1 << 30; // utility marker + static final int UNCOMPENSATE = 1 << 16; // helpJoin return sentinel + static final int SMASK = 0xffff; // short bits for tags // flags for "how" arguments in awaitDone and elsewhere static final int RAN = 1; static final int INTERRUPTIBLE = 2; static final int TIMED = 4; + // conveniences + private static final int HAVE_EXCEPTION = DONE | ABNORMAL | THROWN; + // Fields volatile int status; // accessed directly by pool and workers private transient volatile Aux aux; // either waiters or thrown Exception @@ -324,12 +318,10 @@ private void signalWaiters() { /** * Sets DONE status and wakes up threads waiting to join this task. - * @return status on exit */ - private int setDone() { - int s = getAndBitwiseOrStatus(DONE) | DONE; + private void setDone() { + getAndBitwiseOrStatus(DONE); signalWaiters(); - return s; } /** @@ -350,195 +342,165 @@ final int trySetCancelled() { * If losing a race with setDone or trySetCancelled, the exception * may be recorded but not reported. * - * @return status on exit + * @return true if set */ - final int trySetThrown(Throwable ex) { - Aux h = new Aux(Thread.currentThread(), ex), p = null; - boolean installed = false; + final boolean trySetThrown(Throwable ex) { int s; - while ((s = status) >= 0) { - Aux a; - if (!installed && ((a = aux) == null || a.ex == null) && - (installed = casAux(a, h))) - p = a; // list of waiters replaced by h - if (installed && casStatus(s, s |= (DONE | ABNORMAL | THROWN))) - break; - } - for (; p != null; p = p.next) - LockSupport.unpark(p.thread); - return s; - } - - /** - * Clears out a node after a wait was cancelled by interruption or - * timeout. Similar to AbstractQueuedSynchronizer (which see for - * further description). - */ - private void cancelWait(Aux node) { - outer: for (Aux a; (a = aux) != null && a.ex == null; ) { - for (Aux trail = null;;) { - Aux next = a.next; - if (a == node) { - if (trail != null) - trail.casNext(trail, next); - else if (casAux(a, next)) - break outer; // cannot be re-encountered - break; // restart - } else { - trail = a; - if ((a = next) == null) - break outer; - } - } + boolean set = false, installed = false; + if ((s = status) >= 0) { + Aux h = new Aux(Thread.currentThread(), ex), p = null, a; + do { + if (!installed && ((a = aux) == null || a.ex == null) && + (installed = casAux(a, h))) + p = a; // list of waiters replaced by h + if (installed && (set = casStatus(s, s | HAVE_EXCEPTION))) + break; + } while ((s = status) >= 0); + for (; p != null; p = p.next) + LockSupport.unpark(p.thread); } + return set; } /** - * Records exception unless already done. Overridable in subclasses. - * - * @return status on exit + * Tries to set exception, if so invoking onFJTExceptionSet */ - int trySetException(Throwable ex) { - return trySetThrown(ex); + final void trySetException(Throwable ex) { + if (trySetThrown(ex)) + onFJTExceptionSet(ex); } - /** - * Constructor for subclasses to call. - */ - public ForkJoinTask() {} - - static boolean isExceptionalStatus(int s) { // needed by subclasses - return (s & THROWN) != 0; - } - - /** - * Unless done, calls exec and records status if completed, but - * doesn't wait for completion otherwise. + /* + * Waits for signal, interrupt, timeout, or pool termination. * - * @return status on exit from this method + * @param pool if nonnull, the pool of ForkJoinWorkerThread caller + * @param how flags for TIMED, INTERRUPTIBLE, UNCOMPENSATE + * @param deadline if timed, timeout deadline + * @return ABNORMAL if interrupted, 0 on timeout, else status on exit */ - final int doExec() { - int s; boolean completed; - if ((s = status) >= 0) { - try { - completed = exec(); - } catch (Throwable rex) { - s = trySetException(rex); - completed = false; + private int awaitDone(ForkJoinPool pool, int how, long deadline) { + int s = 0; + try { + if (pool != null && pool.stopping()) // recheck below on interrupt + cancelIgnoringExceptions(this); + boolean queued = false; + Aux node = null; + for (Aux a;;) { // install node + if ((s = status) < 0) + break; + else if (node == null) + node = new Aux(Thread.currentThread(), null); + else if (((a = aux) == null || a.ex == null) && + (queued = casAux(node.next = a, node))) + break; } - if (completed) - s = setDone(); + if (queued) { // await signal or interrupt + int interruptStatus = 0; // < 0 : throw, > 0 : re-interrupt + LockSupport.setCurrentBlocker(this); + for (;;) { + if ((s = status) < 0) + break; + else if (interruptStatus < 0) { + s = ABNORMAL; // interrupted and not done + break; + } + else if (Thread.interrupted()) { + if (pool == null || !pool.stopping()) + interruptStatus = + (how & INTERRUPTIBLE) != 0 ? -1 : 1; + else { + interruptStatus = 1; // re-assert if cleared + cancelIgnoringExceptions(this); + } + } + else if ((how & TIMED) != 0) { + long ns = deadline - System.nanoTime(); + if (ns <= 0L) { + s = 0; + break; + } + LockSupport.parkNanos(ns); + } + else + LockSupport.park(); + } + if (s >= 0) { + // cancellation similar to AbstractQueuedSynchronizer + outer: for (Aux a; (a = aux) != null && a.ex == null; ) { + for (Aux trail = null;;) { + Aux next = a.next; + if (a == node) { + if (trail != null) + trail.casNext(trail, next); + else if (casAux(a, next)) + break outer; // cannot be re-encountered + break; // restart + } else { + trail = a; + if ((a = next) == null) + break outer; + } + } + } + } + else + signalWaiters(); // help clean or signal + LockSupport.setCurrentBlocker(null); + if (interruptStatus > 0) + Thread.currentThread().interrupt(); + } + } finally { + if ((how & UNCOMPENSATE) != 0 && pool != null) + pool.uncompensate(); } return s; } /** - * Helps and/or waits for completion from join, get, or invoke; - * called from either internal or external threads. + * Tries applicable helping steps while joining this task, + * otherwise invokes awaitDone * - * @param how flags for RAN, INTERRUPTIBLE, TIMED - * @param deadline if timed, timeout deadline + * @param interruptible true if wait interruptible + * @param how flags for TIMED, INTERRUPTIBLE * @return ABNORMAL if interrupted, else status on exit */ - private int awaitDone(int how, long deadline) { - int s; ForkJoinWorkerThread wt; ForkJoinPool p; ForkJoinPool.WorkQueue q; - boolean internal; - Thread t = Thread.currentThread(); - if (internal = (t instanceof ForkJoinWorkerThread)) { + private int awaitJoin(int how, long deadline) { + ForkJoinWorkerThread wt; ForkJoinPool p; ForkJoinPool.WorkQueue q; + int s; boolean internal; Thread t; + if (internal = + (t = Thread.currentThread()) instanceof ForkJoinWorkerThread) { q = (wt = (ForkJoinWorkerThread)t).workQueue; p = wt.pool; } else - q = ((p = ForkJoinPool.common) != null) ? p.externalQueue() : null; - if ((s = tryHelpJoin(p, q, internal, how)) < 0) + q = ForkJoinPool.externalQueue(p = ForkJoinPool.common); + if ((s = tryHelpJoinFJT(p, q, internal)) < 0) return s; - int compensation = s; // used below after waiting - int interruptStatus = 0; // < 0 for throw, > 0 for reinterrupt - boolean queued = false; - Aux node = null; - for (;;) { // install node and await signal - Aux a; long ns; - if ((s = status) < 0) - break; - else if (interruptStatus < 0) { - s = ABNORMAL; // interrupted and not done - break; - } - else if (Thread.interrupted()) { - interruptStatus = (how & INTERRUPTIBLE) != 0 ? -1 : 1; - if (p != null && q != null && p.runState < 0) - q.cancelRemainingTasks(this); // terminating - } - else if (!queued) { - if (node == null) - node = new Aux(Thread.currentThread(), null); - if (((a = aux) == null || a.ex == null) && - (queued = casAux(node.next = a, node))) - LockSupport.setCurrentBlocker(this); - } - else if ((how & TIMED) != 0) { - if ((ns = deadline - System.nanoTime()) <= 0L) { - s = 0; - break; - } - LockSupport.parkNanos(ns); - } - else - LockSupport.park(); - } - if (queued) { - LockSupport.setCurrentBlocker(null); - if (s >= 0) - cancelWait(node); - else - signalWaiters(); // help clean or signal - } - if (compensation == UNCOMPENSATE && p != null) - p.uncompensate(); - if (interruptStatus > 0) - Thread.currentThread().interrupt(); - return s; - } - - /** - * Tries to locate and run this task or its subtasks. By default, - * applicable to possibly recursively structured tasks. Overridden - * in jdk subclasses that use different rules or mechanics. - * @return status if done or UNCOMPENSATE to uncompensate after waiting - */ - int tryHelpJoin(ForkJoinPool p, ForkJoinPool.WorkQueue q, - boolean internal, int how) { - int s = 0; - if (q != null && p != null && - ((how & RAN) != 0 || (s = q.tryRemoveAndExec(this, internal)) >= 0) && - internal) - s = p.helpJoin(this, q, (how & TIMED) != 0); - return s; + else if (s == UNCOMPENSATE) + how |= s; + else if ((s = status) < 0) + return s; + return awaitDone(p, how, deadline); } /** - * Cancels, ignoring any exceptions thrown by cancel. Cancel is - * spec'ed not to throw any exceptions, but if it does anyway, we - * have no recourse, so guard against this case. + * Runs a task body: Unless done, calls exec and records status if + * completed, but doesn't wait for completion otherwise. */ - static final void cancelIgnoringExceptions(ForkJoinTask t) { - if (t != null) { + final void doExec() { + boolean completed = false; + if (status >= 0) { try { - t.cancel(true); - } catch (Throwable ignore) { + completed = exec(); + } catch (Throwable rex) { + trySetException(rex); } + if (completed) + setDone(); } } - /* Similarly, for interrupts */ - static final void interruptIgnoringExceptions(Thread t) { - if (t != null) { - try { - t.interrupt(); - } catch (Throwable ignore) { - } - } - } + // Reporting Exceptions /** * Returns a rethrowable exception for this task, if available. @@ -556,7 +518,7 @@ static final void interruptIgnoringExceptions(Thread t) { */ private Throwable getThrowableException() { Throwable ex; Aux a; - if ((a = aux) == null) + if ((status & HAVE_EXCEPTION) != HAVE_EXCEPTION || (a = aux) == null) ex = null; else if ((ex = a.ex) != null && a.thread != Thread.currentThread()) { try { @@ -601,14 +563,6 @@ private void reportException(int s) { ForkJoinTask.uncheckedThrow(getThrowableException()); } - /** - * Optional wrapping for cancellation in reportExecutionException. - * (Overridden by ExecutorServiceTasks.) - */ - Throwable wrapCancellationException(CancellationException cx) { - return cx; - } - /** * Throws exception for (timed or untimed) get, wrapping if * necessary in an ExecutionException. @@ -622,7 +576,7 @@ else if (s >= 0) else if ((rx = getThrowableException()) != null) ex = new ExecutionException(rx); else - ex = wrapCancellationException(new CancellationException()); + ex = wrapFJTCancellationException(new CancellationException()); ForkJoinTask.uncheckedThrow(ex); } @@ -647,8 +601,86 @@ void uncheckedThrow(Throwable t) throws T { throw (T)t; // rely on vacuous cast } + // Utilities shared among ForkJoinTask, ForkJoinPool + + /** + * Sets MARKER bit, returning nonzero if previously set + */ + final int setForkJoinTaskStatusMarkerBit() { + return getAndBitwiseOrStatus(MARKER) & MARKER; + } + + /** + * Returns nonzero if MARKER bit set. + */ + final int getForkJoinTaskStatusMarkerBit() { + return status & MARKER; + } + + /** + * Cancels, ignoring any exceptions thrown by cancel. Cancel is + * spec'ed not to throw any exceptions, but if it does anyway, we + * have no recourse, so guard against this case. + */ + static final void cancelIgnoringExceptions(ForkJoinTask t) { + if (t != null) { + try { + t.cancel(true); + } catch (Throwable ignore) { + } + } + } + + /** Similarly, for interrupts */ + static final void interruptIgnoringExceptions(Thread t) { + if (t != null) { + try { + t.interrupt(); + } catch (Throwable ignore) { + } + } + } + + // Methods overridden only in jdk subclasses + + /** + * Default helping strategy, suitable for most possibly-recursive + * tasks/subtasks. + */ + int tryHelpJoinFJT(ForkJoinPool p, ForkJoinPool.WorkQueue q, + boolean internal) { + return (p == null || q == null) ? 0 : p.helpJoin(this, q, internal); + } + + /** + * Returns true if the given task is in this tasks completion scope. + * (Overridden in class CountedCompleter.) + */ + boolean canHelpCompleteFJT(ForkJoinTask t) { + return t == this; + } + + /** + * Overridable action on setting exception + */ + void onFJTExceptionSet(Throwable ex) { + } + + /** + * Optional wrapping for cancellation in reportExecutionException. + * (Overridden in class ExternalTask.) + */ + Throwable wrapFJTCancellationException(CancellationException cx) { + return cx; + } + // public methods + /** + * Constructor for subclasses to call. + */ + public ForkJoinTask() {} + /** * Arranges to asynchronously execute this task in the pool the * current task is running in, if applicable, or using the {@link @@ -667,7 +699,6 @@ void uncheckedThrow(Throwable t) throws T { public final ForkJoinTask fork() { Thread t; ForkJoinWorkerThread wt; ForkJoinPool p; ForkJoinPool.WorkQueue q; - U.storeStoreFence(); // ensure safely publishable if ((t = Thread.currentThread()) instanceof ForkJoinWorkerThread) { p = (wt = (ForkJoinWorkerThread)t).pool; q = wt.workQueue; @@ -692,7 +723,7 @@ public final ForkJoinTask fork() { public final V join() { int s; if ((s = status) >= 0) - s = awaitDone(0, 0L); + s = awaitJoin(0, 0L); if ((s & ABNORMAL) != 0) reportException(s); return getRawResult(); @@ -708,8 +739,9 @@ public final V join() { */ public final V invoke() { int s; - if ((s = doExec()) >= 0) - s = awaitDone(RAN, 0L); + doExec(); + if ((s = status) >= 0) + s = awaitJoin(0, 0L); if ((s & ABNORMAL) != 0) reportException(s); return getRawResult(); @@ -737,15 +769,16 @@ public static void invokeAll(ForkJoinTask t1, ForkJoinTask t2) { if (t1 == null || t2 == null) throw new NullPointerException(); t2.fork(); - if ((s1 = t1.doExec()) >= 0) - s1 = t1.awaitDone(RAN, 0L); + t1.doExec(); + if ((s1 = t1.status) >= 0) + s1 = t1.awaitJoin(0, 0L); if ((s1 & ABNORMAL) != 0) { cancelIgnoringExceptions(t2); t1.reportException(s1); } else { if ((s2 = t2.status) >= 0) - s2 = t2.awaitDone(0, 0L); + s2 = t2.awaitJoin(0, 0L); if ((s2 & ABNORMAL) != 0) t2.reportException(s2); } @@ -777,8 +810,9 @@ public static void invokeAll(ForkJoinTask... tasks) { } if (i == 0) { int s; - if ((s = t.doExec()) >= 0) - s = t.awaitDone(RAN, 0L); + t.doExec(); + if ((s = t.status) >= 0) + s = t.awaitJoin(0, 0L); if ((s & ABNORMAL) != 0) ex = t.getException(s); break; @@ -791,7 +825,7 @@ public static void invokeAll(ForkJoinTask... tasks) { if ((t = tasks[i]) != null) { int s; if ((s = t.status) >= 0) - s = t.awaitDone(0, 0L); + s = t.awaitJoin(0, 0L); if ((s & ABNORMAL) != 0 && (ex = t.getException(s)) != null) break; } @@ -840,8 +874,9 @@ public static > Collection invokeAll(Collection } if (i == 0) { int s; - if ((s = t.doExec()) >= 0) - s = t.awaitDone(RAN, 0L); + t.doExec(); + if ((s = t.status) >= 0) + s = t.awaitJoin(0, 0L); if ((s & ABNORMAL) != 0) ex = t.getException(s); break; @@ -854,7 +889,7 @@ public static > Collection invokeAll(Collection if ((t = ts.get(i)) != null) { int s; if ((s = t.status) >= 0) - s = t.awaitDone(0, 0L); + s = t.awaitJoin(0, 0L); if ((s & ABNORMAL) != 0 && (ex = t.getException(s)) != null) break; } @@ -946,9 +981,10 @@ public V resultNow() { @Override public Throwable exceptionNow() { - if ((status & (ABNORMAL | THROWN)) != (ABNORMAL | THROWN)) + Throwable ex; + if ((ex = getThrowableException()) == null) throw new IllegalStateException(); - return getThrowableException(); + return ex; } /** @@ -1034,7 +1070,7 @@ public final V get() throws InterruptedException, ExecutionException { if (Thread.interrupted()) s = ABNORMAL; else - s = awaitDone(INTERRUPTIBLE, 0L); + s = awaitJoin(INTERRUPTIBLE, 0L); } if ((s & ABNORMAL) != 0) reportExecutionException(s); @@ -1063,7 +1099,7 @@ public final V get(long timeout, TimeUnit unit) if (Thread.interrupted()) s = ABNORMAL; else if (nanos > 0L) - s = awaitDone(INTERRUPTIBLE | TIMED, nanos + System.nanoTime()); + s = awaitJoin(INTERRUPTIBLE | TIMED, nanos + System.nanoTime()); } if (s >= 0 || (s & ABNORMAL) != 0) reportExecutionException(s); @@ -1079,7 +1115,7 @@ else if (nanos > 0L) public final void quietlyJoin() { int s; if ((s = status) >= 0) - awaitDone(0, 0L); + awaitJoin(0, 0L); } /** @@ -1088,9 +1124,9 @@ public final void quietlyJoin() { * exception. */ public final void quietlyInvoke() { - int s; - if ((s = doExec()) >= 0) - awaitDone(RAN, 0L); + doExec(); + if (status >= 0) + awaitJoin(0, 0L); } /** @@ -1113,7 +1149,7 @@ public final boolean quietlyJoin(long timeout, TimeUnit unit) if (Thread.interrupted()) s = ABNORMAL; else if (nanos > 0L) - s = awaitDone(INTERRUPTIBLE | TIMED, nanos + System.nanoTime()); + s = awaitJoin(INTERRUPTIBLE | TIMED, nanos + System.nanoTime()); } if (s == ABNORMAL) throw new InterruptedException(); @@ -1135,7 +1171,7 @@ public final boolean quietlyJoinUninterruptibly(long timeout, int s; long nanos = unit.toNanos(timeout); if ((s = status) >= 0 && nanos > 0L) - s = awaitDone(TIMED, nanos + System.nanoTime()); + s = awaitJoin(TIMED, nanos + System.nanoTime()); return (s < 0); } @@ -1410,7 +1446,101 @@ public final boolean compareAndSetForkJoinTaskTag(short expect, short update) { } } - // Classes wrapping Runnables and Callables + // Factory methods for adaptors below + + /** + * Returns a new {@code ForkJoinTask} that performs the {@code run} + * method of the given {@code Runnable} as its action, and returns + * a null result upon {@link #join}. + * + * @param runnable the runnable action + * @return the task + */ + public static ForkJoinTask adapt(Runnable runnable) { + return new AdaptedRunnableAction(runnable); + } + + /** + * Returns a new {@code ForkJoinTask} that performs the {@code run} + * method of the given {@code Runnable} as its action, and returns + * the given result upon {@link #join}. + * + * @param runnable the runnable action + * @param result the result upon completion + * @param the type of the result + * @return the task + */ + public static ForkJoinTask adapt(Runnable runnable, T result) { + return new AdaptedRunnable(runnable, result); + } + + /** + * Returns a new {@code ForkJoinTask} that performs the {@code call} + * method of the given {@code Callable} as its action, and returns + * its result upon {@link #join}, translating any checked exceptions + * encountered into {@code RuntimeException}. + * + * @param callable the callable action + * @param the type of the callable's result + * @return the task + */ + public static ForkJoinTask adapt(Callable callable) { + return new AdaptedCallable(callable); + } + + /** + * Returns a new {@code ForkJoinTask} that performs the {@code call} + * method of the given {@code Callable} as its action, and returns + * its result upon {@link #join}, translating any checked exceptions + * encountered into {@code RuntimeException}. Additionally, + * invocations of {@code cancel} with {@code mayInterruptIfRunning + * true} will attempt to interrupt the thread performing the task. + * + * @param callable the callable action + * @param the type of the callable's result + * @return the task + * + * @since 19 + */ + public static ForkJoinTask adaptInterruptible(Callable callable) { + return new AdaptedInterruptibleCallable(callable); + } + + // Serialization support + + private static final long serialVersionUID = -7721805057305804111L; + + /** + * Saves this task to a stream (that is, serializes it). + * + * @param s the stream + * @throws java.io.IOException if an I/O error occurs + * @serialData the current run status and the exception thrown + * during execution, or {@code null} if none + */ + private void writeObject(java.io.ObjectOutputStream s) + throws java.io.IOException { + Aux a; + s.defaultWriteObject(); + s.writeObject((a = aux) == null ? null : a.ex); + } + + /** + * Reconstitutes this task from a stream (that is, deserializes it). + * @param s the stream + * @throws ClassNotFoundException if the class of a serialized object + * could not be found + * @throws java.io.IOException if an I/O error occurs + */ + private void readObject(java.io.ObjectInputStream s) + throws java.io.IOException, ClassNotFoundException { + s.defaultReadObject(); + Object ex = s.readObject(); + if (ex != null) + aux = new Aux(Thread.currentThread(), (Throwable)ex); + } + + // Special subclasses for adaptors and internal tasks /** * Adapter for Runnables. This implements RunnableFuture @@ -1435,10 +1565,6 @@ static final class AdaptedRunnable extends ForkJoinTask public String toString() { return super.toString() + "[Wrapped task = " + runnable + "]"; } - final int tryHelpJoin(ForkJoinPool p, ForkJoinPool.WorkQueue q, - boolean internal, int how) { - return internal ? super.tryHelpJoin(p, q, true, how) : 0; - } private static final long serialVersionUID = 5232453952276885070L; } @@ -1460,10 +1586,6 @@ public final void setRawResult(Void v) { } public String toString() { return super.toString() + "[Wrapped task = " + runnable + "]"; } - final int tryHelpJoin(ForkJoinPool p, ForkJoinPool.WorkQueue q, - boolean internal, int how) { - return internal ? super.tryHelpJoin(p, q, true, how) : 0; - } private static final long serialVersionUID = 5232453952276885070L; } @@ -1509,7 +1631,7 @@ public String toString() { * has not yet been cancelled on entry and might not otherwise be * cancelled by others. */ - static abstract class ExecutorServiceTask extends ForkJoinTask + static abstract class ExternalTask extends ForkJoinTask implements RunnableFuture { transient volatile Thread runner; abstract T compute() throws Exception; @@ -1517,7 +1639,7 @@ static boolean isTerminating(Thread t) { ForkJoinPool p; return ((t instanceof ForkJoinWorkerThread) && (p = ((ForkJoinWorkerThread) t).pool) != null && - p.runState < 0); + p.stopping()); } public final boolean exec() { Thread t = Thread.currentThread(); @@ -1530,7 +1652,7 @@ public final boolean exec() { if (status >= 0) setRawResult(compute()); } catch (Exception ex) { - trySetException(ex); + trySetException(ex); } finally { runner = null; } @@ -1538,27 +1660,25 @@ public final boolean exec() { return true; } public boolean cancel(boolean mayInterruptIfRunning) { - int s = trySetCancelled(); - if (s >= 0 && mayInterruptIfRunning) + if (trySetCancelled() >= 0 && mayInterruptIfRunning) interruptIgnoringExceptions(runner); - return (s >= 0 || (s & (ABNORMAL | THROWN)) == ABNORMAL); - } - final int tryHelpJoin(ForkJoinPool p, ForkJoinPool.WorkQueue q, - boolean internal, int how) { - return internal ? super.tryHelpJoin(p, q, true, how) : 0; + return isCancelled(); } - Throwable wrapCancellationException(CancellationException cx) { + public final void run() { invoke(); } + final Throwable wrapFJTCancellationException(CancellationException cx) { return new ExecutionException(cx); } - public final void run() { invoke(); } + final int tryHelpJoinFJT(ForkJoinPool p, ForkJoinPool.WorkQueue q, + boolean internal) { + return internal ? super.tryHelpJoinFJT(p, q, true) : 0; + } private static final long serialVersionUID = 2838392045355241008L; } /** * Adapter for Callable-based interruptible tasks. */ - static final class AdaptedInterruptibleCallable - extends ExecutorServiceTask { + static final class AdaptedInterruptibleCallable extends ExternalTask { @SuppressWarnings("serial") // Conditionally serializable final Callable callable; @SuppressWarnings("serial") // Conditionally serializable @@ -1579,8 +1699,7 @@ public String toString() { /** * Adapter for Runnable-based interruptible tasks. */ - static final class AdaptedInterruptibleRunnable - extends ExecutorServiceTask { + static final class AdaptedInterruptibleRunnable extends ExternalTask { @SuppressWarnings("serial") // Conditionally serializable final Runnable runnable; @SuppressWarnings("serial") // Conditionally serializable @@ -1602,8 +1721,7 @@ public String toString() { /** * Adapter for Runnables in which failure forces worker exception. */ - static final class RunnableExecuteAction - extends ExecutorServiceTask { + static final class RunnableExecuteAction extends ExternalTask { @SuppressWarnings("serial") // Conditionally serializable final Runnable runnable; RunnableExecuteAction(Runnable runnable) { @@ -1613,17 +1731,15 @@ static final class RunnableExecuteAction public final Void getRawResult() { return null; } public final void setRawResult(Void v) { } final Void compute() { runnable.run(); return null; } - int trySetException(Throwable ex) { // if a handler, invoke it - int s; Thread t; java.lang.Thread.UncaughtExceptionHandler h; - if (isExceptionalStatus(s = trySetThrown(ex)) && - (h = ((t = Thread.currentThread()). + void onFJTExceptionSet(Throwable ex) { // if a handler, invoke it + Thread t; java.lang.Thread.UncaughtExceptionHandler h; + if ((h = ((t = Thread.currentThread()). getUncaughtExceptionHandler())) != null) { try { h.uncaughtException(t, ex); } catch (Throwable ignore) { } } - return s; } public String toString() { return super.toString() + "[Wrapped task = " + runnable + "]"; @@ -1631,98 +1747,6 @@ public String toString() { private static final long serialVersionUID = 5232453952276885070L; } - /** - * Returns a new {@code ForkJoinTask} that performs the {@code run} - * method of the given {@code Runnable} as its action, and returns - * a null result upon {@link #join}. - * - * @param runnable the runnable action - * @return the task - */ - public static ForkJoinTask adapt(Runnable runnable) { - return new AdaptedRunnableAction(runnable); - } - - /** - * Returns a new {@code ForkJoinTask} that performs the {@code run} - * method of the given {@code Runnable} as its action, and returns - * the given result upon {@link #join}. - * - * @param runnable the runnable action - * @param result the result upon completion - * @param the type of the result - * @return the task - */ - public static ForkJoinTask adapt(Runnable runnable, T result) { - return new AdaptedRunnable(runnable, result); - } - - /** - * Returns a new {@code ForkJoinTask} that performs the {@code call} - * method of the given {@code Callable} as its action, and returns - * its result upon {@link #join}, translating any checked exceptions - * encountered into {@code RuntimeException}. - * - * @param callable the callable action - * @param the type of the callable's result - * @return the task - */ - public static ForkJoinTask adapt(Callable callable) { - return new AdaptedCallable(callable); - } - - /** - * Returns a new {@code ForkJoinTask} that performs the {@code call} - * method of the given {@code Callable} as its action, and returns - * its result upon {@link #join}, translating any checked exceptions - * encountered into {@code RuntimeException}. Additionally, - * invocations of {@code cancel} with {@code mayInterruptIfRunning - * true} will attempt to interrupt the thread performing the task. - * - * @param callable the callable action - * @param the type of the callable's result - * @return the task - * - * @since 19 - */ - public static ForkJoinTask adaptInterruptible(Callable callable) { - return new AdaptedInterruptibleCallable(callable); - } - - // Serialization support - - private static final long serialVersionUID = -7721805057305804111L; - - /** - * Saves this task to a stream (that is, serializes it). - * - * @param s the stream - * @throws java.io.IOException if an I/O error occurs - * @serialData the current run status and the exception thrown - * during execution, or {@code null} if none - */ - private void writeObject(java.io.ObjectOutputStream s) - throws java.io.IOException { - Aux a; - s.defaultWriteObject(); - s.writeObject((a = aux) == null ? null : a.ex); - } - - /** - * Reconstitutes this task from a stream (that is, deserializes it). - * @param s the stream - * @throws ClassNotFoundException if the class of a serialized object - * could not be found - * @throws java.io.IOException if an I/O error occurs - */ - private void readObject(java.io.ObjectInputStream s) - throws java.io.IOException, ClassNotFoundException { - s.defaultReadObject(); - Object ex = s.readObject(); - if (ex != null) - trySetThrown((Throwable)ex); - } - static { U = Unsafe.getUnsafe(); STATUS = U.objectFieldOffset(ForkJoinTask.class, "status"); From d2c0742870420d4cd9667a6795131b5ff597b3b1 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Mon, 20 Mar 2023 13:44:21 -0400 Subject: [PATCH 13/61] Avoid more performance regressions --- .../util/concurrent/CountedCompleter.java | 22 +- .../java/util/concurrent/ForkJoinPool.java | 270 +++++++++--------- .../java/util/concurrent/ForkJoinTask.java | 171 ++++++----- 3 files changed, 223 insertions(+), 240 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java b/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java index 74ada2239f380..b664ce3509435 100644 --- a/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java +++ b/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java @@ -729,7 +729,8 @@ public final void quietlyCompleteRoot() { */ public final void helpComplete(int maxTasks) { ForkJoinPool.WorkQueue q; Thread t; boolean internal; - if (internal = (t = Thread.currentThread()) instanceof ForkJoinWorkerThread) + if (internal = + (t = Thread.currentThread()) instanceof ForkJoinWorkerThread) q = ((ForkJoinWorkerThread)t).workQueue; else q = ForkJoinPool.commonQueue(); @@ -739,16 +740,6 @@ public final void helpComplete(int maxTasks) { // ForkJoinTask overrides - @Override - final boolean canHelpCompleteFJT(ForkJoinTask t) { - CountedCompleter f = null; - if (t instanceof CountedCompleter) { - f = (CountedCompleter)t; - do {} while (f != this && (f = f.completer) != null); - } - return f != null; - } - /** * Supports ForkJoinTask exception propagation. */ @@ -760,15 +751,6 @@ final void onFJTExceptionSet(Throwable ex) { a.trySetThrown(ex)); } - /** - * Helps complete subtasks while joining - */ - @Override - final int tryHelpJoinFJT(ForkJoinPool p, ForkJoinPool.WorkQueue q, - boolean internal) { - return (p == null) ? 0 : p.helpComplete(this, q, internal); - } - /** * Implements execution conventions for CountedCompleters. */ diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 2329ca44438d5..3b5f47a950994 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -989,12 +989,6 @@ public class ForkJoinPool extends AbstractExecutorService { * to interrupt all workers and cancel all tasks. It also uses a * CountDownLatch instead of a Condition for termination because * of lock change. - * * Refactorings to accommodate the increasing ways that tasks - * can be submitted, executed, joined. Includes reliance on new - * ForkJoinTask methods that can be overridden only by jdk - * subclasses (and with names including "FJT" to be - * extra-cautious about potential name clashes with external - * subclasses.) * * Many other adjustments to avoid performance regressions due * to the above. */ @@ -1033,12 +1027,6 @@ public class ForkJoinPool extends AbstractExecutorService { static final int SMASK = 0xffff; // short bits == max index static final int MAX_CAP = 0x7fff; // max #workers - 1 - // pool.runState bits - static final int STOP = 1 << 0; - static final int SHUTDOWN = 1 << 1; - static final int TERMINATED = 1 << 2; - static final int LOCKED = 1 << 3; - // {pool, workQueue}.config bits and sentinels static final int FIFO = 1 << 16; // fifo queue or access mode static final int ALIVE = 1 << 17; // initialized, not deregistered @@ -1049,45 +1037,6 @@ public class ForkJoinPool extends AbstractExecutorService { static final int UNCOMPENSATE = 1 << 16; // tryCompensate return - /* - * Bits and masks for ctl and bounds are packed with 4 16 bit subfields: - * RC: Number of released (unqueued) workers - * TC: Number of total workers - * SS: version count and status of top waiting thread - * ID: poolIndex of top of Treiber stack of waiters - * - * When convenient, we can extract the lower 32 stack top bits - * (including version bits) as sp=(int)ctl. When sp is non-zero, - * there are waiting workers. Count fields may be transiently - * negative during termination because of out-of-order updates. - * To deal with this, we use casts in and out of "short" and/or - * signed shifts to maintain signedness. Because it occupies - * uppermost bits, we can add one release count using getAndAdd of - * RC_UNIT, rather than CAS, when returning from a blocked join. - * Other updates of multiple subfields require CAS. - */ - - // Lower and upper word masks - static final long SP_MASK = 0xffffffffL; - static final long UC_MASK = ~SP_MASK; - // Release counts - static final int RC_SHIFT = 48; - static final long RC_UNIT = 0x0001L << RC_SHIFT; - static final long RC_MASK = 0xffffL << RC_SHIFT; - // Total counts - static final int TC_SHIFT = 32; - static final long TC_UNIT = 0x0001L << TC_SHIFT; - static final long TC_MASK = 0xffffL << TC_SHIFT; - // sp bits - static final int SS_SEQ = 1 << 16; // version count - static final int INACTIVE = 1 << 31; // phase or source bit when idle - static final int RESCAN = ~INACTIVE; // initial value for scan srcs - - // spin/yield/sleep control for runState locking and helpQuiesce - static final long MAX_SPINS = 1L << 6; // max calls to onSpinWait - static final long MIN_SLEEP = 1L << 10; // approx 1 usec as nanos - static final long MAX_SLEEP = 1L << 20; // approx 1 sec as nanos - // Static utilities /** @@ -1278,12 +1227,11 @@ final int queueSize() { * Pushes a task. Called only by owner or if already locked * * @param task the task. Caller must ensure non-null. - * @param pool the pool. Must be non-null unless terminating. + * @param p the pool. Must be non-null unless terminating. * @param signal true if signal when queue may have appeared empty * @throws RejectedExecutionException if array cannot be resized */ - final void push(ForkJoinTask task, ForkJoinPool pool, boolean signal) { - U.storeStoreFence(); // ensure safe publication + final void push(ForkJoinTask task, ForkJoinPool p, boolean signal) { ForkJoinTask[] a = array; int b = base, s = top++, cap, m; if (a != null && (cap = a.length) > 0) { @@ -1295,15 +1243,16 @@ final void push(ForkJoinTask task, ForkJoinPool pool, boolean signal) { if (a[m & (s - 1)] != null) signal = false; // not empty } - if (signal && pool != null) - pool.signalWork(); + if (signal && p != null) + p.signalWork(); } } /** * Grows the task array if possible and adds the task. */ - private void growAndPush(ForkJoinTask task, ForkJoinTask[] a, int s) { + private void growAndPush(ForkJoinTask task, ForkJoinTask[] a, + int s) { int cap; if (a != null && (cap = a.length) > 0) { // always true here int newCap = (cap < 1 << 24) ? cap << 2 : cap << 1; @@ -1356,7 +1305,7 @@ else if ((t = getAndClearSlot(a, (cap - 1) & b)) != null) { } } } while (p - b > 0); - U.storeFence(); // for timely index updates + U.storeStoreFence(); // for timely index updates } return t; } @@ -1436,7 +1385,7 @@ else if ((u = cmpExSlotToNull(a, k, t)) == t) { pool.signalWork(); // propagate return t; } - if (u == null && access == 0 && top == b) + if (u == null && access == 0 && top - b <= 0) break; // empty b = base; } @@ -1506,9 +1455,9 @@ final boolean tryRemoveAndExec(ForkJoinTask task, boolean internal) { ForkJoinTask[] a = array; int b = base, p = top, s = p - 1, d = p - b, cap; if (task != null && d > 0 && a != null && (cap = a.length) > 0) { - for (int m = cap - 1, i = s; ; --i) { + for (int i = s; ; --i) { ForkJoinTask t; int k; - if ((t = a[k = i & m]) == task) { + if ((t = a[k = i & (cap - 1)]) == task) { if (!internal) { if (getAndSetAccess(1) != 0) break; // fail if locked @@ -1524,7 +1473,7 @@ else if (getAndClearSlot(a, k) == null) else if (i == base) // act as poll base = i + 1; else // swap top - a[k] = getAndClearSlot(a, (top = s) & m); + a[k] = getAndClearSlot(a, (top = s) & (cap - 1)); releaseAccess(); task.doExec(); return true; @@ -1538,10 +1487,7 @@ else if (t == null || --d == 0) /** * Tries to pop and run tasks within the target's computation - * until done, not found, or limit exceeded. Note: The nominal - * param type here could just be ForkJoinTask, because it is - * only called in these contexts, but using CountedCompleter - * base type has been found to reduce warmup times. + * until done, not found, or limit exceeded. * * @param task root of computation * @param limit max runs, or zero for no limit @@ -1557,9 +1503,13 @@ final int helpComplete(CountedCompleter task, boolean internal, return status; if ((a = array) == null || (cap = a.length) <= 0 || (t = a[k = (cap - 1) & (s = (p = top) - 1)]) == null || - !task.canHelpCompleteFJT(t)) // screen tasks + !(t instanceof CountedCompleter)) break; - if (!internal) { + CountedCompleter f = (CountedCompleter)t; + do {} while (f != task && (f = f.completer) != null); + if (f == null) // screen tasks + break; + else if (!internal) { if (getAndSetAccess(1) != 0) break; // fail if locked if (top != p || cmpExSlotToNull(a, k, t) != t) { @@ -1589,7 +1539,8 @@ final void helpAsyncBlocker(ManagedBlocker blocker) { if (blocker != null) { for (;;) { int b = base, cap, k; ForkJoinTask[] a; - if ((a = array) == null || (cap = a.length) <= 0 || top - b <= 0) + if ((a = array) == null || (cap = a.length) <= 0 || + top - b <= 0) break; ForkJoinTask t = a[k = (cap - 1) & b]; U.loadFence(); @@ -1623,7 +1574,8 @@ else if (a[nk] == null) */ final boolean isApparentlyUnblocked() { Thread wt; Thread.State s; - return ((wt = owner) != null && phase >= 0 && (config & ALIVE) != 0 && + return ((wt = owner) != null && phase >= 0 && + (config & ALIVE) != 0 && (s = wt.getState()) != Thread.State.BLOCKED && s != Thread.State.WAITING && s != Thread.State.TIMED_WAITING); @@ -1677,6 +1629,53 @@ final void setClearThreadLocals() { */ static volatile RuntimePermission modifyThreadPermission; + // Bits and masks for pool control + + /* + * Bits and masks for ctl and bounds are packed with 4 16 bit subfields: + * RC: Number of released (unqueued) workers + * TC: Number of total workers + * SS: version count and status of top waiting thread + * ID: poolIndex of top of Treiber stack of waiters + * + * When convenient, we can extract the lower 32 stack top bits + * (including version bits) as sp=(int)ctl. When sp is non-zero, + * there are waiting workers. Count fields may be transiently + * negative during termination because of out-of-order updates. + * To deal with this, we use casts in and out of "short" and/or + * signed shifts to maintain signedness. Because it occupies + * uppermost bits, we can add one release count using getAndAdd of + * RC_UNIT, rather than CAS, when returning from a blocked join. + * Other updates of multiple subfields require CAS. + */ + + // Lower and upper word masks + private static final long SP_MASK = 0xffffffffL; + private static final long UC_MASK = ~SP_MASK; + // Release counts + private static final int RC_SHIFT = 48; + private static final long RC_UNIT = 0x0001L << RC_SHIFT; + private static final long RC_MASK = 0xffffL << RC_SHIFT; + // Total counts + private static final int TC_SHIFT = 32; + private static final long TC_UNIT = 0x0001L << TC_SHIFT; + private static final long TC_MASK = 0xffffL << TC_SHIFT; + + // sp bits (also used for source field) + private static final int SS_SEQ = 1 << 16; // version count + private static final int INACTIVE = 1 << 31; // phase/source bit if idle + private static final int RESCAN = ~INACTIVE; // initial value for srcs + + // pool.runState bits + private static final int STOP = 1 << 0; + private static final int SHUTDOWN = 1 << 1; + private static final int TERMINATED = 1 << 2; + private static final int LOCKED = 1 << 3; // lowest seqlock bit + + // spin/yield/sleep limits for runState locking and elsewhere; powers of 2 + private static final int SPIN_WAITS = 1 << 7; // max calls to onSpinWait + private static final int MIN_SLEEP = 1 << 10; // approx 1 usec as nanos + private static final int MAX_SLEEP = 1 << 20; // approx 1 sec as nanos // Instance fields volatile long stealCount; // collects worker nsteals @@ -1750,30 +1749,29 @@ private void acquireRunStateLock() { spinLockRunState(); } private void spinLockRunState() { // spin/yield/sleep - for (long waits = 0L;;) { - int s; + for (int waits = 0, s;;) { if (((s = runState) & LOCKED) == 0) { if (casRunState(s, s + LOCKED)) break; - waits = 0L; + waits = 0; } - else if (waits < MAX_SPINS) { + else if (waits < SPIN_WAITS) { Thread.onSpinWait(); ++waits; } - else if (waits == MAX_SPINS) { + else if (waits == SPIN_WAITS) { Thread.yield(); waits = MIN_SLEEP; } else { - LockSupport.parkNanos(this, waits); + LockSupport.parkNanos(this, (long)waits); if (waits < MAX_SLEEP) waits <<= 1; } } } - final boolean stopping() { return (runState & STOP) != 0; } // for ForkJoinTask + final boolean stopping() { return (runState & STOP) != 0; } // Creating, registering, and deregistering workers @@ -1843,7 +1841,8 @@ final void registerWorker(WorkQueue w) { try { if ((runState & STOP) == 0) { boolean resize = false; - if (qs == (qs = queues) && id >= 0 && id < n && qs[id] == null) + if (qs == (qs = queues) && id >= 0 && id < n && + qs[id] == null) qs[id] = w; else if (qs != null && (n = qs.length) > 0) { id = (seed << 1) | 1; // restart @@ -1892,7 +1891,7 @@ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { cfg = 0; else if (((cfg = w.config) & ALIVE) != 0) { w.config = cfg & ~ALIVE; - if (w.phase < 0) // possible if stopped before release + if (w.phase < 0) // possible if stopped before released reactivate(w); if (w.top - w.base > 0) { for (ForkJoinTask t; (t = w.nextLocalTask()) != null; ) @@ -1900,23 +1899,23 @@ else if (((cfg = w.config) & ALIVE) != 0) { } } long c = ctl; - if ((cfg & TRIMMED) == 0) // decrement counts + if ((cfg & TRIMMED) == 0) // decrement counts do {} while (c != (c = compareAndExchangeCtl( c, ((RC_MASK & (c - RC_UNIT)) | (TC_MASK & (c - TC_UNIT)) | (SP_MASK & c))))); - else if ((int)c == 0) // was dropped on timeout - cfg &= ~ALIVE; // suppress replacement if last + else if ((int)c == 0) // was dropped on timeout + cfg &= ~ALIVE; // suppress replacement if last if ((tryTerminate(false, false) & STOP) == 0 && w != null) { - WorkQueue[] qs; int n, i; // remove index unless terminating + WorkQueue[] qs; int n, i; // remove index unless terminating long ns = w.nsteals & 0xffffffffL; acquireRunStateLock(); if ((runState & STOP) == 0 && (qs = queues) != null && (n = qs.length) > 0 && qs[i = cfg & (n - 1)] == w) { qs[i] = null; - stealCount += ns; // accumulate steals + stealCount += ns; // accumulate steals } releaseRunStateLock(); if ((cfg & ALIVE) != 0) @@ -2093,19 +2092,19 @@ else if (awaitWork(w)) private int scan(WorkQueue w, int srcs, int r) { WorkQueue[] qs = queues; int n = (w == null || qs == null) ? 0 : qs.length; + int usrcs = ((srcs << 16) | 1) & RESCAN; // base of next srcs window for (int step = (r >>> 16) | 1, i = n; i > 0; --i, r += step) { int j, cap, b, k; WorkQueue q; ForkJoinTask[] a; - if ((q = qs[j = r & (n - 1)]) != null && + if ((q = qs[j = r & SMASK & (n - 1)]) != null && (a = q.array) != null && (cap = a.length) > 0) { ForkJoinTask t = a[k = (cap - 1) & (b = q.base)], u; U.loadFence(); // to re-read b and t - int nb = b + 1, nk = (cap - 1) & nb; - int qsrcs = ((j + 1) | (srcs << 16)) & RESCAN; + int nb = b + 1, nk = (cap - 1) & nb, qsrcs = usrcs + j; if (q.base != b) return srcs; // inconsistent if (t == null) { if (a[k] != null || a[nk] != null) - return qsrcs; + return qsrcs; // nonempty; restart } else if ((u = WorkQueue.cmpExSlotToNull(a, k, t)) == t) { q.base = nb; @@ -2131,7 +2130,6 @@ private boolean awaitWork(WorkQueue w) { if (w == null) return false; // currently impossible int p = (w.phase + SS_SEQ) & ~INACTIVE; // next phase number - int par = parallelism; long pc = ctl, sp = p & SP_MASK, qc; w.phase = p | INACTIVE; do { // enqueue @@ -2139,14 +2137,16 @@ private boolean awaitWork(WorkQueue w) { } while (pc != (pc = compareAndExchangeCtl( pc, qc = ((pc - RC_UNIT) & UC_MASK) | sp))); boolean idle = false; // true if pool idle - if ((qc & RC_MASK) <= 0L && !(idle = checkQuiescence())) + int active = (short)(qc >>> RC_SHIFT); + int nspins = active + (((short)(qc >>> TC_SHIFT)) << 2); + if (active <= 0 && !(idle = checkQuiescence())) reactivate(null); // ensure live if may be tasks if ((runState & STOP) != 0) return false; - for (int spins = (par << 2) + 0xf;;) { // avg #accesses in scan+signal - if (w.phase >= 0) // spin before block + for (int spins = nspins;;) { // spin before block + if (w.phase >= 0) return true; - if (--spins < 0) + if (--spins < 0) // avg #accesses in scan+signal break; Thread.onSpinWait(); } @@ -2182,16 +2182,18 @@ private boolean awaitWork(WorkQueue w) { * @param submissionsOnly if true, only scan submission queues */ private ForkJoinTask pollScan(boolean submissionsOnly) { - int r = ThreadLocalRandom.nextSecondarySeed(); - if (submissionsOnly) // even indices only - r &= ~1; - int step = (submissionsOnly) ? 2 : 1; - WorkQueue[] qs; int n; WorkQueue q; ForkJoinTask t; - if ((runState & STOP) == 0 && (qs = queues) != null && (n = qs.length) > 0) { - for (int i = n; i > 0; i -= step, r += step) { - if ((q = qs[r & (n - 1)]) != null && - (t = q.poll(this)) != null) - return t; + if ((runState & STOP) == 0) { + WorkQueue[] qs; int n; WorkQueue q; ForkJoinTask t; + int r = ThreadLocalRandom.nextSecondarySeed(); + if (submissionsOnly) // even indices only + r &= ~1; + int step = (submissionsOnly) ? 2 : 1; + if ((qs = queues) != null && (n = qs.length) > 0) { + for (int i = n; i > 0; i -= step, r += step) { + if ((q = qs[r & (n - 1)]) != null && + (t = q.poll(this)) != null) + return t; + } } } return null; @@ -2270,7 +2272,6 @@ final void uncompensate() { * * @param task the task * @param w caller's WorkQueue - * @param internal true if w is owned by a ForkJoinWorkerThread * @return task status on exit, or UNCOMPENSATE for compensated blocking */ final int helpJoin(ForkJoinTask task, WorkQueue w, boolean internal) { @@ -2294,10 +2295,10 @@ final int helpJoin(ForkJoinTask task, WorkQueue w, boolean internal) { else if ((runState & STOP) != 0) return 0; rescan = false; - int n = ((qs = queues) == null) ? 0 : qs.length, m = n - 1; + int n = ((qs = queues) == null) ? 0 : qs.length; scan: for (int i = n >>> 1; i > 0; --i, r += 2) { int j, cap; WorkQueue q; ForkJoinTask[] a; - if ((q = qs[j = r & m]) != null && + if ((q = qs[j = r & SMASK & (n - 1)]) != null && (a = q.array) != null && (cap = a.length) > 0) { for (int src = j + 1;;) { int sq = q.source, b = q.base, k = (cap - 1) & b; @@ -2311,7 +2312,7 @@ else if ((runState & STOP) != 0) break; } if (v == 0 || --d == 0 || - (p = qs[(v - 1) & m]) == null) { + (p = qs[(v - 1) & (n - 1)]) == null) { eligible = false; break; } @@ -2371,21 +2372,28 @@ final int helpComplete(CountedCompleter task, WorkQueue w, else if ((runState & STOP) != 0) return 0; rescan = locals = false; - int n = ((qs = queues) == null) ? 0 : qs.length, m = n - 1; + int n = ((qs = queues) == null) ? 0 : qs.length; scan: for (int i = n; i > 0; --i, ++r) { int j, cap; WorkQueue q; ForkJoinTask[] a; - if ((q = qs[j = r & m]) != null && + if ((q = qs[j = r & SMASK & (n - 1)]) != null && (a = q.array) != null && (cap = a.length) > 0) { for (int b = q.base, k;;) { ForkJoinTask t = a[k = (cap - 1) & b]; U.loadFence(); int nb = b + 1, nk = (cap - 1) & nb; - boolean eligible = task.canHelpCompleteFJT(t); + boolean eligible = false; + if (t instanceof CountedCompleter) { + CountedCompleter f = (CountedCompleter)t; + do {} while (f != task && (f = f.completer) != null); + if (f != null) + eligible = true; + } if ((s = task.status) < 0) return s; // validate if (b == (b = q.base) && a[k] == t) { if (t == null) { - rescan |= (a[nk] != null); // revisit + if (!rescan && a[nk] != null) + rescan = true; // revisit break; } if (!eligible) @@ -2419,9 +2427,9 @@ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { if (w == null || (phase = w.phase) < 0) return 0; int wsrc = w.source, r = w.config + 1, returnStatus = 1; + int activePhase = phase, inactivePhase = phase | INACTIVE, waits = 0; long maxSleep = Math.min(nanos >>> 8, MAX_SLEEP); // approx 1% nanos - long startTime = System.nanoTime(), waits = 0L; - int activePhase = phase, inactivePhase = phase | INACTIVE; + long startTime = System.nanoTime(); boolean locals = true; for (int prevState = -1, rs = runState; ; prevState = rs) { WorkQueue[] qs; WorkQueue q; @@ -2437,9 +2445,9 @@ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { u.doExec(); } boolean rescan = false, busy = false; - int n = ((qs = queues) == null) ? 0 : qs.length, m = n - 1; + int n = ((qs = queues) == null) ? 0 : qs.length; scan: for (int i = n, j; i > 0; --i, ++r) { - if ((q = qs[j = m & r]) != null && q != w) { + if ((q = qs[j = r & SMASK & (n - 1)]) != null && q != w) { for (int src = j + 1;;) { ForkJoinTask[] a = q.array; int b = q.base, cap, k; @@ -2479,20 +2487,20 @@ else if (!busy) break; else if (phase >= 0) { w.phase = phase = inactivePhase; - waits = 0L; // to recheck, then yield/sleep + waits = 0; // to recheck, then yield/sleep } else if (System.nanoTime() - startTime > nanos) { returnStatus = 0; // timed out break; } - else if (waits == 0L) // same as spinLockRunState except - waits = 1L; // with rescan instead of onSpinWait - else if (waits == 1L) { + else if (waits == 0) // same as spinLockRunState except + waits = 1 ; // with rescan instead of onSpinWait + else if (waits == 1) { waits = MIN_SLEEP; Thread.yield(); } else { - LockSupport.parkNanos(this, waits); + LockSupport.parkNanos(this, (long)waits); if (waits < maxSleep) waits <<= 1; } @@ -2509,9 +2517,11 @@ else if (waits == 1L) { * @return positive if quiescent, negative if interrupted, else 0 */ private int externalHelpQuiesce(long nanos, boolean interruptible) { + if (checkQuiescence()) + return 1; + long startTime = System.nanoTime(); long maxSleep = Math.min(nanos >>> 8, MAX_SLEEP); - long startTime = System.nanoTime(), waits = 0L; - for (;;) { // same structure as helpQuiesce, using poll vs scanning + for (int waits = 0;;) { // similar to helpQuiesce, using poll vs scanning ForkJoinTask t; if ((runState & STOP) != 0) return 1; @@ -2519,20 +2529,20 @@ else if (interruptible && Thread.interrupted()) return -1; else if ((t = pollScan(false)) != null) { t.doExec(); - waits = 0L; + waits = 0; } else if (checkQuiescence()) return 1; else if (System.nanoTime() - startTime > nanos) return 0; - else if (waits == 0L) - waits = 1L; - else if (waits == 1L) { + else if (waits == 0) + waits = 1; + else if (waits == 1) { waits = MIN_SLEEP; Thread.yield(); } else { - LockSupport.parkNanos(this, waits); + LockSupport.parkNanos(this, (long)waits); if (waits < maxSleep) waits <<= 1; } @@ -2617,6 +2627,7 @@ else if (isSubmit && (runState & SHUTDOWN) != 0) { private ForkJoinTask poolSubmit(boolean signalIfEmpty, boolean external, ForkJoinTask task) { + U.storeStoreFence(); // ensure safe publication if (task == null) throw new NullPointerException(); Thread t = Thread.currentThread(); ForkJoinWorkerThread wt = ((t instanceof ForkJoinWorkerThread) ? @@ -2758,9 +2769,10 @@ private int tryTerminate(boolean now, boolean enable) { WorkQueue w = ((current instanceof ForkJoinWorkerThread) ? ((ForkJoinWorkerThread)current).workQueue : null); int r = (w != null) ? w.config : 0; // stagger traversals - int n = ((qs = queues) == null) ? 0 : qs.length, m = n - 1; + int n = ((qs = queues) == null) ? 0 : qs.length; for (int i = n, j; i > 0; --i, ++r) { - if ((q = qs[j = m & r]) != null && (q.config & ALIVE) != 0) { + if ((q = qs[j = r & SMASK & (n - 1)]) != null && + (q.config & ALIVE) != 0) { if ((j & 1) != 0) ForkJoinTask.interruptIgnoringExceptions(q.owner); for (ForkJoinTask t; (t = q.poll(null)) != null; ) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index 96a3dca4a81d7..f65eccae12209 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -380,78 +380,80 @@ final void trySetException(Throwable ex) { */ private int awaitDone(ForkJoinPool pool, int how, long deadline) { int s = 0; - try { - if (pool != null && pool.stopping()) // recheck below on interrupt - cancelIgnoringExceptions(this); - boolean queued = false; - Aux node = null; - for (Aux a;;) { // install node - if ((s = status) < 0) - break; - else if (node == null) - node = new Aux(Thread.currentThread(), null); - else if (((a = aux) == null || a.ex == null) && - (queued = casAux(node.next = a, node))) - break; - } - if (queued) { // await signal or interrupt - int interruptStatus = 0; // < 0 : throw, > 0 : re-interrupt - LockSupport.setCurrentBlocker(this); - for (;;) { + if ((how & UNCOMPENSATE) != 0 || (s = status) >= 0) { + try { + if (pool != null && pool.stopping()) + cancelIgnoringExceptions(this); // recheck below on interrupt + boolean queued = false; + Aux node = null; + for (Aux a;;) { // install node if ((s = status) < 0) break; - else if (interruptStatus < 0) { - s = ABNORMAL; // interrupted and not done + else if (node == null) + node = new Aux(Thread.currentThread(), null); + else if (((a = aux) == null || a.ex == null) && + (queued = casAux(node.next = a, node))) break; - } - else if (Thread.interrupted()) { - if (pool == null || !pool.stopping()) - interruptStatus = - (how & INTERRUPTIBLE) != 0 ? -1 : 1; - else { - interruptStatus = 1; // re-assert if cleared - cancelIgnoringExceptions(this); - } - } - else if ((how & TIMED) != 0) { - long ns = deadline - System.nanoTime(); - if (ns <= 0L) { - s = 0; + } + if (queued) { // await signal or interrupt + int interruptStatus = 0; // < 0 : throw, > 0 : re-interrupt + LockSupport.setCurrentBlocker(this); + for (;;) { + if ((s = status) < 0) + break; + else if (interruptStatus < 0) { + s = ABNORMAL; // interrupted and not done break; } - LockSupport.parkNanos(ns); + else if (Thread.interrupted()) { + if (pool == null || !pool.stopping()) + interruptStatus = + (how & INTERRUPTIBLE) != 0 ? -1 : 1; + else { + interruptStatus = 1; // re-assert if cleared + cancelIgnoringExceptions(this); + } + } + else if ((how & TIMED) != 0) { + long ns = deadline - System.nanoTime(); + if (ns <= 0L) { + s = 0; + break; + } + LockSupport.parkNanos(ns); + } + else + LockSupport.park(); } - else - LockSupport.park(); - } - if (s >= 0) { - // cancellation similar to AbstractQueuedSynchronizer - outer: for (Aux a; (a = aux) != null && a.ex == null; ) { - for (Aux trail = null;;) { - Aux next = a.next; - if (a == node) { - if (trail != null) - trail.casNext(trail, next); - else if (casAux(a, next)) - break outer; // cannot be re-encountered - break; // restart - } else { - trail = a; - if ((a = next) == null) - break outer; + if (s >= 0) { + // cancellation similar to AbstractQueuedSynchronizer + outer: for (Aux a; (a = aux) != null && a.ex == null; ) { + for (Aux trail = null;;) { + Aux next = a.next; + if (a == node) { + if (trail != null) + trail.casNext(trail, next); + else if (casAux(a, next)) + break outer; // cannot be re-encountered + break; // restart + } else { + trail = a; + if ((a = next) == null) + break outer; + } } } } + else + signalWaiters(); // help clean or signal + LockSupport.setCurrentBlocker(null); + if (interruptStatus > 0) + Thread.currentThread().interrupt(); } - else - signalWaiters(); // help clean or signal - LockSupport.setCurrentBlocker(null); - if (interruptStatus > 0) - Thread.currentThread().interrupt(); + } finally { + if ((how & UNCOMPENSATE) != 0 && pool != null) + pool.uncompensate(); } - } finally { - if ((how & UNCOMPENSATE) != 0 && pool != null) - pool.uncompensate(); } return s; } @@ -461,12 +463,12 @@ else if (casAux(a, next)) * otherwise invokes awaitDone * * @param interruptible true if wait interruptible - * @param how flags for TIMED, INTERRUPTIBLE + * @param how flags for RAN, TIMED, INTERRUPTIBLE * @return ABNORMAL if interrupted, else status on exit */ private int awaitJoin(int how, long deadline) { ForkJoinWorkerThread wt; ForkJoinPool p; ForkJoinPool.WorkQueue q; - int s; boolean internal; Thread t; + boolean internal; Thread t; int s; if (internal = (t = Thread.currentThread()) instanceof ForkJoinWorkerThread) { q = (wt = (ForkJoinWorkerThread)t).workQueue; @@ -474,12 +476,18 @@ private int awaitJoin(int how, long deadline) { } else q = ForkJoinPool.externalQueue(p = ForkJoinPool.common); - if ((s = tryHelpJoinFJT(p, q, internal)) < 0) - return s; - else if (s == UNCOMPENSATE) - how |= s; - else if ((s = status) < 0) - return s; + if (q != null && p != null) { + if (this instanceof CountedCompleter) + s = p.helpComplete((CountedCompleter)this, q, internal); + else if (!(this instanceof ExternalTask) || internal) + s = p.helpJoin(this, q, internal); + else + s = 0; + if (s < 0) + return s; + if (s == UNCOMPENSATE) + how |= s; + } return awaitDone(p, how, deadline); } @@ -643,23 +651,6 @@ static final void interruptIgnoringExceptions(Thread t) { // Methods overridden only in jdk subclasses - /** - * Default helping strategy, suitable for most possibly-recursive - * tasks/subtasks. - */ - int tryHelpJoinFJT(ForkJoinPool p, ForkJoinPool.WorkQueue q, - boolean internal) { - return (p == null || q == null) ? 0 : p.helpJoin(this, q, internal); - } - - /** - * Returns true if the given task is in this tasks completion scope. - * (Overridden in class CountedCompleter.) - */ - boolean canHelpCompleteFJT(ForkJoinTask t) { - return t == this; - } - /** * Overridable action on setting exception */ @@ -699,6 +690,7 @@ public ForkJoinTask() {} public final ForkJoinTask fork() { Thread t; ForkJoinWorkerThread wt; ForkJoinPool p; ForkJoinPool.WorkQueue q; + U.storeStoreFence(); // ensure safe publication if ((t = Thread.currentThread()) instanceof ForkJoinWorkerThread) { p = (wt = (ForkJoinWorkerThread)t).pool; q = wt.workQueue; @@ -1246,7 +1238,8 @@ public static boolean inForkJoinPool() { */ public boolean tryUnfork() { Thread t; ForkJoinPool.WorkQueue q; boolean internal; - if (internal = (t = Thread.currentThread()) instanceof ForkJoinWorkerThread) + if (internal = + (t = Thread.currentThread()) instanceof ForkJoinWorkerThread) q = ((ForkJoinWorkerThread)t).workQueue; else q = ForkJoinPool.commonQueue(); @@ -1668,10 +1661,6 @@ public boolean cancel(boolean mayInterruptIfRunning) { final Throwable wrapFJTCancellationException(CancellationException cx) { return new ExecutionException(cx); } - final int tryHelpJoinFJT(ForkJoinPool p, ForkJoinPool.WorkQueue q, - boolean internal) { - return internal ? super.tryHelpJoinFJT(p, q, true) : 0; - } private static final long serialVersionUID = 2838392045355241008L; } From 815deccbe49319ac98ccf1128db5781481d5d481 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Fri, 31 Mar 2023 15:35:13 -0400 Subject: [PATCH 14/61] More compatibility updates --- .../java/util/concurrent/ForkJoinPool.java | 814 ++++++++---------- .../java/util/concurrent/ForkJoinTask.java | 489 +++++++---- 2 files changed, 679 insertions(+), 624 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 3b5f47a950994..dec57eb82a9c1 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -48,10 +48,8 @@ import java.util.Collections; import java.util.List; import java.util.function.Predicate; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.CountDownLatch; import java.util.concurrent.locks.LockSupport; -import java.util.concurrent.locks.AbstractQueuedSynchronizer; import jdk.internal.access.JavaUtilConcurrentFJPAccess; import jdk.internal.access.SharedSecrets; import jdk.internal.misc.Unsafe; @@ -212,7 +210,7 @@ public class ForkJoinPool extends AbstractExecutorService { * 2. Async (FIFO) mode and striped submission queues * 3. Completion-based tasks (mainly CountedCompleters) * 4. CommonPool and parallelStream support - * 5. ExternalTasks for externally submitted tasks + * 5. InterruptibleTasks for externally submitted tasks * * Most changes involve adaptions of base algorithms using * combinations of static and dynamic bitwise mode settings (both @@ -431,11 +429,13 @@ public class ForkJoinPool extends AbstractExecutorService { * properties detect changes and conditionally upgrade to * coordinate with some runState updates. It is typically held for * less than a dozen instructions unless the queue array is being - * resized, during which contention is rare. Still, to be - * conservative, acquireRunStateLock is implemented as a - * spin/yield/sleep loop. In addition to checking pool status, - * reads of runState serve as acquire fences before reading other - * non-volatile fields. + * resized, during which contention is rare. To be conservative, + * acquireRunStateLock is implemented as a spin/sleep loop. Here + * and elsewhere spin constants (SPIN_WAITS) are very short + * (currently 32) so have no discernable overhead even on systems + * with few available processors that may require blocking. In + * addition to checking pool status, reads of runState serve as + * acquire fences before reading other non-volatile fields. * * Array "queues" holds references to WorkQueues. It is updated * (only during worker creation and termination) under the @@ -625,12 +625,13 @@ public class ForkJoinPool extends AbstractExecutorService { * generated (which is often due to garbage collectors and other * activities). One way to ameliorate is for workers to rescan * multiple times, even when there are unlikely to be tasks. But - * this causes enough memory and CAS contention to prefer using - * quieter spinwaits in awaitWork; currently set to small values - * that only cover near-miss scenarios for deactivate vs activate - * races. Because idle workers are often not yet blocked (via - * LockSupport.park), we use the WorkQueue source field to - * advertise that a waiter actually needs unparking upon signal. + * this causes enough memory traffic and CAS contention to prefer + * using quieter short spinwaits in awaitWork and elsewhere. + * Those in awaitWork are set to small values that only cover + * near-miss scenarios for inactivate/activate races. Because idle + * workers are often not yet blocked (via LockSupport.park), we + * use the WorkQueue source field to advertise that a waiter + * actually needs unparking upon signal. * * Quiescence. Workers scan looking for work, giving up when they * don't find any, without being sure that none are available. @@ -706,7 +707,7 @@ public class ForkJoinPool extends AbstractExecutorService { * task would otherwise be blocked waiting for completion of * another, basically, just by running that task or one of its * subtasks if not already stolen. As described below, these - * mechanics are disabled for ExternalTasks, that guarantee + * mechanics are disabled for InterruptibleTasks, that guarantee * that callers do not executed submitted tasks. * * The basic structure of joining is an extended spin/block scheme @@ -848,7 +849,7 @@ public class ForkJoinPool extends AbstractExecutorService { * may be JVM-dependent and must access particular Thread class * fields to achieve this effect. * - * ExternalTasks + * InterruptibleTasks * ==================== * * Regular ForkJoinTasks manage task cancellation (method cancel) @@ -860,10 +861,10 @@ public class ForkJoinPool extends AbstractExecutorService { * returns immediately if the current thread is interrupted). * * To comply with ExecutorService specs, we use subclasses of - * abstract class ExternalTask for tasks that require + * abstract class InterruptibleTask for tasks that require * stronger interruption and cancellation guarantees. External * submitters never run these tasks, even if in the common pool. - * ExternalTasks include a "runner" field (implemented + * InterruptibleTasks include a "runner" field (implemented * similarly to FutureTask) to support cancel(true). Upon pool * shutdown, runners are interrupted so they can cancel. Since * external joining callers never run these tasks, they must await @@ -887,7 +888,7 @@ public class ForkJoinPool extends AbstractExecutorService { * requiring ForkJoinWorkerThreads in factories. There are * several constructions relying on this. However as of this * writing, virtual thread bodies are by default run as some form - * of ExternalTask. + * of InterruptibleTask. * * Memory placement * ================ @@ -965,7 +966,7 @@ public class ForkJoinPool extends AbstractExecutorService { * perform reasonably even when interpreted (not compiled). * * The order of declarations in this file is (with a few exceptions): - * (1) Static constants + * (1) Static configuration constants * (2) Static utility functions * (3) Nested (static) classes * (4) Fields, along with constants used when unpacking some of them @@ -979,7 +980,7 @@ public class ForkJoinPool extends AbstractExecutorService { * * The main sources of differences from previous version are: * - * * New abstract class ForkJoinTask.ExternalTask ensures + * * New abstract class ForkJoinTask.InterruptibleTask ensures * handling of tasks submitted under the ExecutorService * API consistent with specs. * * Method checkQuiescence() replaces previous quiescence-related @@ -1228,10 +1229,11 @@ final int queueSize() { * * @param task the task. Caller must ensure non-null. * @param p the pool. Must be non-null unless terminating. - * @param signal true if signal when queue may have appeared empty + * @param signal true if signal when queue may have been or appeared empty * @throws RejectedExecutionException if array cannot be resized */ - final void push(ForkJoinTask task, ForkJoinPool p, boolean signal) { + final void push(ForkJoinTask task, ForkJoinPool pool, + boolean signal) { ForkJoinTask[] a = array; int b = base, s = top++, cap, m; if (a != null && (cap = a.length) > 0) { @@ -1243,13 +1245,15 @@ final void push(ForkJoinTask task, ForkJoinPool p, boolean signal) { if (a[m & (s - 1)] != null) signal = false; // not empty } - if (signal && p != null) - p.signalWork(); + if (signal && pool != null) + pool.signalWork(); } } /** * Grows the task array if possible and adds the task. + * @param a the current task array + * @param s the old top value */ private void growAndPush(ForkJoinTask task, ForkJoinTask[] a, int s) { @@ -1266,10 +1270,10 @@ private void growAndPush(ForkJoinTask task, ForkJoinTask[] a, "Queue capacity exceeded"); } if (newCap > 0) { // always true here - int newMask = newCap - 1, k = s, n = cap, m = cap - 1; + int newMask = newCap - 1, k = s, b = k - cap, m = cap - 1; do { // poll old, push to new; exit if lose to other pollers - newArray[k-- & newMask] = task; - } while (n-- >= 0 && + newArray[k & newMask] = task; + } while (--k != b && (task = getAndClearSlot(a, k & m)) != null); U.storeFence(); array = newArray; @@ -1449,9 +1453,8 @@ final void topLevelExec(ForkJoinTask task, WorkQueue q) { /** * Deep form of tryUnpush: Traverses from top and removes and * runs task if present. - * @return true if successful */ - final boolean tryRemoveAndExec(ForkJoinTask task, boolean internal) { + final void tryRemoveAndExec(ForkJoinTask task, boolean internal) { ForkJoinTask[] a = array; int b = base, p = top, s = p - 1, d = p - b, cap; if (task != null && d > 0 && a != null && (cap = a.length) > 0) { @@ -1476,13 +1479,12 @@ else if (i == base) // act as poll a[k] = getAndClearSlot(a, (top = s) & (cap - 1)); releaseAccess(); task.doExec(); - return true; + break; } else if (t == null || --d == 0) break; } } - return false; } /** @@ -1493,10 +1495,9 @@ else if (t == null || --d == 0) * @param limit max runs, or zero for no limit * @return task status if known done; else 0 */ - final int helpComplete(CountedCompleter task, boolean internal, - int limit) { + final int helpComplete(ForkJoinTask task, boolean internal, int limit) { if (task != null) { - for (;;) { + outer: for (;;) { ForkJoinTask[] a; ForkJoinTask t; int status, p, s, cap, k; if ((status = task.status) < 0) @@ -1506,10 +1507,13 @@ final int helpComplete(CountedCompleter task, boolean internal, !(t instanceof CountedCompleter)) break; CountedCompleter f = (CountedCompleter)t; - do {} while (f != task && (f = f.completer) != null); - if (f == null) // screen tasks - break; - else if (!internal) { + for (int steps = cap;;) { // bound path + if (f == task) + break; + if ((f = f.completer) == null || --steps == 0) + break outer; + } + if (!internal) { if (getAndSetAccess(1) != 0) break; // fail if locked if (top != p || cmpExSlotToNull(a, k, t) != t) { @@ -1629,7 +1633,7 @@ final void setClearThreadLocals() { */ static volatile RuntimePermission modifyThreadPermission; - // Bits and masks for pool control + // Bits and masks for pool control fields /* * Bits and masks for ctl and bounds are packed with 4 16 bit subfields: @@ -1664,7 +1668,7 @@ final void setClearThreadLocals() { // sp bits (also used for source field) private static final int SS_SEQ = 1 << 16; // version count private static final int INACTIVE = 1 << 31; // phase/source bit if idle - private static final int RESCAN = ~INACTIVE; // initial value for srcs + private static final int ACTIVE = ~INACTIVE; // initial value for srcs // pool.runState bits private static final int STOP = 1 << 0; @@ -1672,10 +1676,10 @@ final void setClearThreadLocals() { private static final int TERMINATED = 1 << 2; private static final int LOCKED = 1 << 3; // lowest seqlock bit - // spin/yield/sleep limits for runState locking and elsewhere; powers of 2 - private static final int SPIN_WAITS = 1 << 7; // max calls to onSpinWait + // spin/sleep limits for runState locking and elsewhere; powers of 2 private static final int MIN_SLEEP = 1 << 10; // approx 1 usec as nanos private static final int MAX_SLEEP = 1 << 20; // approx 1 sec as nanos + private static final int SPIN_WAITS = 1 << 5; // calls to onSpinWait // Instance fields volatile long stealCount; // collects worker nsteals @@ -1748,7 +1752,7 @@ private void acquireRunStateLock() { if (((s = runState) & LOCKED) != 0 || !casRunState(s, s + LOCKED)) spinLockRunState(); } - private void spinLockRunState() { // spin/yield/sleep + private void spinLockRunState() { // spin/sleep for (int waits = 0, s;;) { if (((s = runState) & LOCKED) == 0) { if (casRunState(s, s + LOCKED)) @@ -1759,11 +1763,9 @@ else if (waits < SPIN_WAITS) { Thread.onSpinWait(); ++waits; } - else if (waits == SPIN_WAITS) { - Thread.yield(); - waits = MIN_SLEEP; - } else { + if (waits < MIN_SLEEP) + waits = MIN_SLEEP; LockSupport.parkNanos(this, (long)waits); if (waits < MAX_SLEEP) waits <<= 1; @@ -1771,7 +1773,9 @@ else if (waits == SPIN_WAITS) { } } - final boolean stopping() { return (runState & STOP) != 0; } + static boolean poolIsStopping(ForkJoinPool p) { // Used by ForkJoinTask + return p != null && (p.runState & STOP) != 0; + } // Creating, registering, and deregistering workers @@ -1931,20 +1935,20 @@ else if ((int)c == 0) // was dropped on timeout final void signalWork() { int pc = parallelism; for (long c = ctl;;) { + WorkQueue[] qs = queues; Thread owner = null; boolean create = false; - WorkQueue[] qs = queues; long ac = (c + RC_UNIT) & RC_MASK, nc; int deficit = pc - (short)(c >>> TC_SHIFT); - int sp = (int)c & ~INACTIVE, n; + int sp = (int)c & ACTIVE, i = sp & SMASK; if ((short)(c >>> RC_SHIFT) >= pc) break; - if (qs == null || (n = qs.length) <= 0) + if (qs == null || qs.length <= i) break; - WorkQueue v = qs[sp & (n - 1)]; + WorkQueue v = qs[i]; if (sp != 0 && v != null) { - owner = v.owner; nc = (v.stackPred & SP_MASK) | (c & TC_MASK); + owner = v.owner; } else if (deficit <= 0) break; @@ -1973,11 +1977,10 @@ private void reactivate(WorkQueue w) { for (long c = ctl;;) { // mostly, the no-create case of signalWork WorkQueue v; WorkQueue[] qs = queues; - int sp = (int)c & ~INACTIVE, n; + int sp = (int)c & ACTIVE, i = sp & SMASK; long pc = (UC_MASK & (c + RC_UNIT)) | (c & TC_MASK); - if (qs == null || (n = qs.length) <= 0 || sp == 0 || - (w == null && (c & RC_MASK) > 0L) || - (v = qs[sp & (n - 1)]) == null) + if ((w == null && (c & RC_MASK) > 0L) || sp == 0 || + qs == null || qs.length <= i || (v = qs[i]) == null) break; Thread owner = v.owner; long nc = (v.stackPred & SP_MASK) | pc; @@ -2001,7 +2004,7 @@ private boolean tryTrim(WorkQueue w) { if (w != null) { int pred = w.stackPred, cfg = w.config | TRIMMED; long c = ctl; - int sp = (int)c & ~INACTIVE; + int sp = (int)c & ACTIVE; if ((sp & SMASK) == (cfg & SMASK) && compareAndSetCtl(c, ((pred & SP_MASK) | (UC_MASK & (c - TC_UNIT))))) { @@ -2015,41 +2018,43 @@ private boolean tryTrim(WorkQueue w) { /** * Internal version of isQuiescent and related functionality. - * Returns true if terminating or submission queues are empty and - * unlocked, and all workers are inactive. If so, and quiescent - * shutdown is enabled, starts termination. + * Returns negative if terminating, 0 if submission queues are + * empty and unlocked, and all workers are inactive, else + * positive. */ - private boolean checkQuiescence() { + private int checkQuiescence() { boolean scanned = false; - for (int prevState = -1, rs = runState; ; prevState = rs) { + outer: for (int prevState = -1, rs = runState; ; prevState = rs) { WorkQueue[] qs; int n; - if ((rs & STOP) != 0) // terminating - break; + if ((rs & STOP) != 0) // terminating + return -1; if ((ctl & RC_MASK) > 0L) - return false; // active workers exist - if (scanned) { // try to trigger termination - if ((rs & SHUTDOWN) != 0 && !casRunState(rs, rs | STOP)) - scanned = false; // inconsistent; retry - else - break; + break; // active workers exist + if (scanned) { + if ((rs & SHUTDOWN) == 0) + return 0; + if (casRunState(rs, rs | STOP)) // try to trigger termination + return -1; + scanned = false; // inconsistent; retry } - if ((qs = queues) == null || (n = qs.length) <= 0) - break; // currently impossible - for (int i = 0;;) { // alternates even for external, odd internal - WorkQueue q, w; - if ((q = qs[i]) != null && (q.access != 0 || q.top - q.base > 0)) - return false; // possibly nonempty - if (++i == n) - break; - if ((w = qs[i]) != null && w.phase > 0) - return false; // active worker - if (++i == n) - break; + if ((qs = queues) != null && (n = qs.length) > 0) { + for (int i = 0;;) { // alternates even for external, odd internal + WorkQueue q, w; + if ((q = qs[i]) != null && + (q.access != 0 || q.top - q.base > 0)) + break outer; // possibly nonempty + if (++i == n) + break; + if ((w = qs[i]) != null && w.phase > 0) + break outer; // active worker + if (++i == n) + break; + } } if ((rs = runState) == prevState && (rs & LOCKED) == 0) - scanned = true; // same unlocked q state + scanned = true; // same unlocked state } - return true; + return 1; } /** @@ -2059,21 +2064,33 @@ private boolean checkQuiescence() { * @param w caller's WorkQueue (may be null on failed initialization) */ final void runWorker(WorkQueue w) { - if (w != null) { // skip on failed init - int r = w.stackPred; // use seed from registerWorker - for (int rs = 0, srcs = RESCAN;;) { - r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift - if (rs != (rs = runState)) { // stop or rescan - if ((rs & STOP) != 0) - break; - srcs &= RESCAN; + int rs = runState; + if (w != null && (rs & STOP) == 0) { + int phase = w.phase, r = w.stackPred; // use seed from registerWorker + for (int srcs = ACTIVE;;) { + if ((srcs = scan(w, srcs, r)) < 0) { + if (rs != (rs = runState)) { + if ((rs & STOP) != 0) // stop or rescan because stale + break; + srcs &= ACTIVE; + } else { + long c; // try to inactivate and enqueue + int p = phase + SS_SEQ, np = p | INACTIVE; + long nc = ((((c = ctl) - RC_UNIT) & UC_MASK) | + (p & ACTIVE & SP_MASK)); + w.stackPred = (int)c; // set ctl stack link + w.phase = np; + if (compareAndSetCtl(c, nc)) { + if ((phase = awaitWork(w, nc)) < 0) + break; + srcs = ACTIVE; // restart + } else { + w.phase = phase; // back out on ctl contention + srcs &= ACTIVE; + } + } } - if (srcs >= 0) - srcs = scan(w, srcs, r); - else if (awaitWork(w)) - srcs = RESCAN; - else - break; + r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift } } } @@ -2092,30 +2109,30 @@ else if (awaitWork(w)) private int scan(WorkQueue w, int srcs, int r) { WorkQueue[] qs = queues; int n = (w == null || qs == null) ? 0 : qs.length; - int usrcs = ((srcs << 16) | 1) & RESCAN; // base of next srcs window + int nsrcs = ((srcs << 16) | 1) & ACTIVE; // base of next srcs window for (int step = (r >>> 16) | 1, i = n; i > 0; --i, r += step) { int j, cap, b, k; WorkQueue q; ForkJoinTask[] a; if ((q = qs[j = r & SMASK & (n - 1)]) != null && (a = q.array) != null && (cap = a.length) > 0) { ForkJoinTask t = a[k = (cap - 1) & (b = q.base)], u; - U.loadFence(); // to re-read b and t - int nb = b + 1, nk = (cap - 1) & nb, qsrcs = usrcs + j; + U.loadFence(); // to re-read b and t + int nb = b + 1, nk = (cap - 1) & nb, qsrcs = nsrcs + j; if (q.base != b) - return srcs; // inconsistent + return srcs; // inconsistent if (t == null) { if (a[k] != null || a[nk] != null) - return qsrcs; // nonempty; restart + return qsrcs; // nonempty; restart } else if ((u = WorkQueue.cmpExSlotToNull(a, k, t)) == t) { q.base = nb; w.source = qsrcs; if (qsrcs != srcs && a[nk] != null) - signalWork(); // propagate at most twice/run + signalWork(); // propagate at most twice/run w.topLevelExec(t, q); return qsrcs; } else if (u != null || (qsrcs != srcs && a[nk] != null)) - return qsrcs; // limit retries under contention + return qsrcs; // limit retries under contention } } return srcs | INACTIVE; @@ -2124,54 +2141,50 @@ else if (u != null || (qsrcs != srcs && a[nk] != null)) /** * Advances phase, enqueues, and awaits signal or termination. * - * @return false for exit + * @param queuedCtl ctl value at point of inactivation + * @return current phase, or negative for exit */ - private boolean awaitWork(WorkQueue w) { + private int awaitWork(WorkQueue w, long queuedCtl) { if (w == null) - return false; // currently impossible - int p = (w.phase + SS_SEQ) & ~INACTIVE; // next phase number - long pc = ctl, sp = p & SP_MASK, qc; - w.phase = p | INACTIVE; - do { // enqueue - w.stackPred = (int)pc; // set ctl stack link - } while (pc != (pc = compareAndExchangeCtl( - pc, qc = ((pc - RC_UNIT) & UC_MASK) | sp))); + return -1; // currently impossible + int phase, quiescence; boolean idle = false; // true if pool idle - int active = (short)(qc >>> RC_SHIFT); - int nspins = active + (((short)(qc >>> TC_SHIFT)) << 2); - if (active <= 0 && !(idle = checkQuiescence())) - reactivate(null); // ensure live if may be tasks - if ((runState & STOP) != 0) - return false; - for (int spins = nspins;;) { // spin before block - if (w.phase >= 0) - return true; - if (--spins < 0) // avg #accesses in scan+signal + if ((queuedCtl & RC_MASK) <= 0L) { + if ((quiescence = checkQuiescence()) < 0) + return -1; // quiescent termination + else if (!(idle = (quiescence == 0)) && (ctl & RC_MASK) <= 0L) + reactivate(null); // ensure live if may be tasks + } + int spins = Math.max(SPIN_WAITS, ((short)(queuedCtl >>> TC_SHIFT)) << 1); + do { // spin before block for approx #accesses to scan+signal + if ((phase = w.phase) >= 0) break; Thread.onSpinWait(); - } - long deadline = idle ? keepAlive + System.currentTimeMillis() : 0L; - LockSupport.setCurrentBlocker(this); // emulate LockSupport.park - for (;;) { // await signal or termination - w.source = INACTIVE; // enable unpark - if (w.phase < 0) - U.park(idle, deadline); - w.source = 0; // disable unpark - if (w.phase >= 0) { - LockSupport.setCurrentBlocker(null); - return true; - } - if (idle) { // check for idle timeout - if (deadline - System.currentTimeMillis() < TIMEOUT_SLOP) { - if (tryTrim(w)) - return false; - deadline += keepAlive; // not at head; restart timer + } while (--spins != 0); + if (spins == 0) { + long deadline = idle ? System.currentTimeMillis() + keepAlive : 0L; + LockSupport.setCurrentBlocker(this); // emulate LockSupport.park + for (;;) { + Thread.interrupted(); // clear status + if ((runState & STOP) != 0) + return -1; + w.source = INACTIVE; // enable unpark + if (w.phase < 0) + U.park(idle, deadline); + w.source = 0; // disable unpark + if ((phase = w.phase) >= 0) + break; + if (idle) { // check for idle timeout + if (deadline - System.currentTimeMillis() < TIMEOUT_SLOP) { + if (tryTrim(w)) + return -1; + deadline += keepAlive; // not at head; restart timer + } } } - Thread.interrupted(); // clear status for next park - if ((runState & STOP) != 0) - return false; + LockSupport.setCurrentBlocker(null); } + return phase; } /** @@ -2185,7 +2198,7 @@ private ForkJoinTask pollScan(boolean submissionsOnly) { if ((runState & STOP) == 0) { WorkQueue[] qs; int n; WorkQueue q; ForkJoinTask t; int r = ThreadLocalRandom.nextSecondarySeed(); - if (submissionsOnly) // even indices only + if (submissionsOnly) // even indices only r &= ~1; int step = (submissionsOnly) ? 2 : 1; if ((qs = queues) != null && (n = qs.length) > 0) { @@ -2219,7 +2232,7 @@ private int tryCompensate(long c, boolean canSaturate) { maxTotal = (short)(b >>> SWIDTH) + pc, active = (short)(c >>> RC_SHIFT), total = (short)(c >>> TC_SHIFT), - sp = (int)c & ~INACTIVE; + sp = (int)c & ACTIVE; if (((rs = runState) & STOP) != 0) // terminating return 0; else if ((rs & LOCKED) != 0) // unstable @@ -2272,145 +2285,165 @@ final void uncompensate() { * * @param task the task * @param w caller's WorkQueue + * @param internal true if w is owned by a ForkJoinWorkerThread + * @param locals true if check local queue * @return task status on exit, or UNCOMPENSATE for compensated blocking */ - final int helpJoin(ForkJoinTask task, WorkQueue w, boolean internal) { - int s; - if (w == null || task == null) - return 0; - if (w.tryRemoveAndExec(task, internal) && (s = task.status) < 0) + final int helpJoin(ForkJoinTask task, WorkQueue w, boolean internal, + boolean locals) { + int s = 0; + if (task == null) return s; - if (!internal) - return 0; - int wsrc = w.source & SMASK, wid = (w.config & SMASK) + 1, r = wid + 1; - long sctl = 0L; // track stability - for (boolean rescan = true;;) { - WorkQueue[] qs; - if ((s = task.status) < 0) - return s; - if (!rescan && sctl == (sctl = ctl)) { - if ((s = tryCompensate(sctl, false)) >= 0) - return s; // block - } - else if ((runState & STOP) != 0) - return 0; - rescan = false; - int n = ((qs = queues) == null) ? 0 : qs.length; - scan: for (int i = n >>> 1; i > 0; --i, r += 2) { - int j, cap; WorkQueue q; ForkJoinTask[] a; - if ((q = qs[j = r & SMASK & (n - 1)]) != null && - (a = q.array) != null && (cap = a.length) > 0) { - for (int src = j + 1;;) { - int sq = q.source, b = q.base, k = (cap - 1) & b; - ForkJoinTask t = a[k]; - U.loadFence(); - boolean eligible; // check steal chain - for (int d = n, v = sq;;) { // may be cyclic; bound - WorkQueue p; - if ((v &= SMASK) == wid) { - eligible = true; - break; - } - if (v == 0 || --d == 0 || - (p = qs[(v - 1) & (n - 1)]) == null) { - eligible = false; - break; - } - v = p.source; - } + if (locals && w != null) + w.tryRemoveAndExec(task, internal); + if ((s = task.status) >= 0 && internal && w != null) { + int wsrc = w.source & SMASK; + int wid = (w.config & SMASK) + 1, r = wid + 1; + long sctl = ctl; // track stability + outer: for (boolean rescan = true;;) { + WorkQueue[] qs; + if ((s = task.status) < 0) + break; + if (!rescan) { + for (int spins = SPIN_WAITS; spins != 0; --spins) { + Thread.onSpinWait(); // stability check if ((s = task.status) < 0) - return s; // validate - int nb = b + 1, qdepth = q.top - b; - if (q.source == sq && q.base == b && a[k] == t) { - if (t == null) { - rescan |= (qdepth > 0); - break; // revisit if nonempty + break outer; + } + if (sctl == (sctl = ctl) && + (s = tryCompensate(sctl, false)) >= 0) + break; + } + rescan = false; + if ((runState & STOP) != 0) + break; + int n = ((qs = queues) == null) ? 0 : qs.length; + scan: for (int i = n >>> 1; i > 0; --i, r += 2) { + int j, cap; WorkQueue q; ForkJoinTask[] a; + if ((q = qs[j = r & SMASK & (n - 1)]) != null && + (a = q.array) != null && (cap = a.length) > 0) { + for (int src = j + 1;;) { + int sq = q.source, b = q.base, k = (cap - 1) & b; + ForkJoinTask t = a[k]; + U.loadFence(); + boolean eligible; // check steal chain + for (int steps = n, v = sq;;) { + WorkQueue p; + if ((v &= SMASK) == wid) { + eligible = true; + break; + } + if (v == 0 || (p = qs[(v - 1) & (n - 1)]) == null || + --steps == 0) { // bound steps + eligible = false; + break; + } + v = p.source; } - if (t != task && !eligible) - break; - if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { - q.base = nb; - w.source = src; - t.doExec(); - w.source = wsrc; + if ((s = task.status) < 0) + break outer; // validate + int nb = b + 1, qdepth = q.top - b; + if (q.source == sq && q.base == b && a[k] == t) { + if (t == null) { + rescan |= (qdepth > 0); + break; // revisit if nonempty + } + if (t != task && !eligible) + break; + if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { + q.base = nb; + w.source = src; + t.doExec(); + w.source = wsrc; + } + rescan = true; // restart at same index + break scan; } - rescan = true; // restart at same index - break scan; } } } } } + return s; } /** - * Helps join tasks with completers. + * Version of helpJoin for CountedCompleters. * * @param task root of computation (only called when a CountedCompleter) * @param w caller's WorkQueue * @param internal true if w is owned by a ForkJoinWorkerThread - * @param timed true if this is a timed join * @return task status on exit, or UNCOMPENSATE for compensated blocking */ - final int helpComplete(CountedCompleter task, WorkQueue w, - boolean internal) { - if (w == null || task == null) - return 0; - int r = w.config + 1; // for indexing - long sctl = 0L; // track stability - for (boolean rescan = true, locals = true;;) { - int s; WorkQueue[] qs; - if ((locals && (s = w.helpComplete(task, internal, 0)) < 0) || - (s = task.status) < 0) - return s; - if (!rescan && sctl == (sctl = ctl)) { - if (!internal) - return 0; - if ((s = tryCompensate(sctl, false)) >= 0) - return s; - } - else if ((runState & STOP) != 0) - return 0; - rescan = locals = false; - int n = ((qs = queues) == null) ? 0 : qs.length; - scan: for (int i = n; i > 0; --i, ++r) { - int j, cap; WorkQueue q; ForkJoinTask[] a; - if ((q = qs[j = r & SMASK & (n - 1)]) != null && - (a = q.array) != null && (cap = a.length) > 0) { - for (int b = q.base, k;;) { - ForkJoinTask t = a[k = (cap - 1) & b]; - U.loadFence(); - int nb = b + 1, nk = (cap - 1) & nb; - boolean eligible = false; - if (t instanceof CountedCompleter) { - CountedCompleter f = (CountedCompleter)t; - do {} while (f != task && (f = f.completer) != null); - if (f != null) - eligible = true; - } + final int helpComplete(ForkJoinTask task, WorkQueue w, boolean internal) { + int s = 0; + if (task != null && (s = task.status) >= 0 && w != null) { + int r = w.config + 1; // for indexing + long sctl = ctl; // track stability + outer: for (boolean rescan = true, locals = true;;) { + WorkQueue[] qs; + if (locals && (s = w.helpComplete(task, internal, 0)) < 0) + break; + if ((s = task.status) < 0) + break; + if (!rescan) { + for (int spins = SPIN_WAITS; spins != 0; --spins) { + Thread.onSpinWait(); // stabilty check if ((s = task.status) < 0) - return s; // validate - if (b == (b = q.base) && a[k] == t) { - if (t == null) { - if (!rescan && a[nk] != null) - rescan = true; // revisit - break; + break outer; + } + if ((sctl == (sctl = ctl)) && + (!internal || (s = tryCompensate(sctl, false)) >= 0)) + break; + } + rescan = locals = false; + if ((runState & STOP) != 0) + break; + int n = ((qs = queues) == null) ? 0 : qs.length; + scan: for (int i = n; i > 0; --i, ++r) { + int j, cap; WorkQueue q; ForkJoinTask[] a; + if ((q = qs[j = r & SMASK & (n - 1)]) != null && + (a = q.array) != null && (cap = a.length) > 0) { + for (int b = q.base, k;;) { + ForkJoinTask t = a[k = (cap - 1) & b]; + U.loadFence(); + int nb = b + 1, nk = (cap - 1) & nb, steps = cap; + boolean eligible = false; + if (t instanceof CountedCompleter) { + CountedCompleter f = (CountedCompleter)t; + do { + if (f == task) { + eligible = true; + break; + } + } while ((f = f.completer) != null && + --steps > 0); // bound steps } - if (!eligible) - break; - if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { - q.base = nb; - U.storeFence(); - t.doExec(); - locals = true; // process local queue + if ((s = task.status) < 0) + break outer; // validate + if (b == (b = q.base) && a[k] == t) { + if (t == null) { + if (!rescan && a[nk] != null) + rescan = true; // revisit + break; + } + if (!eligible) + break; + if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { + q.base = nb; + U.storeFence(); + t.doExec(); + locals = true; // process local queue + } + rescan = true; + break scan; } - rescan = true; - break scan; } } } } } + return s; } /** @@ -2486,19 +2519,15 @@ else if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { else if (!busy) break; else if (phase >= 0) { + waits = 0; // to recheck, then sleep w.phase = phase = inactivePhase; - waits = 0; // to recheck, then yield/sleep } else if (System.nanoTime() - startTime > nanos) { returnStatus = 0; // timed out break; } else if (waits == 0) // same as spinLockRunState except - waits = 1 ; // with rescan instead of onSpinWait - else if (waits == 1) { - waits = MIN_SLEEP; - Thread.yield(); - } + waits = MIN_SLEEP; // with rescan instead of onSpinWait else { LockSupport.parkNanos(this, (long)waits); if (waits < maxSleep) @@ -2517,36 +2546,31 @@ else if (waits == 1) { * @return positive if quiescent, negative if interrupted, else 0 */ private int externalHelpQuiesce(long nanos, boolean interruptible) { - if (checkQuiescence()) - return 1; - long startTime = System.nanoTime(); - long maxSleep = Math.min(nanos >>> 8, MAX_SLEEP); - for (int waits = 0;;) { // similar to helpQuiesce, using poll vs scanning - ForkJoinTask t; - if ((runState & STOP) != 0) - return 1; - else if (interruptible && Thread.interrupted()) - return -1; - else if ((t = pollScan(false)) != null) { - t.doExec(); - waits = 0; - } - else if (checkQuiescence()) - return 1; - else if (System.nanoTime() - startTime > nanos) - return 0; - else if (waits == 0) - waits = 1; - else if (waits == 1) { - waits = MIN_SLEEP; - Thread.yield(); - } - else { - LockSupport.parkNanos(this, (long)waits); - if (waits < maxSleep) - waits <<= 1; + if (checkQuiescence() > 0) { + long startTime = System.nanoTime(); + long maxSleep = Math.min(nanos >>> 8, MAX_SLEEP); + for (int waits = 0;;) { + ForkJoinTask t; + if (interruptible && Thread.interrupted()) + return -1; + else if ((t = pollScan(false)) != null) { + waits = 0; + t.doExec(); + } + else if (checkQuiescence() <= 0) + break; + else if (System.nanoTime() - startTime > nanos) + return 0; + else if (waits == 0) + waits = MIN_SLEEP; + else { + LockSupport.parkNanos(this, (long)waits); + if (waits < maxSleep) + waits <<= 1; + } } } + return 1; } /** @@ -2759,7 +2783,7 @@ private int tryTerminate(boolean now, boolean enable) { else { if ((isShutdown = (rs & SHUTDOWN)) == 0 && enable) getAndBitwiseOrRunState(isShutdown = SHUTDOWN); - if (isShutdown != 0 && checkQuiescence()) + if (isShutdown != 0 && checkQuiescence() < 0) rs = STOP | SHUTDOWN; } } @@ -3141,6 +3165,8 @@ public ForkJoinTask submit(ForkJoinTask task) { @Override public ForkJoinTask submit(Callable task) { return poolSubmit(true, false, + (Thread.currentThread() instanceof ForkJoinWorkerThread) ? + new ForkJoinTask.AdaptedCallable(task) : new ForkJoinTask.AdaptedInterruptibleCallable(task)); } @@ -3152,6 +3178,8 @@ public ForkJoinTask submit(Callable task) { @Override public ForkJoinTask submit(Runnable task, T result) { return poolSubmit(true, false, + (Thread.currentThread() instanceof ForkJoinWorkerThread) ? + new ForkJoinTask.AdaptedRunnable(task, result) : new ForkJoinTask.AdaptedInterruptibleRunnable(task, result)); } @@ -3163,9 +3191,12 @@ public ForkJoinTask submit(Runnable task, T result) { @Override @SuppressWarnings("unchecked") public ForkJoinTask submit(Runnable task) { - return poolSubmit(true, false, (task instanceof ForkJoinTask) - ? (ForkJoinTask) task // avoid re-wrap - : new ForkJoinTask.AdaptedInterruptibleRunnable(task, null)); + ForkJoinTask f = (task instanceof ForkJoinTask) ? + (ForkJoinTask) task : // avoid re-wrap + ((Thread.currentThread() instanceof ForkJoinWorkerThread) ? + new ForkJoinTask.AdaptedRunnable(task, null) : + new ForkJoinTask.AdaptedInterruptibleRunnable(task, null)); + return poolSubmit(true, false, f); } // Added mainly for possible use in Loom @@ -3244,183 +3275,48 @@ public int setParallelism(int size) { } /** - * @throws NullPointerException {@inheritDoc} - * @throws RejectedExecutionException {@inheritDoc} + * Common support for timed and untimed invokeAll */ - @Override - public List> invokeAll(Collection> tasks) { + private List> invokeAll(Collection> tasks, + boolean timed, long deadline) + throws InterruptedException { ArrayList> futures = new ArrayList<>(tasks.size()); try { for (Callable t : tasks) { - ForkJoinTask.AdaptedInterruptibleCallable f = - new ForkJoinTask.AdaptedInterruptibleCallable(t); + ForkJoinTask f = ForkJoinTask.adaptInterruptible(t); futures.add(f); poolSubmit(true, false, f); } - for (int i = futures.size() - 1; i >= 0; --i) { - ForkJoinTask.AdaptedInterruptibleCallable f = - (ForkJoinTask.AdaptedInterruptibleCallable)futures.get(i); - f.quietlyJoin(); - } + for (int i = futures.size() - 1; i >= 0; --i) + ((ForkJoinTask)futures.get(i)) + .quietlyJoinPoolInvokeAllTask(timed, deadline); return futures; } catch (Throwable t) { - for (Future e : futures) { - if (e != null) - e.cancel(true); - } + for (Future e : futures) + e.cancel(true); throw t; } } @Override - public List> invokeAll(Collection> tasks, - long timeout, TimeUnit unit) + public List> invokeAll(Collection> tasks) throws InterruptedException { - long nanos = unit.toNanos(timeout); - ArrayList> futures = new ArrayList<>(tasks.size()); - try { - for (Callable t : tasks) { - ForkJoinTask.AdaptedInterruptibleCallable f = - new ForkJoinTask.AdaptedInterruptibleCallable(t); - futures.add(f); - poolSubmit(true, false, f); - } - long startTime = System.nanoTime(), ns = nanos; - boolean timedOut = (ns < 0L); - for (int i = futures.size() - 1; i >= 0; --i) { - ForkJoinTask.AdaptedInterruptibleCallable f = - (ForkJoinTask.AdaptedInterruptibleCallable)futures.get(i); - while (!f.isDone()) { - if (!timedOut) - timedOut = !f.quietlyJoin(ns, TimeUnit.NANOSECONDS); - if (timedOut) - f.cancel(true); - else - ns = nanos - (System.nanoTime() - startTime); - } - } - return futures; - } catch (Throwable t) { - for (Future e : futures) { - if (e != null) - e.cancel(true); - } - throw t; - } - } - - /** - * Task (that is never forked) to hold results for invokeAny, or - * to report exception if all subtasks fail or are cancelled or - * the pool is terminating. - */ - static final class InvokeAnyRoot extends ForkJoinTask.ExternalTask { - private static final long serialVersionUID = 2838392045355241008L; - @SuppressWarnings("serial") // Conditionally serializable - volatile E result; - final AtomicInteger count; // in case all fail - @SuppressWarnings("serial") - InvokeAnyRoot(int n) { - if (n <= 0) - throw new IllegalArgumentException(); - count = new AtomicInteger(n); - } - final void tryComplete(InvokeAnyTask f, E v, Throwable ex, - boolean completed) { - ForkJoinPool p; - if (f != null && !isDone()) { - if ((p = getPool()) != null && p.stopping()) - trySetCancelled(); - else if (f.setForkJoinTaskStatusMarkerBit() == 0) { - if (completed) { - result = v; - quietlyComplete(); - } - else if (count.getAndDecrement() <= 1) { - if (ex == null) - trySetCancelled(); - else - trySetException(ex); - } - } - } - } - public final E compute() { return null; } // never forked - public final E getRawResult() { return result; } - public final void setRawResult(E v) { result = v; } - // Common support for timed and untimed versions of invokeAny - final E invokeAny(Collection> tasks, - ForkJoinPool pool, boolean timed, long nanos) - throws InterruptedException, ExecutionException, TimeoutException { - ArrayList> fs = new ArrayList<>(count.get()); - try { - for (Callable c : tasks) { - InvokeAnyTask f = new InvokeAnyTask(this, c); - fs.add(f); - pool.poolSubmit(true, false, f); - if (isDone()) - break; - } - if (timed) - return get(nanos, TimeUnit.NANOSECONDS); - else - return get(); - } finally { - for (InvokeAnyTask f : fs) - ForkJoinTask.cancelIgnoringExceptions(f); - } - } + return invokeAll(tasks, false, 0L); } - /** - * ExternalTask with results in InvokeAnyRoot (and never - * independently joined). - */ - static final class InvokeAnyTask extends ForkJoinTask.ExternalTask { - final InvokeAnyRoot root; - @SuppressWarnings("serial") // Conditionally serializable - final Callable callable; - InvokeAnyTask(InvokeAnyRoot root, Callable callable) { - if (callable == null) throw new NullPointerException(); - this.callable = callable; - this.root = root; - } - final Void compute() throws Exception { - InvokeAnyRoot r = root; - E v = null; Throwable ex = null; boolean completed = false; - if (r != null && !r.isDone()) { - try { - v = callable.call(); - completed = true; - } catch (Exception rex) { - ex = rex; - } finally { - r.tryComplete(this, v, ex, completed); - } - } - return null; - } - public final boolean cancel(boolean mayInterruptIfRunning) { - InvokeAnyRoot r; - boolean stat = super.cancel(mayInterruptIfRunning); - if ((r = root) != null) - r.tryComplete(this, null, null, false); - return stat; - } - public final Void getRawResult() { return null; } - public final void setRawResult(Void v) { } - public String toString() { - return super.toString() + "[Wrapped task = " + callable + "]"; - } - private static final long serialVersionUID = 2838392045355241008L; + @Override + public List> invokeAll(Collection> tasks, + long timeout, TimeUnit unit) + throws InterruptedException { + return invokeAll(tasks, true, unit.toNanos(timeout) + System.nanoTime()); } @Override public T invokeAny(Collection> tasks) throws InterruptedException, ExecutionException { try { - return new InvokeAnyRoot(tasks.size()). - invokeAny(tasks, this, false, 0L); + return new ForkJoinTask.InvokeAnyRoot() + .invokeAny(tasks, this, false, 0L); } catch (TimeoutException cannotHappen) { assert false; return null; @@ -3431,8 +3327,8 @@ public T invokeAny(Collection> tasks) public T invokeAny(Collection> tasks, long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { - return new InvokeAnyRoot(tasks.size()). - invokeAny(tasks, this, true, unit.toNanos(timeout)); + return new ForkJoinTask.InvokeAnyRoot() + .invokeAny(tasks, this, true, unit.toNanos(timeout)); } /** @@ -3538,7 +3434,7 @@ public int getActiveThreadCount() { * @return {@code true} if all threads are currently idle */ public boolean isQuiescent() { - return checkQuiescence(); + return checkQuiescence() <= 0; } /** @@ -4047,12 +3943,16 @@ private static void unmanagedBlock(ManagedBlocker blocker) @Override protected RunnableFuture newTaskFor(Runnable runnable, T value) { - return new ForkJoinTask.AdaptedInterruptibleRunnable(runnable, value); + return (Thread.currentThread() instanceof ForkJoinWorkerThread) ? + new ForkJoinTask.AdaptedRunnable(runnable, value) : + new ForkJoinTask.AdaptedInterruptibleRunnable(runnable, value); } @Override protected RunnableFuture newTaskFor(Callable callable) { - return new ForkJoinTask.AdaptedInterruptibleCallable(callable); + return (Thread.currentThread() instanceof ForkJoinWorkerThread) ? + new ForkJoinTask.AdaptedCallable(callable) : + new ForkJoinTask.AdaptedInterruptibleCallable(callable); } static { diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index f65eccae12209..8a34cb36510ff 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -214,16 +214,16 @@ public abstract class ForkJoinTask implements Future, Serializable { * holds bits recording completion status. Note that there is no * status bit representing "running", recording whether incomplete * tasks are queued vs executing. However these cases can be - * distinguished in subclasses of ExternalTask that adds - * this capability by recording the running thread. Cancellation - * is recorded in status bits (ABNORMAL but not THROWN), but - * reported in joining methods by throwing an exception. Other - * exceptions of completed (THROWN) tasks are recorded in the - * "aux" field, but are reconstructed (see getThrowableException) - * to produce more useful stack traces when reported. Sentinels - * for interruptions or timeouts while waiting for completion are - * not recorded as status bits but are included in return values - * of methods in which they occur. + * distinguished in subclasses of InterruptibleTask that adds this + * capability by recording the running thread. Cancellation is + * recorded in status bits (ABNORMAL but not THROWN), but reported + * in joining methods by throwing an exception. Other exceptions + * of completed (THROWN) tasks are recorded in the "aux" field, + * but are reconstructed (see getThrowableException) to produce + * more useful stack traces when reported. Sentinels for + * interruptions or timeouts while waiting for completion are not + * recorded as status bits but are included in return values of + * methods in which they occur. * * The methods of this class are more-or-less layered into * (1) basic status maintenance @@ -269,21 +269,19 @@ final boolean casNext(Aux c, Aux v) { // used only in cancellation * control bits occupy only (some of) the upper half (16 bits) of * status field. The lower bits are used for user-defined tags. */ - static final int DONE = 1 << 31; // must be negative - static final int ABNORMAL = 1 << 16; - static final int THROWN = 1 << 17; - static final int MARKER = 1 << 30; // utility marker - - static final int UNCOMPENSATE = 1 << 16; // helpJoin return sentinel - static final int SMASK = 0xffff; // short bits for tags + private static final int DONE = 1 << 31; // must be negative + private static final int ABNORMAL = 1 << 16; + private static final int THROWN = 1 << 17; + private static final int MARKER = 1 << 30; // utility marker + private static final int HAVE_EXCEPTION = DONE | ABNORMAL | THROWN; - // flags for "how" arguments in awaitDone and elsewhere - static final int RAN = 1; - static final int INTERRUPTIBLE = 2; - static final int TIMED = 4; + private static final int UNCOMPENSATE = 1 << 16; // helpJoin return sentinel + private static final int SMASK = 0xffff; // short bits for tags - // conveniences - private static final int HAVE_EXCEPTION = DONE | ABNORMAL | THROWN; + // flags for "how" arguments in awaitDone and callers + private static final int RAN = 1 << 0; + private static final int INTERRUPTIBLE = 1 << 1; + private static final int TIMED = 1 << 2; // Fields volatile int status; // accessed directly by pool and workers @@ -348,7 +346,7 @@ final boolean trySetThrown(Throwable ex) { int s; boolean set = false, installed = false; if ((s = status) >= 0) { - Aux h = new Aux(Thread.currentThread(), ex), p = null, a; + Aux a, p = null, h = new Aux(Thread.currentThread(), ex); do { if (!installed && ((a = aux) == null || a.ex == null) && (installed = casAux(a, h))) @@ -370,125 +368,126 @@ final void trySetException(Throwable ex) { onFJTExceptionSet(ex); } + /** + * Cleans out a wait node after an interrupted or timed out wait. + * Similar to AbstractQueuedSynchronizer cleanup. + */ + private void cancelWait(Aux node) { + for (Aux a; (a = aux) != null && a.ex == null; ) { + for (Aux trail = null;;) { + Aux next = a.next; + if (a == node) { + if (trail != null) + trail.casNext(trail, next); + else if (casAux(a, next)) + return; // cannot be re-encountered + break; // restart + } else { + trail = a; + if ((a = next) == null) + return; + } + } + } + } + /* * Waits for signal, interrupt, timeout, or pool termination. * * @param pool if nonnull, the pool of ForkJoinWorkerThread caller - * @param how flags for TIMED, INTERRUPTIBLE, UNCOMPENSATE + * @param compensation result from a helping method + * @param how flags for TIMED, INTERRUPTIBLE * @param deadline if timed, timeout deadline * @return ABNORMAL if interrupted, 0 on timeout, else status on exit */ - private int awaitDone(ForkJoinPool pool, int how, long deadline) { - int s = 0; - if ((how & UNCOMPENSATE) != 0 || (s = status) >= 0) { - try { - if (pool != null && pool.stopping()) - cancelIgnoringExceptions(this); // recheck below on interrupt - boolean queued = false; - Aux node = null; - for (Aux a;;) { // install node + private int awaitDone(ForkJoinPool pool, int compensation, int how, + long deadline) { + int s, interrupts = 0; // < 0 : throw; > 0 : re-interrupt + boolean queued = false; + Aux node = null; + if ((s = status) >= 0) { + try { // spinwait if out of memory + node = new Aux(Thread.currentThread(), null); + } catch (OutOfMemoryError ex) { + } + for (Aux a;;) { // install node + if ((s = status) < 0) + break; + else if (node == null) + Thread.onSpinWait(); + else if (((a = aux) == null || a.ex == null) && + (queued = casAux(node.next = a, node))) + break; + } + if (queued) { // await signal or interrupt + LockSupport.setCurrentBlocker(this); + for (;;) { if ((s = status) < 0) break; - else if (node == null) - node = new Aux(Thread.currentThread(), null); - else if (((a = aux) == null || a.ex == null) && - (queued = casAux(node.next = a, node))) + else if (interrupts < 0) { + s = ABNORMAL; // interrupted and not done break; - } - if (queued) { // await signal or interrupt - int interruptStatus = 0; // < 0 : throw, > 0 : re-interrupt - LockSupport.setCurrentBlocker(this); - for (;;) { - if ((s = status) < 0) - break; - else if (interruptStatus < 0) { - s = ABNORMAL; // interrupted and not done - break; - } - else if (Thread.interrupted()) { - if (pool == null || !pool.stopping()) - interruptStatus = - (how & INTERRUPTIBLE) != 0 ? -1 : 1; - else { - interruptStatus = 1; // re-assert if cleared - cancelIgnoringExceptions(this); - } - } - else if ((how & TIMED) != 0) { - long ns = deadline - System.nanoTime(); - if (ns <= 0L) { - s = 0; - break; - } - LockSupport.parkNanos(ns); + } + else if (Thread.interrupted()) { + if (!ForkJoinPool.poolIsStopping(pool)) + interrupts = (how & INTERRUPTIBLE) != 0 ? -1 : 1; + else { + interrupts = 1; // re-assert if cleared + cancelIgnoringExceptions(this); } - else - LockSupport.park(); } - if (s >= 0) { - // cancellation similar to AbstractQueuedSynchronizer - outer: for (Aux a; (a = aux) != null && a.ex == null; ) { - for (Aux trail = null;;) { - Aux next = a.next; - if (a == node) { - if (trail != null) - trail.casNext(trail, next); - else if (casAux(a, next)) - break outer; // cannot be re-encountered - break; // restart - } else { - trail = a; - if ((a = next) == null) - break outer; - } - } + else if ((how & TIMED) != 0) { + long ns = deadline - System.nanoTime(); + if (ns <= 0L) { + s = 0; + break; } + LockSupport.parkNanos(ns); } else - signalWaiters(); // help clean or signal - LockSupport.setCurrentBlocker(null); - if (interruptStatus > 0) - Thread.currentThread().interrupt(); + LockSupport.park(); } - } finally { - if ((how & UNCOMPENSATE) != 0 && pool != null) - pool.uncompensate(); + if (s >= 0) + cancelWait(node); + else + signalWaiters(); + LockSupport.setCurrentBlocker(null); + if (interrupts > 0) + Thread.currentThread().interrupt(); } } + if (compensation == UNCOMPENSATE && pool != null) + pool.uncompensate(); return s; } /** * Tries applicable helping steps while joining this task, - * otherwise invokes awaitDone + * otherwise invokes blocking version of awaitDone * - * @param interruptible true if wait interruptible * @param how flags for RAN, TIMED, INTERRUPTIBLE + * @param deadline if timed, timeout deadline * @return ABNORMAL if interrupted, else status on exit */ - private int awaitJoin(int how, long deadline) { + private int awaitDone(int how, long deadline) { + int s = 0; ForkJoinWorkerThread wt; ForkJoinPool p; ForkJoinPool.WorkQueue q; - boolean internal; Thread t; int s; + Thread t; boolean internal; if (internal = (t = Thread.currentThread()) instanceof ForkJoinWorkerThread) { - q = (wt = (ForkJoinWorkerThread)t).workQueue; - p = wt.pool; + p = (wt = (ForkJoinWorkerThread)t).pool; + q = wt.workQueue; } else q = ForkJoinPool.externalQueue(p = ForkJoinPool.common); - if (q != null && p != null) { - if (this instanceof CountedCompleter) - s = p.helpComplete((CountedCompleter)this, q, internal); - else if (!(this instanceof ExternalTask) || internal) - s = p.helpJoin(this, q, internal); - else - s = 0; - if (s < 0) - return s; - if (s == UNCOMPENSATE) - how |= s; - } - return awaitDone(p, how, deadline); + if (p != null && + (s = ((this instanceof CountedCompleter) ? + p.helpComplete(this, q, internal) : + p.helpJoin(this, q, internal, + !(this instanceof InterruptibleTask) && + (how & RAN) == 0))) < 0) + return s; + return awaitDone(internal ? p : null, s, how, deadline); } /** @@ -659,7 +658,7 @@ void onFJTExceptionSet(Throwable ex) { /** * Optional wrapping for cancellation in reportExecutionException. - * (Overridden in class ExternalTask.) + * (Overridden in class InterruptibleTask.) */ Throwable wrapFJTCancellationException(CancellationException cx) { return cx; @@ -715,7 +714,7 @@ public final ForkJoinTask fork() { public final V join() { int s; if ((s = status) >= 0) - s = awaitJoin(0, 0L); + s = awaitDone(0, 0L); if ((s & ABNORMAL) != 0) reportException(s); return getRawResult(); @@ -733,7 +732,7 @@ public final V invoke() { int s; doExec(); if ((s = status) >= 0) - s = awaitJoin(0, 0L); + s = awaitDone(RAN, 0L); if ((s & ABNORMAL) != 0) reportException(s); return getRawResult(); @@ -763,14 +762,14 @@ public static void invokeAll(ForkJoinTask t1, ForkJoinTask t2) { t2.fork(); t1.doExec(); if ((s1 = t1.status) >= 0) - s1 = t1.awaitJoin(0, 0L); + s1 = t1.awaitDone(RAN, 0L); if ((s1 & ABNORMAL) != 0) { cancelIgnoringExceptions(t2); t1.reportException(s1); } else { if ((s2 = t2.status) >= 0) - s2 = t2.awaitJoin(0, 0L); + s2 = t2.awaitDone(0, 0L); if ((s2 & ABNORMAL) != 0) t2.reportException(s2); } @@ -804,7 +803,7 @@ public static void invokeAll(ForkJoinTask... tasks) { int s; t.doExec(); if ((s = t.status) >= 0) - s = t.awaitJoin(0, 0L); + s = t.awaitDone(RAN, 0L); if ((s & ABNORMAL) != 0) ex = t.getException(s); break; @@ -817,7 +816,7 @@ public static void invokeAll(ForkJoinTask... tasks) { if ((t = tasks[i]) != null) { int s; if ((s = t.status) >= 0) - s = t.awaitJoin(0, 0L); + s = t.awaitDone(0, 0L); if ((s & ABNORMAL) != 0 && (ex = t.getException(s)) != null) break; } @@ -868,7 +867,7 @@ public static > Collection invokeAll(Collection int s; t.doExec(); if ((s = t.status) >= 0) - s = t.awaitJoin(0, 0L); + s = t.awaitDone(RAN, 0L); if ((s & ABNORMAL) != 0) ex = t.getException(s); break; @@ -881,7 +880,7 @@ public static > Collection invokeAll(Collection if ((t = ts.get(i)) != null) { int s; if ((s = t.status) >= 0) - s = t.awaitJoin(0, 0L); + s = t.awaitDone(0, 0L); if ((s & ABNORMAL) != 0 && (ex = t.getException(s)) != null) break; } @@ -1058,12 +1057,9 @@ public final void quietlyComplete() { */ public final V get() throws InterruptedException, ExecutionException { int s; - if ((s = status) >= 0) { - if (Thread.interrupted()) - s = ABNORMAL; - else - s = awaitJoin(INTERRUPTIBLE, 0L); - } + if ((s = status) >= 0) + s = ((Thread.interrupted()) ? ABNORMAL : + awaitDone(INTERRUPTIBLE, 0L)); if ((s & ABNORMAL) != 0) reportExecutionException(s); return getRawResult(); @@ -1091,7 +1087,7 @@ public final V get(long timeout, TimeUnit unit) if (Thread.interrupted()) s = ABNORMAL; else if (nanos > 0L) - s = awaitJoin(INTERRUPTIBLE | TIMED, nanos + System.nanoTime()); + s = awaitDone(INTERRUPTIBLE | TIMED, nanos + System.nanoTime()); } if (s >= 0 || (s & ABNORMAL) != 0) reportExecutionException(s); @@ -1105,9 +1101,8 @@ else if (nanos > 0L) * known to have aborted. */ public final void quietlyJoin() { - int s; - if ((s = status) >= 0) - awaitJoin(0, 0L); + if (status >= 0) + awaitDone(0, 0L); } /** @@ -1118,7 +1113,7 @@ public final void quietlyJoin() { public final void quietlyInvoke() { doExec(); if (status >= 0) - awaitJoin(0, 0L); + awaitDone(RAN, 0L); } /** @@ -1141,7 +1136,7 @@ public final boolean quietlyJoin(long timeout, TimeUnit unit) if (Thread.interrupted()) s = ABNORMAL; else if (nanos > 0L) - s = awaitJoin(INTERRUPTIBLE | TIMED, nanos + System.nanoTime()); + s = awaitDone(INTERRUPTIBLE | TIMED, nanos + System.nanoTime()); } if (s == ABNORMAL) throw new InterruptedException(); @@ -1163,10 +1158,28 @@ public final boolean quietlyJoinUninterruptibly(long timeout, int s; long nanos = unit.toNanos(timeout); if ((s = status) >= 0 && nanos > 0L) - s = awaitJoin(TIMED, nanos + System.nanoTime()); + s = awaitDone(TIMED, nanos + System.nanoTime()); return (s < 0); } + /** + * Utility for possibly-timed ForkJoinPool.invokeAll + */ + final void quietlyJoinPoolInvokeAllTask(boolean timed, long deadline) + throws InterruptedException { + int s; + if ((s = status) >= 0) { + if (Thread.interrupted()) + s = ABNORMAL; + else if (!timed || deadline - System.nanoTime() > 0L) + s = awaitDone(INTERRUPTIBLE | (timed ? TIMED : 0), deadline); + if (s == ABNORMAL) + throw new InterruptedException(); + else if (s >= 0) + cancel(true); + } + } + /** * Possibly executes tasks until the pool hosting the current task * {@linkplain ForkJoinPool#isQuiescent is quiescent}. This @@ -1499,6 +1512,43 @@ public static ForkJoinTask adaptInterruptible(Callable calla return new AdaptedInterruptibleCallable(callable); } + /** + * Returns a new {@code ForkJoinTask} that performs the {@code run} + * method of the given {@code Runnable} as its action, and returns + * the given result upon {@link #join}, translating any checked exceptions + * encountered into {@code RuntimeException}. Additionally, + * invocations of {@code cancel} with {@code mayInterruptIfRunning + * true} will attempt to interrupt the thread performing the task. + * + * @param runnable the runnable action + * @param result the result upon completion + * @param the type of the result + * @return the task + * + * @since 21 + */ + public static ForkJoinTask adaptInterruptible(Runnable runnable, T result) { + return new AdaptedInterruptibleRunnable(runnable, result); + } + + /** + * Returns a new {@code ForkJoinTask} that performs the {@code + * run} method of the given {@code Runnable} as its action, and + * returns null upon {@link #join}, translating any checked + * exceptions encountered into {@code RuntimeException}. + * Additionally, invocations of {@code cancel} with {@code + * mayInterruptIfRunning true} will attempt to interrupt the + * thread performing the task. + * + * @param runnable the runnable action + * @return the task + * + * @since 21 + */ + public static ForkJoinTask adaptInterruptible(Runnable runnable) { + return new AdaptedInterruptibleRunnable(runnable, null); + } + // Serialization support private static final long serialVersionUID = -7721805057305804111L; @@ -1533,6 +1583,14 @@ private void readObject(java.io.ObjectInputStream s) aux = new Aux(Thread.currentThread(), (Throwable)ex); } + static { + U = Unsafe.getUnsafe(); + STATUS = U.objectFieldOffset(ForkJoinTask.class, "status"); + AUX = U.objectFieldOffset(ForkJoinTask.class, "aux"); + Class dep1 = LockSupport.class; // ensure loaded + Class dep2 = Aux.class; + } + // Special subclasses for adaptors and internal tasks /** @@ -1615,28 +1673,23 @@ public String toString() { } /** - * Tasks with semantics conforming to ExecutorService - * conventions. Tasks are interruptible when cancelled, including - * cases of cancellation upon pool termination. In addition to - * recording the running thread to enable interrupt in - * cancel(true), the task checks for termination before executing - * the compute method, to cover shutdown races in which the task - * has not yet been cancelled on entry and might not otherwise be - * cancelled by others. + * Tasks with semantics conforming to ExecutorService conventions. + * Tasks are interruptible when cancelled, including cases of + * cancellation upon pool termination. In addition to recording + * the running thread to enable interrupt in cancel(true), the + * task checks for termination before executing the compute + * method, to cover shutdown races in which the task has not yet + * been cancelled on entry and might not otherwise be cancelled by + * others. */ - static abstract class ExternalTask extends ForkJoinTask + static abstract class InterruptibleTask extends ForkJoinTask implements RunnableFuture { transient volatile Thread runner; abstract T compute() throws Exception; - static boolean isTerminating(Thread t) { - ForkJoinPool p; - return ((t instanceof ForkJoinWorkerThread) && - (p = ((ForkJoinWorkerThread) t).pool) != null && - p.stopping()); - } public final boolean exec() { Thread t = Thread.currentThread(); - if (isTerminating(t)) + if ((t instanceof ForkJoinWorkerThread) && + ForkJoinPool.poolIsStopping(((ForkJoinWorkerThread)t).pool)) cancel(false); else { Thread.interrupted(); @@ -1657,17 +1710,23 @@ public boolean cancel(boolean mayInterruptIfRunning) { interruptIgnoringExceptions(runner); return isCancelled(); } - public final void run() { invoke(); } + public final void run() { quietlyInvoke(); } final Throwable wrapFJTCancellationException(CancellationException cx) { return new ExecutionException(cx); } + Object adaptee() { return null; } // for printing and diagnostics + public String toString() { + Object a = adaptee(); + String s = super.toString(); + return (a != null) ? (s + "[Wrapped task = " + a.toString() + "]") : s; + } private static final long serialVersionUID = 2838392045355241008L; } /** * Adapter for Callable-based interruptible tasks. */ - static final class AdaptedInterruptibleCallable extends ExternalTask { + static final class AdaptedInterruptibleCallable extends InterruptibleTask { @SuppressWarnings("serial") // Conditionally serializable final Callable callable; @SuppressWarnings("serial") // Conditionally serializable @@ -1679,16 +1738,14 @@ static final class AdaptedInterruptibleCallable extends ExternalTask { public final T getRawResult() { return result; } public final void setRawResult(T v) { result = v; } final T compute() throws Exception { return callable.call(); } - public String toString() { - return super.toString() + "[Wrapped task = " + callable + "]"; - } + final Object adaptee() { return callable; } private static final long serialVersionUID = 2838392045355241008L; } /** * Adapter for Runnable-based interruptible tasks. */ - static final class AdaptedInterruptibleRunnable extends ExternalTask { + static final class AdaptedInterruptibleRunnable extends InterruptibleTask { @SuppressWarnings("serial") // Conditionally serializable final Runnable runnable; @SuppressWarnings("serial") // Conditionally serializable @@ -1701,16 +1758,14 @@ static final class AdaptedInterruptibleRunnable extends ExternalTask { public final T getRawResult() { return result; } public final void setRawResult(T v) { } final T compute() { runnable.run(); return result; } - public String toString() { - return super.toString() + "[Wrapped task = " + runnable + "]"; - } + final Object adaptee() { return runnable; } private static final long serialVersionUID = 2838392045355241008L; } /** * Adapter for Runnables in which failure forces worker exception. */ - static final class RunnableExecuteAction extends ExternalTask { + static final class RunnableExecuteAction extends InterruptibleTask { @SuppressWarnings("serial") // Conditionally serializable final Runnable runnable; RunnableExecuteAction(Runnable runnable) { @@ -1720,6 +1775,7 @@ static final class RunnableExecuteAction extends ExternalTask { public final Void getRawResult() { return null; } public final void setRawResult(Void v) { } final Void compute() { runnable.run(); return null; } + final Object adaptee() { return runnable; } void onFJTExceptionSet(Throwable ex) { // if a handler, invoke it Thread t; java.lang.Thread.UncaughtExceptionHandler h; if ((h = ((t = Thread.currentThread()). @@ -1730,18 +1786,117 @@ void onFJTExceptionSet(Throwable ex) { // if a handler, invoke it } } } - public String toString() { - return super.toString() + "[Wrapped task = " + runnable + "]"; - } private static final long serialVersionUID = 5232453952276885070L; } - static { - U = Unsafe.getUnsafe(); - STATUS = U.objectFieldOffset(ForkJoinTask.class, "status"); - AUX = U.objectFieldOffset(ForkJoinTask.class, "aux"); - Class dep1 = LockSupport.class; // ensure loaded - Class dep2 = Aux.class; + /** + * Task (that is never forked) to hold results for + * ForkJoinPool.invokeAny, or to report exception if all subtasks + * fail or are cancelled or the pool is terminating. Both + * InvokeAnyRoot and InvokeAnyTask objects exist only transiently + * during invokeAny invocations, so serialization support would be + * nonsensical and is omitted. + */ + @SuppressWarnings("serial") + static final class InvokeAnyRoot extends InterruptibleTask { + volatile T result; + volatile int count; // number of tasks; decremented in case all tasks fail + InvokeAnyRoot() { } + final void tryComplete(InvokeAnyTask f, T v, Throwable ex, + boolean completed) { + if (f != null && !isDone()) { + if (ForkJoinPool.poolIsStopping(getPool())) + trySetCancelled(); + else if (f.setForkJoinTaskStatusMarkerBit() == 0) { + if (completed) { + result = v; + quietlyComplete(); + } + else if (U.getAndAddInt(this, COUNT, -1) <= 1) { + if (ex == null) + trySetCancelled(); + else + trySetException(ex); + } + } + } + } + public final T compute() { return null; } // never forked + public final T getRawResult() { return result; } + public final void setRawResult(T v) { } + + // Common support for timed and untimed versions of invokeAny + final T invokeAny(Collection> tasks, + ForkJoinPool pool, boolean timed, long nanos) + throws InterruptedException, ExecutionException, TimeoutException { + if ((count = tasks.size()) <= 0) + throw new IllegalArgumentException(); + if (pool == null) + throw new NullPointerException(); + InvokeAnyTask t = null; // list of submitted tasks + try { + for (Callable c : tasks) + pool.execute((ForkJoinTask) + (t = new InvokeAnyTask(c, this, t))); + return timed ? get(nanos, TimeUnit.NANOSECONDS) : get(); + } finally { + for (; t != null; t = t.pred) + t.onRootCompletion(); + } + } + + private static final Unsafe U; + private static final long COUNT; + static { + U = Unsafe.getUnsafe(); + COUNT = U.objectFieldOffset(InvokeAnyRoot.class, "count"); + } } + /** + * Task with results in InvokeAnyRoot (and never independently + * joined). + */ + @SuppressWarnings("serial") + static final class InvokeAnyTask extends InterruptibleTask { + final Callable callable; + final InvokeAnyRoot root; + final InvokeAnyTask pred; // to traverse on cancellation + InvokeAnyTask(Callable callable, InvokeAnyRoot root, + InvokeAnyTask pred) { + if (callable == null) throw new NullPointerException(); + this.callable = callable; + this.root = root; + this.pred = pred; + } + final Void compute() throws Exception { + InvokeAnyRoot r = root; + T v = null; Throwable ex = null; boolean completed = false; + if (r != null && !r.isDone()) { + try { + v = callable.call(); + completed = true; + } catch (Exception rex) { + ex = rex; + } finally { + r.tryComplete(this, v, ex, completed); + } + } + return null; + } + public final boolean cancel(boolean mayInterruptIfRunning) { + InvokeAnyRoot r; + boolean stat = super.cancel(mayInterruptIfRunning); + if ((r = root) != null) + r.tryComplete(this, null, null, false); + return stat; + } + final void onRootCompletion() { + if (!isDone()) + super.cancel(true); // no need for tryComplete + } + public final Void getRawResult() { return null; } + public final void setRawResult(Void v) { } + final Object adaptee() { return callable; } + } } From 201165c94bb8ff0b5f9f27bf353a22146c3a004c Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Mon, 3 Apr 2023 08:32:29 -0400 Subject: [PATCH 15/61] Separate invokeAll pending resolution --- .../java/util/concurrent/ForkJoinPool.java | 146 ++++++++---------- .../java/util/concurrent/ForkJoinTask.java | 95 ++++++------ 2 files changed, 113 insertions(+), 128 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index dec57eb82a9c1..a6c742696422c 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -431,9 +431,8 @@ public class ForkJoinPool extends AbstractExecutorService { * less than a dozen instructions unless the queue array is being * resized, during which contention is rare. To be conservative, * acquireRunStateLock is implemented as a spin/sleep loop. Here - * and elsewhere spin constants (SPIN_WAITS) are very short - * (currently 32) so have no discernable overhead even on systems - * with few available processors that may require blocking. In + * and elsewhere spin constants (SPIN_WAITS) are short enough to + * apply even on systems with few available processors. In * addition to checking pool status, reads of runState serve as * acquire fences before reading other non-volatile fields. * @@ -1677,24 +1676,24 @@ final void setClearThreadLocals() { private static final int LOCKED = 1 << 3; // lowest seqlock bit // spin/sleep limits for runState locking and elsewhere; powers of 2 + private static final int SPIN_WAITS = 1 << 6; // calls to onSpinWait private static final int MIN_SLEEP = 1 << 10; // approx 1 usec as nanos private static final int MAX_SLEEP = 1 << 20; // approx 1 sec as nanos - private static final int SPIN_WAITS = 1 << 5; // calls to onSpinWait // Instance fields volatile long stealCount; // collects worker nsteals volatile long threadIds; // for worker thread names - final long keepAlive; // milliseconds before dropping if idle final long bounds; // min, max threads packed as shorts + final long keepAlive; // milliseconds before dropping if idle final int config; // static configuration bits volatile int runState; // SHUTDOWN, STOP, TERMINATED, seq bits - WorkQueue[] queues; // main registry volatile CountDownLatch termination; // lazily constructed - final String workerNamePrefix; // null for common pool + final Predicate saturate; final ForkJoinWorkerThreadFactory factory; final UncaughtExceptionHandler ueh; // per-worker UEH - final Predicate saturate; + final String workerNamePrefix; // null for common pool final SharedThreadContainer container; + WorkQueue[] queues; // main registry @jdk.internal.vm.annotation.Contended("fjpctl") // segregate volatile long ctl; // main pool control @@ -1934,38 +1933,32 @@ else if ((int)c == 0) // was dropped on timeout */ final void signalWork() { int pc = parallelism; - for (long c = ctl;;) { + for (long c = ctl, u; ; c = u) { WorkQueue[] qs = queues; - Thread owner = null; - boolean create = false; + WorkQueue v = null; long ac = (c + RC_UNIT) & RC_MASK, nc; - int deficit = pc - (short)(c >>> TC_SHIFT); int sp = (int)c & ACTIVE, i = sp & SMASK; - if ((short)(c >>> RC_SHIFT) >= pc) + if (qs == null || qs.length <= i || (short)(c >>> RC_SHIFT) >= pc) break; - if (qs == null || qs.length <= i) - break; - WorkQueue v = qs[i]; - if (sp != 0 && v != null) { + if (sp != 0 && (v = qs[i]) != null) nc = (v.stackPred & SP_MASK) | (c & TC_MASK); - owner = v.owner; - } - else if (deficit <= 0) - break; - else { - create = true; + else if ((short)(c >>> TC_SHIFT) < pc) nc = ((c + TC_UNIT) & TC_MASK); - } - if (c == (c = compareAndExchangeCtl(c, nc | ac))) { - if (create) + else + break; + if (c == (u = compareAndExchangeCtl(c, nc | ac))) { + if (v == null) createWorker(); else { v.phase = sp; + Thread owner = v.owner; if (v.source < 0) U.unpark(owner); } break; } + else if ((u & RC_MASK) > (c & RC_MASK)) + break; // OK if lost to another signaller } } @@ -1982,12 +1975,12 @@ private void reactivate(WorkQueue w) { if ((w == null && (c & RC_MASK) > 0L) || sp == 0 || qs == null || qs.length <= i || (v = qs[i]) == null) break; - Thread owner = v.owner; long nc = (v.stackPred & SP_MASK) | pc; if (w != null && w != v && w.phase >= 0) break; if (c == (c = compareAndExchangeCtl(c, nc))) { v.phase = sp; + Thread owner = v.owner; if (v.source < 0) U.unpark(owner); if (v == w || w == null) @@ -2068,13 +2061,14 @@ final void runWorker(WorkQueue w) { if (w != null && (rs & STOP) == 0) { int phase = w.phase, r = w.stackPred; // use seed from registerWorker for (int srcs = ACTIVE;;) { + r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift if ((srcs = scan(w, srcs, r)) < 0) { if (rs != (rs = runState)) { - if ((rs & STOP) != 0) // stop or rescan because stale + if ((rs & STOP) != 0) // stop or rescan if stale break; srcs &= ACTIVE; } else { - long c; // try to inactivate and enqueue + long c; // try to inactivate/enqueue int p = phase + SS_SEQ, np = p | INACTIVE; long nc = ((((c = ctl) - RC_UNIT) & UC_MASK) | (p & ACTIVE & SP_MASK)); @@ -2090,7 +2084,6 @@ final void runWorker(WorkQueue w) { } } } - r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift } } } @@ -2149,23 +2142,26 @@ private int awaitWork(WorkQueue w, long queuedCtl) { return -1; // currently impossible int phase, quiescence; boolean idle = false; // true if pool idle - if ((queuedCtl & RC_MASK) <= 0L) { + long deadline = 0L; // for timed wait if idle + int active = (short)(queuedCtl >>> RC_SHIFT), + total = (short)(queuedCtl >>> TC_SHIFT), + spins = ((total << 1) & SMASK) + SPIN_WAITS; + if (active <= 0) { if ((quiescence = checkQuiescence()) < 0) return -1; // quiescent termination - else if (!(idle = (quiescence == 0)) && (ctl & RC_MASK) <= 0L) - reactivate(null); // ensure live if may be tasks + else if (idle = (quiescence == 0)) + deadline = keepAlive + System.currentTimeMillis(); + else + reactivate(null); // ensure live if may be tasks } - int spins = Math.max(SPIN_WAITS, ((short)(queuedCtl >>> TC_SHIFT)) << 1); - do { // spin before block for approx #accesses to scan+signal + do { // spin before block if ((phase = w.phase) >= 0) break; Thread.onSpinWait(); - } while (--spins != 0); + } while (--spins != 0); // ~ #accesses to scan+signal if (spins == 0) { - long deadline = idle ? System.currentTimeMillis() + keepAlive : 0L; LockSupport.setCurrentBlocker(this); // emulate LockSupport.park for (;;) { - Thread.interrupted(); // clear status if ((runState & STOP) != 0) return -1; w.source = INACTIVE; // enable unpark @@ -2181,6 +2177,7 @@ else if (!(idle = (quiescence == 0)) && (ctl & RC_MASK) <= 0L) deadline += keepAlive; // not at head; restart timer } } + Thread.interrupted(); // clear status for next park } LockSupport.setCurrentBlocker(null); } @@ -2286,37 +2283,28 @@ final void uncompensate() { * @param task the task * @param w caller's WorkQueue * @param internal true if w is owned by a ForkJoinWorkerThread - * @param locals true if check local queue * @return task status on exit, or UNCOMPENSATE for compensated blocking */ - final int helpJoin(ForkJoinTask task, WorkQueue w, boolean internal, - boolean locals) { + final int helpJoin(ForkJoinTask task, WorkQueue w, boolean internal) { int s = 0; if (task == null) return s; - if (locals && w != null) + if (w != null) w.tryRemoveAndExec(task, internal); if ((s = task.status) >= 0 && internal && w != null) { int wsrc = w.source & SMASK; int wid = (w.config & SMASK) + 1, r = wid + 1; - long sctl = ctl; // track stability + long sctl = 0L; // track stability outer: for (boolean rescan = true;;) { WorkQueue[] qs; if ((s = task.status) < 0) break; - if (!rescan) { - for (int spins = SPIN_WAITS; spins != 0; --spins) { - Thread.onSpinWait(); // stability check - if ((s = task.status) < 0) - break outer; - } - if (sctl == (sctl = ctl) && - (s = tryCompensate(sctl, false)) >= 0) - break; - } - rescan = false; if ((runState & STOP) != 0) break; + if (!rescan && sctl == (sctl = ctl) && + (s = tryCompensate(sctl, false)) >= 0) + break; + rescan = false; int n = ((qs = queues) == null) ? 0 : qs.length; scan: for (int i = n >>> 1; i > 0; --i, r += 2) { int j, cap; WorkQueue q; ForkJoinTask[] a; @@ -2379,26 +2367,19 @@ final int helpComplete(ForkJoinTask task, WorkQueue w, boolean internal) { int s = 0; if (task != null && (s = task.status) >= 0 && w != null) { int r = w.config + 1; // for indexing - long sctl = ctl; // track stability + long sctl = 0L; // track stability outer: for (boolean rescan = true, locals = true;;) { WorkQueue[] qs; if (locals && (s = w.helpComplete(task, internal, 0)) < 0) break; if ((s = task.status) < 0) break; - if (!rescan) { - for (int spins = SPIN_WAITS; spins != 0; --spins) { - Thread.onSpinWait(); // stabilty check - if ((s = task.status) < 0) - break outer; - } - if ((sctl == (sctl = ctl)) && - (!internal || (s = tryCompensate(sctl, false)) >= 0)) - break; - } - rescan = locals = false; if ((runState & STOP) != 0) break; + if (!rescan && sctl == (sctl = ctl) && + (!internal || (s = tryCompensate(sctl, false)) >= 0)) + break; + rescan = locals = false; int n = ((qs = queues) == null) ? 0 : qs.length; scan: for (int i = n; i > 0; --i, ++r) { int j, cap; WorkQueue q; ForkJoinTask[] a; @@ -3274,11 +3255,8 @@ public int setParallelism(int size) { return getAndSetParallelism(size); } - /** - * Common support for timed and untimed invokeAll - */ - private List> invokeAll(Collection> tasks, - boolean timed, long deadline) + @Override + public List> invokeAll(Collection> tasks) throws InterruptedException { ArrayList> futures = new ArrayList<>(tasks.size()); try { @@ -3289,7 +3267,7 @@ private List> invokeAll(Collection> tasks, } for (int i = futures.size() - 1; i >= 0; --i) ((ForkJoinTask)futures.get(i)) - .quietlyJoinPoolInvokeAllTask(timed, deadline); + .quietlyJoinPoolInvokeAllTask(0L); return futures; } catch (Throwable t) { for (Future e : futures) @@ -3298,17 +3276,27 @@ private List> invokeAll(Collection> tasks, } } - @Override - public List> invokeAll(Collection> tasks) - throws InterruptedException { - return invokeAll(tasks, false, 0L); - } - @Override public List> invokeAll(Collection> tasks, long timeout, TimeUnit unit) throws InterruptedException { - return invokeAll(tasks, true, unit.toNanos(timeout) + System.nanoTime()); + long deadline = ForkJoinTask.nanosTimeoutToDeadline(unit.toNanos(timeout)); + ArrayList> futures = new ArrayList<>(tasks.size()); + try { + for (Callable t : tasks) { + ForkJoinTask f = ForkJoinTask.adaptInterruptible(t); + futures.add(f); + poolSubmit(true, false, f); + } + for (int i = futures.size() - 1; i >= 0; --i) + ((ForkJoinTask)futures.get(i)) + .quietlyJoinPoolInvokeAllTask(deadline); + return futures; + } catch (Throwable t) { + for (Future e : futures) + e.cancel(true); + throw t; + } } @Override diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index 8a34cb36510ff..beffb084a09e7 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -275,14 +275,9 @@ final boolean casNext(Aux c, Aux v) { // used only in cancellation private static final int MARKER = 1 << 30; // utility marker private static final int HAVE_EXCEPTION = DONE | ABNORMAL | THROWN; - private static final int UNCOMPENSATE = 1 << 16; // helpJoin return sentinel + private static final int UNCOMPENSATE = 1 << 16; // helpJoin sentinel private static final int SMASK = 0xffff; // short bits for tags - // flags for "how" arguments in awaitDone and callers - private static final int RAN = 1 << 0; - private static final int INTERRUPTIBLE = 1 << 1; - private static final int TIMED = 1 << 2; - // Fields volatile int status; // accessed directly by pool and workers private transient volatile Aux aux; // either waiters or thrown Exception @@ -396,12 +391,12 @@ else if (casAux(a, next)) * * @param pool if nonnull, the pool of ForkJoinWorkerThread caller * @param compensation result from a helping method - * @param how flags for TIMED, INTERRUPTIBLE - * @param deadline if timed, timeout deadline + * @param interruptible if wait is interruptible + * @param deadline if nonzero, timeout deadline * @return ABNORMAL if interrupted, 0 on timeout, else status on exit */ - private int awaitDone(ForkJoinPool pool, int compensation, int how, - long deadline) { + private int awaitDone(ForkJoinPool pool, int compensation, + boolean interruptible, long deadline) { int s, interrupts = 0; // < 0 : throw; > 0 : re-interrupt boolean queued = false; Aux node = null; @@ -421,24 +416,23 @@ else if (((a = aux) == null || a.ex == null) && } if (queued) { // await signal or interrupt LockSupport.setCurrentBlocker(this); - for (;;) { + for (long ns;;) { if ((s = status) < 0) break; - else if (interrupts < 0) { + if (interrupts < 0) { s = ABNORMAL; // interrupted and not done break; } - else if (Thread.interrupted()) { + if (Thread.interrupted()) { if (!ForkJoinPool.poolIsStopping(pool)) - interrupts = (how & INTERRUPTIBLE) != 0 ? -1 : 1; + interrupts = interruptible ? -1 : 1; else { interrupts = 1; // re-assert if cleared cancelIgnoringExceptions(this); } } - else if ((how & TIMED) != 0) { - long ns = deadline - System.nanoTime(); - if (ns <= 0L) { + else if (deadline != 0L) { + if ((ns = deadline - System.nanoTime()) <= 0) { s = 0; break; } @@ -465,11 +459,11 @@ else if ((how & TIMED) != 0) { * Tries applicable helping steps while joining this task, * otherwise invokes blocking version of awaitDone * - * @param how flags for RAN, TIMED, INTERRUPTIBLE - * @param deadline if timed, timeout deadline + * @param interruptible if wait is interruptible + * @param deadline if nonzero, timeout deadline * @return ABNORMAL if interrupted, else status on exit */ - private int awaitDone(int how, long deadline) { + private int awaitDone(boolean interruptible, long deadline) { int s = 0; ForkJoinWorkerThread wt; ForkJoinPool p; ForkJoinPool.WorkQueue q; Thread t; boolean internal; @@ -483,11 +477,18 @@ private int awaitDone(int how, long deadline) { if (p != null && (s = ((this instanceof CountedCompleter) ? p.helpComplete(this, q, internal) : - p.helpJoin(this, q, internal, - !(this instanceof InterruptibleTask) && - (how & RAN) == 0))) < 0) + (this instanceof InterruptibleTask) && !internal ? status : + p.helpJoin(this, q, internal))) < 0) return s; - return awaitDone(internal ? p : null, s, how, deadline); + return awaitDone(internal ? p : null, s, interruptible, deadline); + } + + /** + * Convert timeout value to deadline, avoiding zero + */ + static long nanosTimeoutToDeadline(long nanos) { + long deadline = System.nanoTime() + nanos; + return (deadline == 0L) ? 1L : deadline; } /** @@ -714,7 +715,7 @@ public final ForkJoinTask fork() { public final V join() { int s; if ((s = status) >= 0) - s = awaitDone(0, 0L); + s = awaitDone(false, 0L); if ((s & ABNORMAL) != 0) reportException(s); return getRawResult(); @@ -732,7 +733,7 @@ public final V invoke() { int s; doExec(); if ((s = status) >= 0) - s = awaitDone(RAN, 0L); + s = awaitDone(false, 0L); if ((s & ABNORMAL) != 0) reportException(s); return getRawResult(); @@ -762,14 +763,14 @@ public static void invokeAll(ForkJoinTask t1, ForkJoinTask t2) { t2.fork(); t1.doExec(); if ((s1 = t1.status) >= 0) - s1 = t1.awaitDone(RAN, 0L); + s1 = t1.awaitDone(false, 0L); if ((s1 & ABNORMAL) != 0) { cancelIgnoringExceptions(t2); t1.reportException(s1); } else { if ((s2 = t2.status) >= 0) - s2 = t2.awaitDone(0, 0L); + s2 = t2.awaitDone(false, 0L); if ((s2 & ABNORMAL) != 0) t2.reportException(s2); } @@ -794,16 +795,15 @@ public static void invokeAll(ForkJoinTask... tasks) { Throwable ex = null; int last = tasks.length - 1; for (int i = last; i >= 0; --i) { - ForkJoinTask t; + ForkJoinTask t; int s; if ((t = tasks[i]) == null) { ex = new NullPointerException(); break; } if (i == 0) { - int s; t.doExec(); if ((s = t.status) >= 0) - s = t.awaitDone(RAN, 0L); + s = t.awaitDone(false, 0L); if ((s & ABNORMAL) != 0) ex = t.getException(s); break; @@ -812,11 +812,10 @@ public static void invokeAll(ForkJoinTask... tasks) { } if (ex == null) { for (int i = 1; i <= last; ++i) { - ForkJoinTask t; + ForkJoinTask t; int s; if ((t = tasks[i]) != null) { - int s; if ((s = t.status) >= 0) - s = t.awaitDone(0, 0L); + s = t.awaitDone(false, 0L); if ((s & ABNORMAL) != 0 && (ex = t.getException(s)) != null) break; } @@ -858,16 +857,15 @@ public static > Collection invokeAll(Collection Throwable ex = null; int last = ts.size() - 1; // nearly same as array version for (int i = last; i >= 0; --i) { - ForkJoinTask t; + ForkJoinTask t; int s; if ((t = ts.get(i)) == null) { ex = new NullPointerException(); break; } if (i == 0) { - int s; t.doExec(); if ((s = t.status) >= 0) - s = t.awaitDone(RAN, 0L); + s = t.awaitDone(false, 0L); if ((s & ABNORMAL) != 0) ex = t.getException(s); break; @@ -876,11 +874,10 @@ public static > Collection invokeAll(Collection } if (ex == null) { for (int i = 1; i <= last; ++i) { - ForkJoinTask t; + ForkJoinTask t; int s; if ((t = ts.get(i)) != null) { - int s; if ((s = t.status) >= 0) - s = t.awaitDone(0, 0L); + s = t.awaitDone(false, 0L); if ((s & ABNORMAL) != 0 && (ex = t.getException(s)) != null) break; } @@ -1059,7 +1056,7 @@ public final V get() throws InterruptedException, ExecutionException { int s; if ((s = status) >= 0) s = ((Thread.interrupted()) ? ABNORMAL : - awaitDone(INTERRUPTIBLE, 0L)); + awaitDone(true, 0L)); if ((s & ABNORMAL) != 0) reportExecutionException(s); return getRawResult(); @@ -1087,7 +1084,7 @@ public final V get(long timeout, TimeUnit unit) if (Thread.interrupted()) s = ABNORMAL; else if (nanos > 0L) - s = awaitDone(INTERRUPTIBLE | TIMED, nanos + System.nanoTime()); + s = awaitDone(true, nanosTimeoutToDeadline(nanos)); } if (s >= 0 || (s & ABNORMAL) != 0) reportExecutionException(s); @@ -1102,7 +1099,7 @@ else if (nanos > 0L) */ public final void quietlyJoin() { if (status >= 0) - awaitDone(0, 0L); + awaitDone(false, 0L); } /** @@ -1113,7 +1110,7 @@ public final void quietlyJoin() { public final void quietlyInvoke() { doExec(); if (status >= 0) - awaitDone(RAN, 0L); + awaitDone(false, 0L); } /** @@ -1136,7 +1133,7 @@ public final boolean quietlyJoin(long timeout, TimeUnit unit) if (Thread.interrupted()) s = ABNORMAL; else if (nanos > 0L) - s = awaitDone(INTERRUPTIBLE | TIMED, nanos + System.nanoTime()); + s = awaitDone(true, nanosTimeoutToDeadline(nanos)); } if (s == ABNORMAL) throw new InterruptedException(); @@ -1158,21 +1155,21 @@ public final boolean quietlyJoinUninterruptibly(long timeout, int s; long nanos = unit.toNanos(timeout); if ((s = status) >= 0 && nanos > 0L) - s = awaitDone(TIMED, nanos + System.nanoTime()); + s = awaitDone(false, nanosTimeoutToDeadline(nanos)); return (s < 0); } /** * Utility for possibly-timed ForkJoinPool.invokeAll */ - final void quietlyJoinPoolInvokeAllTask(boolean timed, long deadline) + final void quietlyJoinPoolInvokeAllTask(long deadline) throws InterruptedException { int s; if ((s = status) >= 0) { if (Thread.interrupted()) s = ABNORMAL; - else if (!timed || deadline - System.nanoTime() > 0L) - s = awaitDone(INTERRUPTIBLE | (timed ? TIMED : 0), deadline); + else if (deadline == 0L || deadline - System.nanoTime() > 0L) + s = awaitDone(true, deadline); if (s == ABNORMAL) throw new InterruptedException(); else if (s >= 0) From 12fabf61d2ac502f0fd0080e853e5e16254ba168 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sat, 8 Apr 2023 10:22:57 -0400 Subject: [PATCH 16/61] Add invokeAnyUninterruptibly --- .../java/util/concurrent/ForkJoinPool.java | 449 ++++++++++-------- .../java/util/concurrent/ForkJoinTask.java | 229 +++++---- 2 files changed, 347 insertions(+), 331 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index a6c742696422c..4c88e9e622f08 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -463,8 +463,8 @@ public class ForkJoinPool extends AbstractExecutorService { * these indices remain OK (with at most some unnecessary extra * work) even if an underlying worker failed and was replaced by * another at the same index. There are two contexts requiring - * precautions about this: In checkQuiescence, the SeqLock is used - * to recheck upon queue array updates. And during termination, + * extra precautions: In checkQuiescence, the SeqLock is used to + * recheck upon queue array updates. And during termination, * worker queue array updates are disabled. * * Queuing Idle Workers. Unlike HPC work-stealing frameworks, we @@ -479,7 +479,7 @@ public class ForkJoinPool extends AbstractExecutorService { * is the main limiting factor in overall performance, which is * compounded at program start-up by JIT compilation and * allocation. On the other hand, throughput degrades when too - * many threads poll for too few tasks. + * many threads poll for too few tasks. (See below.) * * The "ctl" field atomically maintains total and "released" * worker counts, plus the head of the available worker queue @@ -550,14 +550,14 @@ public class ForkJoinPool extends AbstractExecutorService { * * * If instead, tasks come in serially from only a single * producer, each worker taking its first (since the last - * activation) task from a queue should signal another if there - * are more tasks in that queue. This is equivalent to, but - * generally faster than, arranging the stealer take multiple - * tasks, re-pushing one or more on its own queue, and + * activation) task from a queue should propagate a signal if + * there are more tasks in that queue. This is equivalent to, + * but generally faster than, arranging the stealer take + * multiple tasks, re-pushing one or more on its own queue, and * signalling (because its queue is empty), also resulting in * logarithmic full activation time * - * * Because we don't know about usage patterns (or most commonly, + * * Because we don't know about usage patterns (or most commonly, * mixtures), we use both approaches, which present even more * opportunities to over-signal. Note that in either of these * contexts, signals may be (and often are) unnecessary because @@ -582,12 +582,16 @@ public class ForkJoinPool extends AbstractExecutorService { * method scan) as an encoded sliding window of the last two * sources, and stop signalling when the last two were from the * same source. And similarly not retry when two CAS failures - * were from the same source. This may result in transiently too - * few workers, but once they poll from the new source, they - * rapidly reactivate others. (An implementation / encoding - * quirk: Because of the offset, unnecessary signals for the - * maximum-numbered 32767th worker, which is unlikely to ever be - * produced, are not suppressed.) + * were from the same source. Additionally, we allow attempts to + * propagate signals (argument "propagated" in signalWork) to + * return without singalling if another contending call to + * signalWork increased the number of active workers. These + * mechanisms may result in transiently too few workers, but + * once workers poll from a new source, they rapidly reactivate + * others. (An implementation / encoding quirk: Because of the + * offset, unnecessary signals for the maximum-numbered 32767th + * worker, which is unlikely to ever be produced, are not + * suppressed.) * * * Despite these, signal contention and overhead effects still * occur during ramp-up and ramp-down of small computations. @@ -596,26 +600,25 @@ public class ForkJoinPool extends AbstractExecutorService { * execution of) tasks by polling a pseudo-random permutation of * the array (by starting at a random index, and using a constant * cyclically exhaustive stride.) This maintains top-level - * randomized fairness at the cost of poor locaility (other forms - * of scanning while helping etc described below have better - * locality.) It uses the same basic polling method as - * WorkQueue.poll(), but restarts with a different permutation on - * each invocation. The pseudorandom generator need not have - * high-quality statistical properties in the long term. We use - * Marsaglia XorShifts, seeded with the Weyl sequence from - * ThreadLocalRandom probes, which are cheap and suffice. Scans do - * not otherwise explicitly take into account core affinities, - * loads, cache localities, etc, However, they do exploit temporal - * locality (which usually approximates these) by preferring to - * re-poll from the same queue (using method tryPoll()) after a - * successful poll before trying others (see method topLevelExec), - * which also reduces bookkeeping and scanning overhead. This - * also reduces fairness, which is partially counteracted by - * giving up on contention. + * randomized fairness at the cost of poor locaility, but enables + * faster less-fair techniques once a top-level task is found. It + * uses the same basic polling method as WorkQueue.poll(), but + * restarts with a different permutation on each invocation. The + * pseudorandom generator need not have high-quality statistical + * properties in the long term. We use Marsaglia XorShifts, seeded + * with the Weyl sequence from ThreadLocalRandom probes, which are + * cheap and suffice. Scans do not otherwise explicitly take into + * account core affinities, loads, cache localities, etc, However, + * they do exploit temporal locality (which usually approximates + * these) by preferring to re-poll from the same queue (using + * method tryPoll()) after a successful poll before trying others + * (see method topLevelExec), which also reduces bookkeeping and + * scanning overhead. This also reduces fairness, which is + * partially counteracted by giving up on contention. * * Deactivation. When method scan indicates that no tasks are - * found by a worker, it deactivates (see awaitWork). Note that - * not finding tasks doesn't mean that there won't soon be + * found by a worker, it tries to deactivate (in runWorker). Note + * that not finding tasks doesn't mean that there won't soon be * some. Further, a scan may give up under contention, returning * even without knowing whether any tasks are still present, which * is OK, given the above signalling rules that will eventually @@ -628,9 +631,9 @@ public class ForkJoinPool extends AbstractExecutorService { * using quieter short spinwaits in awaitWork and elsewhere. * Those in awaitWork are set to small values that only cover * near-miss scenarios for inactivate/activate races. Because idle - * workers are often not yet blocked (via LockSupport.park), we - * use the WorkQueue source field to advertise that a waiter - * actually needs unparking upon signal. + * workers are often not yet blocked (parked), we use the + * WorkQueue source field to advertise that a waiter actually + * needs unparking upon signal. * * Quiescence. Workers scan looking for work, giving up when they * don't find any, without being sure that none are available. @@ -650,8 +653,8 @@ public class ForkJoinPool extends AbstractExecutorService { * SeqLock to retry on queue array updates. (It also reports * quiescence if the pool is terminating.) A true report means * only that there was a moment at which quiescence held; further - * actions based on this may be racy (except when using seq lock - * upgrade to trigger quiescent termination). False negatives are + * actions based on this may be racy except when using seq lock + * upgrade to trigger quiescent termination. False negatives are * inevitable (for example when queues indices lag updates, as * described above), which is accommodated when (tentatively) idle * by scanning for work etc, and then re-invoking. This includes @@ -662,7 +665,7 @@ public class ForkJoinPool extends AbstractExecutorService { * rescan. Method helpQuiesce acts similarly but cannot rely on * ctl counts to determine that all workers are inactive because * the caller and any others executing helpQuiesce are not - * included. + * included in counts. * * The quiescence check in awaitWork also serves as a safeguard * when all other workers gave up prematurely and inactivated (due @@ -679,7 +682,7 @@ public class ForkJoinPool extends AbstractExecutorService { * termination only when accessing pool state (usually the * "queues" array). This may take a while but suffices for * structured computational tasks. But not necessarily for - * others. Class ExecutorServicesTask (see below) further arranges + * others. Class InterruptibleTask (see below) further arranges * runState checks before executing task bodies, and ensures * interrupts while terminating. Even so, there are no guarantees * after an abrupt shutdown that remaining tasks complete normally @@ -720,8 +723,8 @@ public class ForkJoinPool extends AbstractExecutorService { * space is bounded in tree/dag structured procedurally parallel * designs to be no more than that if a task were executed only by * the joining thread. This is arranged by associated task - * subclasses and internal tags that also help detect and control - * the ways in which this may occur. + * subclasses that also help detect and control the ways in which + * this may occur. * * Normally, the first option when joining a task that is not done * is to try to take it from the local queue and run it. Method @@ -731,7 +734,7 @@ public class ForkJoinPool extends AbstractExecutorService { * when stolen, steal-backs are restricted to the same rules as * stealing (polling), which requires additional bookkeeping and * scanning. This cost is still very much worthwhile because of - * its impact on task scheduling. + * its impact on task scheduling and resource control. * * The two methods for finding and executing subtasks vary in * details. The algorithm in helpJoin entails a form of "linear @@ -802,7 +805,7 @@ public class ForkJoinPool extends AbstractExecutorService { * compensation during attempts to obtain a stable snapshot. But * users now rely upon the fact that if isReleasable always * returns false, the API can be used to obtain precautionary - * compensation, which is sometimes their only reasonable option + * compensation, which is sometimes the only reasonable option * when running unknown code in tasks; which is now supported more * simply (see method beginCompensatedBlock). * @@ -989,7 +992,7 @@ public class ForkJoinPool extends AbstractExecutorService { * to interrupt all workers and cancel all tasks. It also uses a * CountDownLatch instead of a Condition for termination because * of lock change. - * * Many other adjustments to avoid performance regressions due + * * Many other changes to avoid performance regressions due * to the above. */ @@ -1001,11 +1004,6 @@ public class ForkJoinPool extends AbstractExecutorService { */ static final long DEFAULT_KEEPALIVE = 60_000L; - /** - * Undershoot tolerance for idle timeouts - */ - static final long TIMEOUT_SLOP = 20L; - /** * The default value for common pool maxSpares. Overridable using * the "java.util.concurrent.ForkJoinPool.common.maximumSpares" @@ -1027,14 +1025,13 @@ public class ForkJoinPool extends AbstractExecutorService { static final int SMASK = 0xffff; // short bits == max index static final int MAX_CAP = 0x7fff; // max #workers - 1 - // {pool, workQueue}.config bits and sentinels + // Bits and sentinels used by both pool and workQueues static final int FIFO = 1 << 16; // fifo queue or access mode - static final int ALIVE = 1 << 17; // initialized, not deregistered + static final int REGISTERED = 1 << 17; // true when alive static final int CLEAR_TLS = 1 << 18; // set for Innocuous workers static final int TRIMMED = 1 << 19; // timed out while idle static final int ISCOMMON = 1 << 20; // set for common pool static final int PRESET_SIZE = 1 << 21; // size was set by property - static final int UNCOMPENSATE = 1 << 16; // tryCompensate return // Static utilities @@ -1158,7 +1155,7 @@ public ForkJoinWorkerThread run() { */ static final class WorkQueue { int stackPred; // pool stack (ctl) predecessor link - int config; // index, mode, ORed with ALIVE after init + int config; // index, mode, REGISTERED bits int base; // index of next slot for poll ForkJoinTask[] array; // the queued tasks; power of 2 size final ForkJoinWorkerThread owner; // owning thread or null if shared @@ -1228,11 +1225,10 @@ final int queueSize() { * * @param task the task. Caller must ensure non-null. * @param p the pool. Must be non-null unless terminating. - * @param signal true if signal when queue may have been or appeared empty + * @param signal true if signal if queue may have been or appeared empty * @throws RejectedExecutionException if array cannot be resized */ - final void push(ForkJoinTask task, ForkJoinPool pool, - boolean signal) { + final void push(ForkJoinTask task, ForkJoinPool p, boolean signal) { ForkJoinTask[] a = array; int b = base, s = top++, cap, m; if (a != null && (cap = a.length) > 0) { @@ -1244,8 +1240,8 @@ final void push(ForkJoinTask task, ForkJoinPool pool, if (a[m & (s - 1)] != null) signal = false; // not empty } - if (signal && pool != null) - pool.signalWork(); + if (signal && p != null) + p.signalWork(false); } } @@ -1256,29 +1252,31 @@ final void push(ForkJoinTask task, ForkJoinPool pool, */ private void growAndPush(ForkJoinTask task, ForkJoinTask[] a, int s) { - int cap; - if (a != null && (cap = a.length) > 0) { // always true here + int cap; // rapidly grow until large + if (a != null && (cap = a.length) > 0) { int newCap = (cap < 1 << 24) ? cap << 2 : cap << 1; - ForkJoinTask[] newArray; // rapidly grow until large - try { - newArray = new ForkJoinTask[newCap]; - } catch (Throwable ex) { - top = s; - access = 0; - throw new RejectedExecutionException( - "Queue capacity exceeded"); - } - if (newCap > 0) { // always true here - int newMask = newCap - 1, k = s, b = k - cap, m = cap - 1; - do { // poll old, push to new; exit if lose to other pollers - newArray[k & newMask] = task; - } while (--k != b && - (task = getAndClearSlot(a, k & m)) != null); - U.storeFence(); - array = newArray; - access = 0; + int newMask = newCap - 1, k = s, b = k - cap, m = cap - 1; + if (newCap > 0) { + ForkJoinTask[] newArray = null; + try { + newArray = new ForkJoinTask[newCap]; + } catch (OutOfMemoryError ex) { + } + if (newArray != null) { + do { // poll old, push to new; exit if lose to pollers + newArray[k & newMask] = task; + } while (--k != b && + (task = getAndClearSlot(a, k & m)) != null); + U.storeFence(); + array = newArray; + access = 0; + return; + } } } + top = s; // revert on failure + access = 0; + throw new RejectedExecutionException("Queue capacity exceeded"); } /** @@ -1385,7 +1383,7 @@ else if ((u = cmpExSlotToNull(a, k, t)) == t) { base = nb; U.storeFence(); if (pool != null && a[nk] != null) - pool.signalWork(); // propagate + pool.signalWork(true); // propagate return t; } if (u == null && access == 0 && top - b <= 0) @@ -1578,7 +1576,7 @@ else if (a[nk] == null) final boolean isApparentlyUnblocked() { Thread wt; Thread.State s; return ((wt = owner) != null && phase >= 0 && - (config & ALIVE) != 0 && + (config & REGISTERED) != 0 && (s = wt.getState()) != Thread.State.BLOCKED && s != Thread.State.WAITING && s != Thread.State.TIMED_WAITING); @@ -1680,6 +1678,11 @@ final void setClearThreadLocals() { private static final int MIN_SLEEP = 1 << 10; // approx 1 usec as nanos private static final int MAX_SLEEP = 1 << 20; // approx 1 sec as nanos + /** + * Undershoot tolerance for idle timeouts, in millisecs + */ + private static final long TIMEOUT_SLOP = 20L; + // Instance fields volatile long stealCount; // collects worker nsteals volatile long threadIds; // for worker thread names @@ -1759,8 +1762,8 @@ private void spinLockRunState() { // spin/sleep waits = 0; } else if (waits < SPIN_WAITS) { - Thread.onSpinWait(); ++waits; + Thread.onSpinWait(); } else { if (waits < MIN_SLEEP) @@ -1827,18 +1830,18 @@ final void registerWorker(WorkQueue w) { WorkQueue[] qs; int n; ThreadLocalRandom.localInit(); int seed = ThreadLocalRandom.getProbe(); - int cfg = (config & FIFO) | ALIVE; + int cfg = (config & FIFO) | REGISTERED; if (w != null && (runState & STOP) == 0 && (qs = queues) != null && (n = qs.length) > 0) { w.array = new ForkJoinTask[INITIAL_QUEUE_CAPACITY]; cfg |= w.config; w.stackPred = seed; int id = (seed << 1) | 1; // initial index guess - { // try to find slot without lock (nearly same code below if fail) + { // try to find slot without lock (repeated below if fail) int k = n, m = n - 1; for (; qs[id &= m] != null && k > 0; id -= 2, k -= 2); if (k == 0) - id = -1; + id = n | 1; } acquireRunStateLock(); try { @@ -1848,7 +1851,6 @@ final void registerWorker(WorkQueue w) { qs[id] == null) qs[id] = w; else if (qs != null && (n = qs.length) > 0) { - id = (seed << 1) | 1; // restart int k = n, m = n - 1; for (; qs[id &= m] != null && k > 0; id -= 2, k -= 2); if (k == 0) { @@ -1892,8 +1894,8 @@ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { WorkQueue w; int cfg; if ((w = (wt == null) ? null : wt.workQueue) == null) cfg = 0; - else if (((cfg = w.config) & ALIVE) != 0) { - w.config = cfg & ~ALIVE; + else if (((cfg = w.config) & REGISTERED) != 0) { + w.config = cfg & ~REGISTERED; if (w.phase < 0) // possible if stopped before released reactivate(w); if (w.top - w.base > 0) { @@ -1908,7 +1910,7 @@ else if (((cfg = w.config) & ALIVE) != 0) { (TC_MASK & (c - TC_UNIT)) | (SP_MASK & c))))); else if ((int)c == 0) // was dropped on timeout - cfg &= ~ALIVE; // suppress replacement if last + cfg &= ~REGISTERED; // suppress replacement if last if ((tryTerminate(false, false) & STOP) == 0 && w != null) { WorkQueue[] qs; int n, i; // remove index unless terminating @@ -1921,8 +1923,8 @@ else if ((int)c == 0) // was dropped on timeout stealCount += ns; // accumulate steals } releaseRunStateLock(); - if ((cfg & ALIVE) != 0) - signalWork(); // may replace unless trimmed or uninitialized + if ((cfg & REGISTERED) != 0) + signalWork(false); // may replace unless trimmed or uninitialized } if (ex != null) ForkJoinTask.rethrow(ex); @@ -1930,15 +1932,16 @@ else if ((int)c == 0) // was dropped on timeout /** * Releases an idle worker, or creates one if not enough exist. + * @param propagated true if can exit on any signal success */ - final void signalWork() { + final void signalWork(boolean propagated) { int pc = parallelism; for (long c = ctl, u; ; c = u) { WorkQueue[] qs = queues; WorkQueue v = null; long ac = (c + RC_UNIT) & RC_MASK, nc; int sp = (int)c & ACTIVE, i = sp & SMASK; - if (qs == null || qs.length <= i || (short)(c >>> RC_SHIFT) >= pc) + if ((short)(c >>> RC_SHIFT) >= pc || qs == null || qs.length <= i) break; if (sp != 0 && (v = qs[i]) != null) nc = (v.stackPred & SP_MASK) | (c & TC_MASK); @@ -1951,23 +1954,23 @@ else if ((short)(c >>> TC_SHIFT) < pc) createWorker(); else { v.phase = sp; - Thread owner = v.owner; if (v.source < 0) - U.unpark(owner); + U.unpark(v.owner); } break; } - else if ((u & RC_MASK) > (c & RC_MASK)) - break; // OK if lost to another signaller + else if (propagated && (u & RC_MASK) > (c & RC_MASK)) + break; // another signaller succeeded } } /** * Reactivates the given worker (and possibly others if not top of - * ctl stack), or any worker if null and idle. + * ctl stack), or any worker if null and idle. A variant of the + * no-create case of signalWork. */ private void reactivate(WorkQueue w) { - for (long c = ctl;;) { // mostly, the no-create case of signalWork + for (long c = ctl, u; ; c = u) { WorkQueue v; WorkQueue[] qs = queues; int sp = (int)c & ACTIVE, i = sp & SMASK; @@ -1978,14 +1981,12 @@ private void reactivate(WorkQueue w) { long nc = (v.stackPred & SP_MASK) | pc; if (w != null && w != v && w.phase >= 0) break; - if (c == (c = compareAndExchangeCtl(c, nc))) { + if (c == (u = compareAndExchangeCtl(c, nc))) { v.phase = sp; - Thread owner = v.owner; if (v.source < 0) - U.unpark(owner); + U.unpark(v.owner); if (v == w || w == null) break; - c = ctl; } } } @@ -2011,24 +2012,22 @@ private boolean tryTrim(WorkQueue w) { /** * Internal version of isQuiescent and related functionality. - * Returns negative if terminating, 0 if submission queues are - * empty and unlocked, and all workers are inactive, else - * positive. + * Returns true if terminating or submission queues are empty and + * unlocked, and all workers are inactive. If so, and quiescent + * shutdown is enabled, sets runState mode to STOP. */ - private int checkQuiescence() { + private boolean checkQuiescence() { boolean scanned = false; outer: for (int prevState = -1, rs = runState; ; prevState = rs) { WorkQueue[] qs; int n; - if ((rs & STOP) != 0) // terminating - return -1; - if ((ctl & RC_MASK) > 0L) - break; // active workers exist - if (scanned) { - if ((rs & SHUTDOWN) == 0) - return 0; - if (casRunState(rs, rs | STOP)) // try to trigger termination - return -1; - scanned = false; // inconsistent; retry + if ((rs & STOP) != 0) // terminating + return true; + if ((ctl & RC_MASK) != 0L) + break; // active + if (scanned) { // try to trigger termination + if ((rs & SHUTDOWN) == 0 || casRunState(rs, rs | STOP)) + return true; + scanned = false; // inconsistent; retry } if ((qs = queues) != null && (n = qs.length) > 0) { for (int i = 0;;) { // alternates even for external, odd internal @@ -2045,9 +2044,9 @@ private int checkQuiescence() { } } if ((rs = runState) == prevState && (rs & LOCKED) == 0) - scanned = true; // same unlocked state + scanned = true; // same unlocked state } - return 1; + return false; } /** @@ -2120,12 +2119,12 @@ else if ((u = WorkQueue.cmpExSlotToNull(a, k, t)) == t) { q.base = nb; w.source = qsrcs; if (qsrcs != srcs && a[nk] != null) - signalWork(); // propagate at most twice/run + signalWork(true); // propagate at most twice/run w.topLevelExec(t, q); return qsrcs; } else if (u != null || (qsrcs != srcs && a[nk] != null)) - return qsrcs; // limit retries under contention + return qsrcs; // limit retries if contended } } return srcs | INACTIVE; @@ -2139,48 +2138,44 @@ else if (u != null || (qsrcs != srcs && a[nk] != null)) */ private int awaitWork(WorkQueue w, long queuedCtl) { if (w == null) - return -1; // currently impossible - int phase, quiescence; - boolean idle = false; // true if pool idle - long deadline = 0L; // for timed wait if idle - int active = (short)(queuedCtl >>> RC_SHIFT), - total = (short)(queuedCtl >>> TC_SHIFT), - spins = ((total << 1) & SMASK) + SPIN_WAITS; - if (active <= 0) { - if ((quiescence = checkQuiescence()) < 0) - return -1; // quiescent termination - else if (idle = (quiescence == 0)) - deadline = keepAlive + System.currentTimeMillis(); - else - reactivate(null); // ensure live if may be tasks - } - do { // spin before block + return -1; // currently impossible + boolean idle; // true if pool idle + long deadline = 0L; // for timed wait if idle + int nspins = // approximate avg #accesses needed to scan+signal + (int)((queuedCtl & TC_MASK) >>> (TC_SHIFT - 1)) + SPIN_WAITS + 1; + if ((queuedCtl & RC_MASK) > 0L) + idle = false; + else if (idle = checkQuiescence()) + deadline = keepAlive + System.currentTimeMillis(); + else + reactivate(null); // ensure live if may be tasks + int phase; // nonnegative on normal return + outer: for (;;) { // await signal or termination + int spins = nspins; + if ((runState & STOP) != 0) + return -1; + do { // spin to cushion near-misses + if ((phase = w.phase) >= 0) + break outer; + Thread.onSpinWait(); + } while (--spins > 0); + LockSupport.setCurrentBlocker(this); // emulate LockSupport.park + w.source = INACTIVE; // enable unpark + if (w.phase < 0) + U.park(idle, deadline); + w.source = 0; // disable unpark + LockSupport.setCurrentBlocker(null); if ((phase = w.phase) >= 0) break; - Thread.onSpinWait(); - } while (--spins != 0); // ~ #accesses to scan+signal - if (spins == 0) { - LockSupport.setCurrentBlocker(this); // emulate LockSupport.park - for (;;) { - if ((runState & STOP) != 0) - return -1; - w.source = INACTIVE; // enable unpark - if (w.phase < 0) - U.park(idle, deadline); - w.source = 0; // disable unpark - if ((phase = w.phase) >= 0) - break; - if (idle) { // check for idle timeout - if (deadline - System.currentTimeMillis() < TIMEOUT_SLOP) { - if (tryTrim(w)) - return -1; - deadline += keepAlive; // not at head; restart timer - } + if (idle) { // check for idle timeout + if (deadline - System.currentTimeMillis() < TIMEOUT_SLOP) { + if (tryTrim(w)) + return -1; + deadline += keepAlive; // not at head; restart timer } - Thread.interrupted(); // clear status for next park } - LockSupport.setCurrentBlocker(null); - } + Thread.interrupted(); // clear status for next park + } // loop on interrupt or stale unpark return phase; } @@ -2218,53 +2213,49 @@ private ForkJoinTask pollScan(boolean submissionsOnly) { * unblocked. * * @param c incoming ctl value - * @param canSaturate to override saturate predicate * @return UNCOMPENSATE: block then adjust, 0: block, -1 : retry */ - private int tryCompensate(long c, boolean canSaturate) { + private int tryCompensate(long c) { Predicate sat; - long b = bounds; // unpack fields - int pc = parallelism, rs; + long b = bounds; // unpack fields + int pc = parallelism; int minActive = (short)(b & SMASK), maxTotal = (short)(b >>> SWIDTH) + pc, active = (short)(c >>> RC_SHIFT), total = (short)(c >>> TC_SHIFT), sp = (int)c & ACTIVE; - if (((rs = runState) & STOP) != 0) // terminating - return 0; - else if ((rs & LOCKED) != 0) // unstable - return -1; - else if (sp != 0 && active <= pc) { // activate idle worker + if (sp != 0 && active <= pc) { // activate idle worker WorkQueue[] qs; WorkQueue v; int i; if (ctl == c && (qs = queues) != null && qs.length > (i = sp & SMASK) && (v = qs[i]) != null) { - Thread owner = v.owner; long nc = (v.stackPred & SP_MASK) | (UC_MASK & c); if (compareAndSetCtl(c, nc)) { v.phase = sp; if (v.source < 0) - U.unpark(owner); + U.unpark(v.owner); return UNCOMPENSATE; } } - return -1; // retry + return -1; // inconsistent; retry } - else if (active > minActive && total >= pc) { // reduce active workers + if (active > minActive && total >= pc) { // reduce active workers long nc = ((RC_MASK & (c - RC_UNIT)) | (~RC_MASK & c)); return compareAndSetCtl(c, nc) ? UNCOMPENSATE : -1; } - else if (total < maxTotal && total < MAX_CAP) { // expand pool + if ((runState & STOP) != 0) // rely on caller to notice + return -1; + if (total < maxTotal && total < MAX_CAP) { // expand pool long nc = ((c + TC_UNIT) & TC_MASK) | (c & ~TC_MASK); - return (runState != rs || (!compareAndSetCtl(c, nc)) ? -1 : - !createWorker() ? 0 : UNCOMPENSATE); + return (!compareAndSetCtl(c, nc) ? -1 : + createWorker() ? UNCOMPENSATE : + (runState & STOP) != 0 ? -1 : 0); // recheck } - else if (!compareAndSetCtl(c, c)) // validate + if (!compareAndSetCtl(c, c)) // validate return -1; - else if (canSaturate || ((sat = saturate) != null && sat.test(this))) + if ((sat = saturate) != null && sat.test(this)) return 0; - else - throw new RejectedExecutionException( - "Thread limit exceeded replacing blocked worker"); + throw new RejectedExecutionException( + "Thread limit exceeded replacing blocked worker"); } /** @@ -2302,7 +2293,7 @@ final int helpJoin(ForkJoinTask task, WorkQueue w, boolean internal) { if ((runState & STOP) != 0) break; if (!rescan && sctl == (sctl = ctl) && - (s = tryCompensate(sctl, false)) >= 0) + (s = tryCompensate(sctl)) >= 0) break; rescan = false; int n = ((qs = queues) == null) ? 0 : qs.length; @@ -2338,13 +2329,13 @@ final int helpJoin(ForkJoinTask task, WorkQueue w, boolean internal) { } if (t != task && !eligible) break; + rescan = true; // restart at same index if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { q.base = nb; w.source = src; t.doExec(); w.source = wsrc; } - rescan = true; // restart at same index break scan; } } @@ -2377,7 +2368,7 @@ final int helpComplete(ForkJoinTask task, WorkQueue w, boolean internal) { if ((runState & STOP) != 0) break; if (!rescan && sctl == (sctl = ctl) && - (!internal || (s = tryCompensate(sctl, false)) >= 0)) + (!internal || (s = tryCompensate(sctl)) >= 0)) break; rescan = locals = false; int n = ((qs = queues) == null) ? 0 : qs.length; @@ -2410,13 +2401,13 @@ final int helpComplete(ForkJoinTask task, WorkQueue w, boolean internal) { } if (!eligible) break; + rescan = true; if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { q.base = nb; U.storeFence(); - t.doExec(); locals = true; // process local queue + t.doExec(); } - rescan = true; break scan; } } @@ -2486,9 +2477,9 @@ else if (q.phase > 0) else if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { q.base = nb; w.source = src; + rescan = locals = true; t.doExec(); w.source = wsrc; - rescan = locals = true; break scan; } } @@ -2527,7 +2518,7 @@ else if (waits == 0) // same as spinLockRunState except * @return positive if quiescent, negative if interrupted, else 0 */ private int externalHelpQuiesce(long nanos, boolean interruptible) { - if (checkQuiescence() > 0) { + if (!checkQuiescence()) { long startTime = System.nanoTime(); long maxSleep = Math.min(nanos >>> 8, MAX_SLEEP); for (int waits = 0;;) { @@ -2538,7 +2529,7 @@ else if ((t = pollScan(false)) != null) { waits = 0; t.doExec(); } - else if (checkQuiescence() <= 0) + else if (checkQuiescence()) break; else if (System.nanoTime() - startTime > nanos) return 0; @@ -2605,7 +2596,7 @@ final WorkQueue submissionQueue(boolean isSubmit) { if ((qs = queues) == null || (n = qs.length) <= 0) break; else if ((q = qs[i = (n - 1) & id]) == null) { - WorkQueue w = new WorkQueue(null, id | ALIVE); + WorkQueue w = new WorkQueue(null, id | REGISTERED); w.array = new ForkJoinTask[INITIAL_QUEUE_CAPACITY]; acquireRunStateLock(); // install under lock if (queues == qs && qs[i] == null) // OK even if stopping @@ -2632,11 +2623,12 @@ else if (isSubmit && (runState & SHUTDOWN) != 0) { private ForkJoinTask poolSubmit(boolean signalIfEmpty, boolean external, ForkJoinTask task) { + Thread t; U.storeStoreFence(); // ensure safe publication if (task == null) throw new NullPointerException(); - Thread t = Thread.currentThread(); - ForkJoinWorkerThread wt = ((t instanceof ForkJoinWorkerThread) ? - (ForkJoinWorkerThread)t : null); + ForkJoinWorkerThread wt = + ((t = Thread.currentThread()) instanceof ForkJoinWorkerThread) ? + (ForkJoinWorkerThread)t : null; WorkQueue q = ((!external && wt != null && wt.pool == this) ? wt.workQueue : submissionQueue(true)); q.push(task, this, signalIfEmpty); @@ -2764,7 +2756,7 @@ private int tryTerminate(boolean now, boolean enable) { else { if ((isShutdown = (rs & SHUTDOWN)) == 0 && enable) getAndBitwiseOrRunState(isShutdown = SHUTDOWN); - if (isShutdown != 0 && checkQuiescence() < 0) + if (isShutdown != 0 && checkQuiescence()) rs = STOP | SHUTDOWN; } } @@ -2777,7 +2769,7 @@ private int tryTerminate(boolean now, boolean enable) { int n = ((qs = queues) == null) ? 0 : qs.length; for (int i = n, j; i > 0; --i, ++r) { if ((q = qs[j = r & SMASK & (n - 1)]) != null && - (q.config & ALIVE) != 0) { + (q.config & REGISTERED) != 0) { if ((j & 1) != 0) ForkJoinTask.interruptIgnoringExceptions(q.owner); for (ForkJoinTask t; (t = q.poll(null)) != null; ) @@ -3255,19 +3247,40 @@ public int setParallelism(int size) { return getAndSetParallelism(size); } - @Override - public List> invokeAll(Collection> tasks) - throws InterruptedException { + /** + * Uninterrupible version of {@code InvokeAll}. Executes the given + * tasks, returning a list of Futures holding their status and + * results when all complete, ignoring interrupts. {@link + * Future#isDone} is {@code true} for each element of the returned + * list. Note that a completed task could have + * terminated either normally or by throwing an exception. The + * results of this method are undefined if the given collection is + * modified while this operation is in progress. + * + * @apiNote This method supports usages that previously relied on an + * incompatible override of + * {@link ExecutorService#invokeAll(java.util.Collection)}. + * + * @param tasks the collection of tasks + * @param the type of the values returned from the tasks + * @return a list of Futures representing the tasks, in the same + * sequential order as produced by the iterator for the + * given task list, each of which has completed + * @throws NullPointerException if tasks or any of its elements are {@code null} + * @throws RejectedExecutionException if any task cannot be + * scheduled for execution + * @since 21 + */ + public List> invokeAllUninterruptibly(Collection> tasks) { ArrayList> futures = new ArrayList<>(tasks.size()); try { for (Callable t : tasks) { - ForkJoinTask f = ForkJoinTask.adaptInterruptible(t); + ForkJoinTask f = ForkJoinTask.adapt(t); futures.add(f); poolSubmit(true, false, f); } for (int i = futures.size() - 1; i >= 0; --i) - ((ForkJoinTask)futures.get(i)) - .quietlyJoinPoolInvokeAllTask(0L); + ((ForkJoinTask)futures.get(i)).quietlyJoin(); return futures; } catch (Throwable t) { for (Future e : futures) @@ -3276,11 +3289,12 @@ public List> invokeAll(Collection> tasks) } } - @Override - public List> invokeAll(Collection> tasks, - long timeout, TimeUnit unit) + /** + * Common support for timed and untimed invokeAll + */ + private List> invokeAll(Collection> tasks, + long deadline) throws InterruptedException { - long deadline = ForkJoinTask.nanosTimeoutToDeadline(unit.toNanos(timeout)); ArrayList> futures = new ArrayList<>(tasks.size()); try { for (Callable t : tasks) { @@ -3299,6 +3313,19 @@ public List> invokeAll(Collection> tasks, } } + @Override + public List> invokeAll(Collection> tasks) + throws InterruptedException { + return invokeAll(tasks, 0L); + } + + @Override + public List> invokeAll(Collection> tasks, + long timeout, TimeUnit unit) + throws InterruptedException { + return invokeAll(tasks, ForkJoinTask.nanosTimeoutToDeadline(unit.toNanos(timeout))); + } + @Override public T invokeAny(Collection> tasks) throws InterruptedException, ExecutionException { @@ -3422,7 +3449,7 @@ public int getActiveThreadCount() { * @return {@code true} if all threads are currently idle */ public boolean isQuiescent() { - return checkQuiescence() <= 0; + return checkQuiescence(); } /** @@ -3887,7 +3914,7 @@ private void compensatedBlock(ManagedBlocker blocker) long c = ctl; if (blocker.isReleasable()) break; - if ((comp = tryCompensate(c, false)) >= 0) { + if ((comp = tryCompensate(c)) >= 0) { long post = (comp == 0) ? 0L : RC_UNIT; try { done = blocker.block(); @@ -3897,6 +3924,8 @@ private void compensatedBlock(ManagedBlocker blocker) if (done) break; } + else if ((runState & STOP) != 0) + throw new InterruptedException(); } } @@ -3907,10 +3936,12 @@ private void compensatedBlock(ManagedBlocker blocker) * with the value returned by this method to re-adjust the parallelism. */ final long beginCompensatedBlock() { - int c; - while ((c = tryCompensate(ctl, false)) < 0) - Thread.onSpinWait(); - return (c == 0) ? 0L : RC_UNIT; + for (int c;;) { + if ((c = tryCompensate(ctl)) > 0) + return RC_UNIT; + if (c == 0 || (runState & STOP) != 0) + return 0L; + } } /** diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index beffb084a09e7..1f03e2091f248 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -219,11 +219,11 @@ public abstract class ForkJoinTask implements Future, Serializable { * recorded in status bits (ABNORMAL but not THROWN), but reported * in joining methods by throwing an exception. Other exceptions * of completed (THROWN) tasks are recorded in the "aux" field, - * but are reconstructed (see getThrowableException) to produce - * more useful stack traces when reported. Sentinels for - * interruptions or timeouts while waiting for completion are not - * recorded as status bits but are included in return values of - * methods in which they occur. + * but are reconstructed (in getException) to produce more useful + * stack traces when reported. Sentinels for interruptions or + * timeouts while waiting for completion are not recorded as + * status bits but are included in return values of methods in + * which they occur. * * The methods of this class are more-or-less layered into * (1) basic status maintenance @@ -324,8 +324,14 @@ private void setDone() { */ final int trySetCancelled() { int s; - do {} while ((s = status) >= 0 && !casStatus(s, s | (DONE | ABNORMAL))); - signalWaiters(); + for (;;) { + if ((s = status) < 0) + break; + if (casStatus(s, s | (DONE | ABNORMAL))) { + signalWaiters(); + break; + } + } return s; } @@ -355,6 +361,12 @@ final boolean trySetThrown(Throwable ex) { return set; } + /** + * Overridable action on setting exception + */ + void onFJTExceptionSet(Throwable ex) { + } + /** * Tries to set exception, if so invoking onFJTExceptionSet */ @@ -457,7 +469,9 @@ else if (deadline != 0L) { /** * Tries applicable helping steps while joining this task, - * otherwise invokes blocking version of awaitDone + * otherwise invokes blocking version of awaitDone. Called only + * when pre-checked not to be done, and pre-screened for + * interrupts and timeouts, if applicable. * * @param interruptible if wait is interruptible * @param deadline if nonzero, timeout deadline @@ -522,13 +536,19 @@ final void doExec() { * still correct, although it may contain a misleading stack * trace. * + * @param asExecutionException true if wrap as ExecutionException * @return the exception, or null if none */ - private Throwable getThrowableException() { - Throwable ex; Aux a; - if ((status & HAVE_EXCEPTION) != HAVE_EXCEPTION || (a = aux) == null) - ex = null; - else if ((ex = a.ex) != null && a.thread != Thread.currentThread()) { + private Throwable getException(boolean asExecutionException) { + int s; Throwable ex; Aux a; + if ((s = status) >= 0 || (s & ABNORMAL) == 0) + return null; + else if ((s & THROWN) == 0 || (a = aux) == null || (ex = a.ex) == null) { + ex = new CancellationException(); + if (!asExecutionException || !(this instanceof InterruptibleTask)) + return ex; // else wrap below + } + else if (a.thread != Thread.currentThread()) { try { Constructor noArgCtor = null, oneArgCtor = null; for (Constructor c : ex.getClass().getConstructors()) { @@ -550,42 +570,16 @@ else if (noArgCtor != null) { } catch (Exception ignore) { } } - return ex; - } - - /** - * Returns exception associated with the given status, or null if none. - */ - private Throwable getException(int s) { - Throwable ex = null; - if ((s & ABNORMAL) != 0 && (ex = getThrowableException()) == null) - ex = new CancellationException(); - return ex; - } - - /** - * Throws exception associated with the given status, or - * CancellationException if none recorded. - */ - private void reportException(int s) { - ForkJoinTask.uncheckedThrow(getThrowableException()); + return (asExecutionException) ? new ExecutionException(ex) : ex; } /** - * Throws exception for (timed or untimed) get, wrapping if - * necessary in an ExecutionException. + * Throws thrown exception, or CancellationException if none + * recorded. */ - private void reportExecutionException(int s) { - Throwable ex, rx; - if (s == ABNORMAL) - ex = new InterruptedException(); - else if (s >= 0) - ex = new TimeoutException(); - else if ((rx = getThrowableException()) != null) - ex = new ExecutionException(rx); - else - ex = wrapFJTCancellationException(new CancellationException()); - ForkJoinTask.uncheckedThrow(ex); + private void reportException(boolean asExecutionException) { + ForkJoinTask. + uncheckedThrow(getException(asExecutionException)); } /** @@ -626,9 +620,11 @@ final int getForkJoinTaskStatusMarkerBit() { } /** - * Cancels, ignoring any exceptions thrown by cancel. Cancel is - * spec'ed not to throw any exceptions, but if it does anyway, we - * have no recourse, so guard against this case. + * Cancels with mayInterruptIfRunning, ignoring any exceptions + * thrown by cancel. Used only when cancelling tasks upon pool or + * worker thread termination. Cancel is spec'ed not to throw any + * exceptions, but if it does anyway, we have no recourse, so + * guard against this case. */ static final void cancelIgnoringExceptions(ForkJoinTask t) { if (t != null) { @@ -649,22 +645,6 @@ static final void interruptIgnoringExceptions(Thread t) { } } - // Methods overridden only in jdk subclasses - - /** - * Overridable action on setting exception - */ - void onFJTExceptionSet(Throwable ex) { - } - - /** - * Optional wrapping for cancellation in reportExecutionException. - * (Overridden in class InterruptibleTask.) - */ - Throwable wrapFJTCancellationException(CancellationException cx) { - return cx; - } - // public methods /** @@ -714,10 +694,8 @@ public final ForkJoinTask fork() { */ public final V join() { int s; - if ((s = status) >= 0) - s = awaitDone(false, 0L); - if ((s & ABNORMAL) != 0) - reportException(s); + if ((((s = status) < 0 ? s : awaitDone(false, 0L)) & ABNORMAL) != 0) + reportException(false); return getRawResult(); } @@ -732,10 +710,8 @@ public final V join() { public final V invoke() { int s; doExec(); - if ((s = status) >= 0) - s = awaitDone(false, 0L); - if ((s & ABNORMAL) != 0) - reportException(s); + if ((((s = status) < 0 ? s : awaitDone(false, 0L)) & ABNORMAL) != 0) + reportException(false); return getRawResult(); } @@ -762,18 +738,14 @@ public static void invokeAll(ForkJoinTask t1, ForkJoinTask t2) { throw new NullPointerException(); t2.fork(); t1.doExec(); - if ((s1 = t1.status) >= 0) - s1 = t1.awaitDone(false, 0L); - if ((s1 & ABNORMAL) != 0) { - cancelIgnoringExceptions(t2); - t1.reportException(s1); - } - else { - if ((s2 = t2.status) >= 0) - s2 = t2.awaitDone(false, 0L); - if ((s2 & ABNORMAL) != 0) - t2.reportException(s2); + if ((((s1 = t1.status) < 0 ? s1 : + t1.awaitDone(false, 0L)) & ABNORMAL) != 0) { + t2.cancel(false); + t1.reportException(false); } + else if ((((s2 = t2.status) < 0 ? s2 : + t2.awaitDone(false, 0L)) & ABNORMAL) != 0) + t2.reportException(false); } /** @@ -802,10 +774,9 @@ public static void invokeAll(ForkJoinTask... tasks) { } if (i == 0) { t.doExec(); - if ((s = t.status) >= 0) - s = t.awaitDone(false, 0L); - if ((s & ABNORMAL) != 0) - ex = t.getException(s); + if ((((s = t.status) < 0 ? s : + t.awaitDone(false, 0L)) & ABNORMAL) != 0) + ex = t.getException(false); break; } t.fork(); @@ -813,17 +784,19 @@ public static void invokeAll(ForkJoinTask... tasks) { if (ex == null) { for (int i = 1; i <= last; ++i) { ForkJoinTask t; int s; - if ((t = tasks[i]) != null) { - if ((s = t.status) >= 0) - s = t.awaitDone(false, 0L); - if ((s & ABNORMAL) != 0 && (ex = t.getException(s)) != null) - break; - } + if ((t = tasks[i]) != null && + ((((s = t.status) < 0 ? s : + t.awaitDone(false, 0L)) & ABNORMAL) != 0) && + (ex = t.getException(false)) != null) + break; } } if (ex != null) { - for (int i = 1; i <= last; ++i) - cancelIgnoringExceptions(tasks[i]); + for (int i = 1; i <= last; ++i) { + ForkJoinTask t; + if ((t = tasks[i]) != null) + t.cancel(false); + } rethrow(ex); } } @@ -864,10 +837,9 @@ public static > Collection invokeAll(Collection } if (i == 0) { t.doExec(); - if ((s = t.status) >= 0) - s = t.awaitDone(false, 0L); - if ((s & ABNORMAL) != 0) - ex = t.getException(s); + if ((((s = t.status) < 0 ? s : + t.awaitDone(false, 0L)) & ABNORMAL) != 0) + ex = t.getException(false); break; } t.fork(); @@ -875,17 +847,19 @@ public static > Collection invokeAll(Collection if (ex == null) { for (int i = 1; i <= last; ++i) { ForkJoinTask t; int s; - if ((t = ts.get(i)) != null) { - if ((s = t.status) >= 0) - s = t.awaitDone(false, 0L); - if ((s & ABNORMAL) != 0 && (ex = t.getException(s)) != null) - break; - } + if ((t = ts.get(i)) != null && + ((((s = t.status) < 0 ? s : + t.awaitDone(false, 0L)) & ABNORMAL) != 0) && + (ex = t.getException(false)) != null) + break; } } if (ex != null) { - for (int i = 1; i <= last; ++i) - cancelIgnoringExceptions(ts.get(i)); + for (int i = 1; i <= last; ++i) { + ForkJoinTask t; + if ((t = ts.get(i)) != null) + t.cancel(false); + } rethrow(ex); } return tasks; @@ -970,7 +944,8 @@ public V resultNow() { @Override public Throwable exceptionNow() { Throwable ex; - if ((ex = getThrowableException()) == null) + if ((status & HAVE_EXCEPTION) != HAVE_EXCEPTION || + (ex = getException(false)) == null) throw new IllegalStateException(); return ex; } @@ -983,7 +958,7 @@ public Throwable exceptionNow() { * @return the exception, or {@code null} if none */ public final Throwable getException() { - return getException(status); + return getException(false); } /** @@ -1054,11 +1029,16 @@ public final void quietlyComplete() { */ public final V get() throws InterruptedException, ExecutionException { int s; - if ((s = status) >= 0) - s = ((Thread.interrupted()) ? ABNORMAL : - awaitDone(true, 0L)); - if ((s & ABNORMAL) != 0) - reportExecutionException(s); + if ((s = status) >= 0) { + if (Thread.interrupted()) + s = ABNORMAL; + else + s = awaitDone(true, 0L); + } + if (s == ABNORMAL) + throw new InterruptedException(); + else if ((s & ABNORMAL) != 0) + reportException(true); return getRawResult(); } @@ -1086,8 +1066,12 @@ public final V get(long timeout, TimeUnit unit) else if (nanos > 0L) s = awaitDone(true, nanosTimeoutToDeadline(nanos)); } - if (s >= 0 || (s & ABNORMAL) != 0) - reportExecutionException(s); + if (s == ABNORMAL) + throw new InterruptedException(); + else if (s >= 0) + throw new TimeoutException(); + else if ((s & ABNORMAL) != 0) + reportException(true); return getRawResult(); } @@ -1703,19 +1687,20 @@ public final boolean exec() { return true; } public boolean cancel(boolean mayInterruptIfRunning) { - if (trySetCancelled() >= 0 && mayInterruptIfRunning) - interruptIgnoringExceptions(runner); + if (trySetCancelled() >= 0) { + if (mayInterruptIfRunning) + interruptIgnoringExceptions(runner); + return true; + } return isCancelled(); } public final void run() { quietlyInvoke(); } - final Throwable wrapFJTCancellationException(CancellationException cx) { - return new ExecutionException(cx); - } Object adaptee() { return null; } // for printing and diagnostics public String toString() { Object a = adaptee(); String s = super.toString(); - return (a != null) ? (s + "[Wrapped task = " + a.toString() + "]") : s; + return ((a == null) ? s : + (s + "[Wrapped task = " + a.toString() + "]")); } private static final long serialVersionUID = 2838392045355241008L; } From 4b98ae59292ee8ac38ecd0af41c007e864c16957 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Mon, 17 Apr 2023 07:11:08 -0400 Subject: [PATCH 17/61] minor improvements and renamings --- .../util/concurrent/CountedCompleter.java | 2 +- .../java/util/concurrent/ForkJoinPool.java | 569 +++++++++--------- .../java/util/concurrent/ForkJoinTask.java | 59 +- 3 files changed, 331 insertions(+), 299 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java b/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java index b664ce3509435..7884a131ffb70 100644 --- a/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java +++ b/src/java.base/share/classes/java/util/concurrent/CountedCompleter.java @@ -744,7 +744,7 @@ public final void helpComplete(int maxTasks) { * Supports ForkJoinTask exception propagation. */ @Override - final void onFJTExceptionSet(Throwable ex) { + final void onAuxExceptionSet(Throwable ex) { CountedCompleter a = this, p = a; do {} while (a.onExceptionalCompletion(ex, p) && (a = (p = a).completer) != null && diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 4c88e9e622f08..d966c0e6f3167 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -47,6 +47,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.function.Predicate; import java.util.concurrent.CountDownLatch; import java.util.concurrent.locks.LockSupport; @@ -427,14 +428,14 @@ public class ForkJoinPool extends AbstractExecutorService { * for queue array updates (incrementing by LOCKED units per * operation, rather than the usual even/odd scheme). The seqLock * properties detect changes and conditionally upgrade to - * coordinate with some runState updates. It is typically held for - * less than a dozen instructions unless the queue array is being + * coordinate with runState updates. It is typically held for less + * than a dozen instructions unless the queue array is being * resized, during which contention is rare. To be conservative, * acquireRunStateLock is implemented as a spin/sleep loop. Here - * and elsewhere spin constants (SPIN_WAITS) are short enough to - * apply even on systems with few available processors. In - * addition to checking pool status, reads of runState serve as - * acquire fences before reading other non-volatile fields. + * and elsewhere spin constants are short enough to apply even on + * systems with few available processors. In addition to checking + * pool status, reads of runState serve as acquire fences before + * reading other non-volatile fields. * * Array "queues" holds references to WorkQueues. It is updated * (only during worker creation and termination) under the @@ -708,7 +709,7 @@ public class ForkJoinPool extends AbstractExecutorService { * task) avoid context switching or adding worker threads when one * task would otherwise be blocked waiting for completion of * another, basically, just by running that task or one of its - * subtasks if not already stolen. As described below, these + * subtasks if not already taken. As described below, these * mechanics are disabled for InterruptibleTasks, that guarantee * that callers do not executed submitted tasks. * @@ -819,10 +820,9 @@ public class ForkJoinPool extends AbstractExecutorService { * some System property parsing and with security processing that * takes far longer than the actual construction when * SecurityManagers are used or properties are set. The common - * pool is distinguished internally by having both a null - * workerNamePrefix and ISCOMMON config bit set, along with - * PRESET_SIZE set if parallelism was configured by system - * property. + * pool is distinguished internally by having a null + * workerNamePrefix, along with PRESET_SIZE set if parallelism was + * configured by system property. * * When external threads use the common pool, they can perform * subtask processing (see helpComplete and related methods) upon @@ -924,8 +924,9 @@ public class ForkJoinPool extends AbstractExecutorService { * Currently, arrays are initialized to be fairly small but early * resizes rapidly increase size by more than a factor of two * until very large. (Maintenance note: any changes in fields, - * queues, or their uses must be accompanied by re-evaluation of - * these placement and sizing decisions.) + * queues, or their uses, or JVM layout policies, must be + * accompanied by re-evaluation of these placement and sizing + * decisions.) * * Style notes * =========== @@ -1004,6 +1005,11 @@ public class ForkJoinPool extends AbstractExecutorService { */ static final long DEFAULT_KEEPALIVE = 60_000L; + /** + * Undershoot tolerance for idle timeouts + */ + static final long TIMEOUT_SLOP = 20L; + /** * The default value for common pool maxSpares. Overridable using * the "java.util.concurrent.ForkJoinPool.common.maximumSpares" @@ -1021,18 +1027,64 @@ public class ForkJoinPool extends AbstractExecutorService { static final int INITIAL_QUEUE_CAPACITY = 1 << 6; // Bounds - static final int SWIDTH = 16; // width of short - static final int SMASK = 0xffff; // short bits == max index - static final int MAX_CAP = 0x7fff; // max #workers - 1 - - // Bits and sentinels used by both pool and workQueues - static final int FIFO = 1 << 16; // fifo queue or access mode - static final int REGISTERED = 1 << 17; // true when alive - static final int CLEAR_TLS = 1 << 18; // set for Innocuous workers - static final int TRIMMED = 1 << 19; // timed out while idle - static final int ISCOMMON = 1 << 20; // set for common pool - static final int PRESET_SIZE = 1 << 21; // size was set by property - static final int UNCOMPENSATE = 1 << 16; // tryCompensate return + static final int SWIDTH = 16; // width of short + static final int SMASK = 0xffff; // short bits + static final int SEMASK = 0xfffe; // short even bits + static final int MAX_CAP = 0x7fff; // max worker index + + // pool.runState bits + static final int STOP = 1 << 0; + static final int SHUTDOWN = 1 << 1; + static final int TERMINATED = 1 << 2; + static final int LOCKED = 1 << 3; // lowest seqlock bit + + // spin/sleep limits for runState locking and elsewhere; powers of 2 + static final int SPIN_WAITS = 1 << 7; // calls to onSpinWait + static final int MIN_SLEEP = 1 << 10; // approx 1 usec as nanos + static final int MAX_SLEEP = 1 << 20; // approx 1 sec as nanos + + // {pool, workQueue} bits and sentinels + static final int FIFO = 1 << 16; // fifo queue or access mode + static final int REGISTERED = 1 << 17; // true when alive + static final int CLEAR_TLS = 1 << 18; // set for Innocuous workers + static final int TRIMMED = 1 << 19; // timed out while idle + static final int PRESET_SIZE = 1 << 21; // size was set by property + static final int UNCOMPENSATE = 1 << 16; // tryCompensate return + + /* + * Bits and masks for ctl and bounds are packed with 4 16 bit subfields: + * RC: Number of released (unqueued) workers + * TC: Number of total workers + * SS: version count and status of top waiting thread + * ID: poolIndex of top of Treiber stack of waiters + * + * When convenient, we can extract the lower 32 stack top bits + * (including version bits) as sp=(int)ctl. When sp is non-zero, + * there are waiting workers. Count fields may be transiently + * negative during termination because of out-of-order updates. + * To deal with this, we use casts in and out of "short" and/or + * signed shifts to maintain signedness. Because it occupies + * uppermost bits, we can add one release count using getAndAdd of + * RC_UNIT, rather than CAS, when returning from a blocked join. + * Other updates of multiple subfields require CAS. + */ + + // Lower and upper word masks + static final long SP_MASK = 0xffffffffL; + static final long UC_MASK = ~SP_MASK; + // Release counts + static final int RC_SHIFT = 48; + static final long RC_UNIT = 0x0001L << RC_SHIFT; + static final long RC_MASK = 0xffffL << RC_SHIFT; + // Total counts + static final int TC_SHIFT = 32; + static final long TC_UNIT = 0x0001L << TC_SHIFT; + static final long TC_MASK = 0xffffL << TC_SHIFT; + + // sp bits (also used for source field) + static final int SS_SEQ = 1 << 16; // version count + static final int INACTIVE = 1 << 31; // phase/source bit if idle + static final int ACTIVE = ~INACTIVE; // initial value for srcs // Static utilities @@ -1157,8 +1209,8 @@ static final class WorkQueue { int stackPred; // pool stack (ctl) predecessor link int config; // index, mode, REGISTERED bits int base; // index of next slot for poll - ForkJoinTask[] array; // the queued tasks; power of 2 size final ForkJoinWorkerThread owner; // owning thread or null if shared + ForkJoinTask[] array; // the queued tasks; power of 2 size // fields otherwise causing more unnecessary false-sharing cache misses @jdk.internal.vm.annotation.Contended("w") @@ -1182,7 +1234,7 @@ static ForkJoinTask getAndClearSlot(ForkJoinTask[] a, int i) { return (ForkJoinTask) U.getAndSetReference(a, ((long)i << ASHIFT) + ABASE, null); } - static ForkJoinTask cmpExSlotToNull(ForkJoinTask[] a, int i, + static ForkJoinTask cmpExSlotToNull(ForkJoinTask[] a, int i, ForkJoinTask c) { return (ForkJoinTask) U.compareAndExchangeReference(a, ((long)i << ASHIFT) + ABASE, @@ -1191,7 +1243,7 @@ static ForkJoinTask cmpExSlotToNull(ForkJoinTask[] a, int i, final int getAndSetAccess(int v) { return U.getAndSetInt(this, ACCESS, v); } - final void releaseAccess() { + final void releaseAccess() { // unlock U.putIntRelease(this, ACCESS, 0); } @@ -1224,24 +1276,26 @@ final int queueSize() { * Pushes a task. Called only by owner or if already locked * * @param task the task. Caller must ensure non-null. - * @param p the pool. Must be non-null unless terminating. + * @param pool the pool. Must be non-null unless terminating. * @param signal true if signal if queue may have been or appeared empty * @throws RejectedExecutionException if array cannot be resized */ - final void push(ForkJoinTask task, ForkJoinPool p, boolean signal) { + final void push(ForkJoinTask task, ForkJoinPool pool, + boolean signal) { ForkJoinTask[] a = array; int b = base, s = top++, cap, m; if (a != null && (cap = a.length) > 0) { if ((m = (cap - 1)) == s - b) growAndPush(task, a, s); // may have appeared empty else { + int prev = m & (s - 1); a[m & s] = task; getAndSetAccess(0); // for memory effects if internal - if (a[m & (s - 1)] != null) - signal = false; // not empty + if (a[prev] != null) + signal = false; // not empty } - if (signal && p != null) - p.signalWork(false); + if (signal && pool != null) + pool.signalWork(false); } } @@ -1335,11 +1389,15 @@ else if (top != p || cmpExSlotToNull(a, k, task) != task) { releaseAccess(); return false; } + top = s; + releaseAccess(); + } + else { + if (getAndClearSlot(a, k) == null) + return false; + top = s; + U.storeFence(); } - else if (getAndClearSlot(a, k) == null) - return false; - top = s; - releaseAccess(); return true; } @@ -1433,12 +1491,12 @@ else if (u == null) * remaining local tasks and/or others available from the * given queue, if any. */ - final void topLevelExec(ForkJoinTask task, WorkQueue q) { + final void topLevelExec(ForkJoinTask task, WorkQueue src) { int cfg = config, fifo = cfg & FIFO, nstolen = 1; while (task != null) { task.doExec(); if ((task = nextLocalTask(fifo)) == null && - q != null && (task = q.tryPoll()) != null) + src != null && (task = src.tryPoll()) != null) ++nstolen; } nsteals += nstolen; @@ -1517,11 +1575,15 @@ final int helpComplete(ForkJoinTask task, boolean internal, int limit) { releaseAccess(); break; // missed } + top = s; + releaseAccess(); + } + else { + if (getAndClearSlot(a, k) == null) + break; + top = s; + U.storeFence(); } - else if (getAndClearSlot(a, k) == null) - break; - top = s; - releaseAccess(); t.doExec(); if (limit != 0 && --limit == 0) break; @@ -1630,58 +1692,6 @@ final void setClearThreadLocals() { */ static volatile RuntimePermission modifyThreadPermission; - // Bits and masks for pool control fields - - /* - * Bits and masks for ctl and bounds are packed with 4 16 bit subfields: - * RC: Number of released (unqueued) workers - * TC: Number of total workers - * SS: version count and status of top waiting thread - * ID: poolIndex of top of Treiber stack of waiters - * - * When convenient, we can extract the lower 32 stack top bits - * (including version bits) as sp=(int)ctl. When sp is non-zero, - * there are waiting workers. Count fields may be transiently - * negative during termination because of out-of-order updates. - * To deal with this, we use casts in and out of "short" and/or - * signed shifts to maintain signedness. Because it occupies - * uppermost bits, we can add one release count using getAndAdd of - * RC_UNIT, rather than CAS, when returning from a blocked join. - * Other updates of multiple subfields require CAS. - */ - - // Lower and upper word masks - private static final long SP_MASK = 0xffffffffL; - private static final long UC_MASK = ~SP_MASK; - // Release counts - private static final int RC_SHIFT = 48; - private static final long RC_UNIT = 0x0001L << RC_SHIFT; - private static final long RC_MASK = 0xffffL << RC_SHIFT; - // Total counts - private static final int TC_SHIFT = 32; - private static final long TC_UNIT = 0x0001L << TC_SHIFT; - private static final long TC_MASK = 0xffffL << TC_SHIFT; - - // sp bits (also used for source field) - private static final int SS_SEQ = 1 << 16; // version count - private static final int INACTIVE = 1 << 31; // phase/source bit if idle - private static final int ACTIVE = ~INACTIVE; // initial value for srcs - - // pool.runState bits - private static final int STOP = 1 << 0; - private static final int SHUTDOWN = 1 << 1; - private static final int TERMINATED = 1 << 2; - private static final int LOCKED = 1 << 3; // lowest seqlock bit - - // spin/sleep limits for runState locking and elsewhere; powers of 2 - private static final int SPIN_WAITS = 1 << 6; // calls to onSpinWait - private static final int MIN_SLEEP = 1 << 10; // approx 1 usec as nanos - private static final int MAX_SLEEP = 1 << 20; // approx 1 sec as nanos - - /** - * Undershoot tolerance for idle timeouts, in millisecs - */ - private static final long TIMEOUT_SLOP = 20L; // Instance fields volatile long stealCount; // collects worker nsteals @@ -1690,12 +1700,13 @@ final void setClearThreadLocals() { final long keepAlive; // milliseconds before dropping if idle final int config; // static configuration bits volatile int runState; // SHUTDOWN, STOP, TERMINATED, seq bits - volatile CountDownLatch termination; // lazily constructed + final Predicate saturate; + volatile CountDownLatch termination; // lazily constructed final ForkJoinWorkerThreadFactory factory; final UncaughtExceptionHandler ueh; // per-worker UEH - final String workerNamePrefix; // null for common pool final SharedThreadContainer container; + final String workerNamePrefix; // null for common pool WorkQueue[] queues; // main registry @jdk.internal.vm.annotation.Contended("fjpctl") // segregate @@ -1749,16 +1760,18 @@ private boolean casRunState(int c, int v) { private void releaseRunStateLock() { // increment lock bit U.getAndAddInt(this, RUNSTATE, LOCKED); } - private void acquireRunStateLock() { - int s; // locked when LOCKED set - if (((s = runState) & LOCKED) != 0 || !casRunState(s, s + LOCKED)) - spinLockRunState(); + private int acquireRunStateLock() { // lock and return current state + int s, u; // locked when LOCKED set + if (((s = runState) & LOCKED) == 0 && casRunState(s, u = s + LOCKED)) + return u; + else + return spinLockRunState(); } - private void spinLockRunState() { // spin/sleep - for (int waits = 0, s;;) { + private int spinLockRunState() { // spin/sleep + for (int waits = 0, s, u;;) { if (((s = runState) & LOCKED) == 0) { - if (casRunState(s, s + LOCKED)) - break; + if (casRunState(s, u = s + LOCKED)) + return u; waits = 0; } else if (waits < SPIN_WAITS) { @@ -1827,56 +1840,61 @@ final String nextWorkerThreadName() { * @param w caller's WorkQueue */ final void registerWorker(WorkQueue w) { - WorkQueue[] qs; int n; - ThreadLocalRandom.localInit(); - int seed = ThreadLocalRandom.getProbe(); - int cfg = (config & FIFO) | REGISTERED; - if (w != null && (runState & STOP) == 0 && (qs = queues) != null && - (n = qs.length) > 0) { + if (w != null) { + WorkQueue[] qs; int n; + ThreadLocalRandom.localInit(); + int seed = ThreadLocalRandom.getProbe(); + int id = ((seed << 1) | 1) & SMASK; // initial index guess + w.stackPred = seed; // used by runWorker w.array = new ForkJoinTask[INITIAL_QUEUE_CAPACITY]; - cfg |= w.config; - w.stackPred = seed; - int id = (seed << 1) | 1; // initial index guess - { // try to find slot without lock (repeated below if fail) - int k = n, m = n - 1; - for (; qs[id &= m] != null && k > 0; id -= 2, k -= 2); - if (k == 0) - id = n | 1; - } - acquireRunStateLock(); - try { - if ((runState & STOP) == 0) { - boolean resize = false; - if (qs == (qs = queues) && id >= 0 && id < n && - qs[id] == null) - qs[id] = w; - else if (qs != null && (n = qs.length) > 0) { - int k = n, m = n - 1; - for (; qs[id &= m] != null && k > 0; id -= 2, k -= 2); - if (k == 0) { - id = n | 1; - resize = true; - } + int cfg = w.config | REGISTERED | (config & FIFO); + if ((qs = queues) != null && (n = qs.length) > 0) { + // try to find open slot without lock; rescan below if needed + for (int k = n, m = n - 1; ; id += 2) { + if (qs[id &= m] == null) + break; + if ((k -= 2) <= 0) { + id |= n; + break; } - w.config = id | cfg; - w.phase = id; // now publishable - if (resize) { // expand array - int an = n << 1, am = an - 1; - WorkQueue[] as = new WorkQueue[an]; - as[id & am] = w; - for (int j = 1; j < n; j += 2) - as[j] = qs[j]; - for (int j = 0; j < n; j += 2) { - WorkQueue q; - if ((q = qs[j]) != null) // shared queues may move - as[q.config & am] = q; + } + int rs = acquireRunStateLock(); + try { + if ((qs = queues) != null && (n = qs.length) > 0) { // reread + if (id >= n || qs[id] != null) { // rescan + for (int k = n, m = n - 1; ; id += 2) { + if (qs[id &= m] == null) + break; + if ((k -= 2) <= 0) { + id |= n; + break; + } + } + } + w.config = id | cfg; + w.phase = id; // now publishable + if ((rs & STOP) != 0) + ; // skip if stopping + else if (id < n) + qs[id] = w; + else { // expand array + int an = n << 1, am = an - 1; + WorkQueue[] as = new WorkQueue[an]; + as[id & am] = w; + for (int j = 1; j < n; j += 2) + as[j] = qs[j]; + for (int j = 0; j < n; j += 2) { + WorkQueue q; // shared queues may move + if ((q = qs[j]) != null) + as[q.config & am] = q; + } + U.storeFence(); // fill before publish + queues = as; } - U.storeFence(); // fill before publish - queues = as; } + } finally { + releaseRunStateLock(); } - } finally { - releaseRunStateLock(); } } } @@ -1915,15 +1933,14 @@ else if ((int)c == 0) // was dropped on timeout if ((tryTerminate(false, false) & STOP) == 0 && w != null) { WorkQueue[] qs; int n, i; // remove index unless terminating long ns = w.nsteals & 0xffffffffL; - acquireRunStateLock(); - if ((runState & STOP) == 0 && (qs = queues) != null && - (n = qs.length) > 0 && + int rs = acquireRunStateLock() & STOP; + if (rs == 0 && (qs = queues) != null && (n = qs.length) > 0 && qs[i = cfg & (n - 1)] == w) { qs[i] = null; stealCount += ns; // accumulate steals } releaseRunStateLock(); - if ((cfg & REGISTERED) != 0) + if (rs == 0 && (cfg & REGISTERED) != 0) signalWork(false); // may replace unless trimmed or uninitialized } if (ex != null) @@ -2017,17 +2034,20 @@ private boolean tryTrim(WorkQueue w) { * shutdown is enabled, sets runState mode to STOP. */ private boolean checkQuiescence() { + long prevCtl = 0L; boolean scanned = false; - outer: for (int prevState = -1, rs = runState; ; prevState = rs) { - WorkQueue[] qs; int n; + outer: for (int prevState = 0, rs = runState; ; prevState = rs) { + long c; WorkQueue[] qs; int n; if ((rs & STOP) != 0) // terminating return true; - if ((ctl & RC_MASK) != 0L) + if (((c = ctl) & RC_MASK) != 0L) break; // active - if (scanned) { // try to trigger termination + if (prevCtl != (prevCtl = c)) + scanned = false; // inconsistent + if (scanned) { if ((rs & SHUTDOWN) == 0 || casRunState(rs, rs | STOP)) - return true; - scanned = false; // inconsistent; retry + return true; // try to trigger termination + scanned = false; // retry } if ((qs = queues) != null && (n = qs.length) > 0) { for (int i = 0;;) { // alternates even for external, odd internal @@ -2066,21 +2086,22 @@ final void runWorker(WorkQueue w) { if ((rs & STOP) != 0) // stop or rescan if stale break; srcs &= ACTIVE; - } else { + } + else { long c; // try to inactivate/enqueue int p = phase + SS_SEQ, np = p | INACTIVE; - long nc = ((((c = ctl) - RC_UNIT) & UC_MASK) | - (p & ACTIVE & SP_MASK)); + long nc = ((p & ACTIVE & SP_MASK) | + ((c = ctl) - RC_UNIT) & UC_MASK); w.stackPred = (int)c; // set ctl stack link w.phase = np; - if (compareAndSetCtl(c, nc)) { - if ((phase = awaitWork(w, nc)) < 0) - break; - srcs = ACTIVE; // restart - } else { + if (!compareAndSetCtl(c, nc)) { w.phase = phase; // back out on ctl contention srcs &= ACTIVE; } + else if ((phase = awaitWork(w, nc)) > 0) + srcs = ACTIVE; // restart + else + break; // terminated } } } @@ -2131,7 +2152,7 @@ else if (u != null || (qsrcs != srcs && a[nk] != null)) } /** - * Advances phase, enqueues, and awaits signal or termination. + * Awaits signal or termination. * * @param queuedCtl ctl value at point of inactivation * @return current phase, or negative for exit @@ -2141,9 +2162,8 @@ private int awaitWork(WorkQueue w, long queuedCtl) { return -1; // currently impossible boolean idle; // true if pool idle long deadline = 0L; // for timed wait if idle - int nspins = // approximate avg #accesses needed to scan+signal - (int)((queuedCtl & TC_MASK) >>> (TC_SHIFT - 1)) + SPIN_WAITS + 1; - if ((queuedCtl & RC_MASK) > 0L) + int nspins = ((short)(queuedCtl >>> TC_SHIFT) + 0xb) * 3; + if ((short)(queuedCtl >>> RC_SHIFT) > 0) idle = false; else if (idle = checkQuiescence()) deadline = keepAlive + System.currentTimeMillis(); @@ -2151,7 +2171,7 @@ else if (idle = checkQuiescence()) reactivate(null); // ensure live if may be tasks int phase; // nonnegative on normal return outer: for (;;) { // await signal or termination - int spins = nspins; + int spins = nspins; // approx #accesses to scan+signal if ((runState & STOP) != 0) return -1; do { // spin to cushion near-misses @@ -2217,14 +2237,16 @@ private ForkJoinTask pollScan(boolean submissionsOnly) { */ private int tryCompensate(long c) { Predicate sat; - long b = bounds; // unpack fields + long b = bounds; // unpack fields int pc = parallelism; int minActive = (short)(b & SMASK), maxTotal = (short)(b >>> SWIDTH) + pc, active = (short)(c >>> RC_SHIFT), total = (short)(c >>> TC_SHIFT), sp = (int)c & ACTIVE; - if (sp != 0 && active <= pc) { // activate idle worker + if ((runState & STOP) != 0) // terminating + return 0; + else if (sp != 0 && active <= pc) { // activate idle worker WorkQueue[] qs; WorkQueue v; int i; if (ctl == c && (qs = queues) != null && qs.length > (i = sp & SMASK) && (v = qs[i]) != null) { @@ -2236,26 +2258,24 @@ private int tryCompensate(long c) { return UNCOMPENSATE; } } - return -1; // inconsistent; retry + return -1; // inconsistent; retry } - if (active > minActive && total >= pc) { // reduce active workers + else if (active > minActive && total >= pc) { // reduce active workers long nc = ((RC_MASK & (c - RC_UNIT)) | (~RC_MASK & c)); return compareAndSetCtl(c, nc) ? UNCOMPENSATE : -1; } - if ((runState & STOP) != 0) // rely on caller to notice - return -1; - if (total < maxTotal && total < MAX_CAP) { // expand pool + else if (total < maxTotal && total < MAX_CAP) { long nc = ((c + TC_UNIT) & TC_MASK) | (c & ~TC_MASK); - return (!compareAndSetCtl(c, nc) ? -1 : - createWorker() ? UNCOMPENSATE : - (runState & STOP) != 0 ? -1 : 0); // recheck + return (!compareAndSetCtl(c, nc) ? -1 : // expand pool + createWorker() ? UNCOMPENSATE : 0); } - if (!compareAndSetCtl(c, c)) // validate + else if (!compareAndSetCtl(c, c)) // validate return -1; - if ((sat = saturate) != null && sat.test(this)) + else if ((sat = saturate) != null && sat.test(this)) return 0; - throw new RejectedExecutionException( - "Thread limit exceeded replacing blocked worker"); + else + throw new RejectedExecutionException( + "Thread limit exceeded replacing blocked worker"); } /** @@ -2290,8 +2310,6 @@ final int helpJoin(ForkJoinTask task, WorkQueue w, boolean internal) { WorkQueue[] qs; if ((s = task.status) < 0) break; - if ((runState & STOP) != 0) - break; if (!rescan && sctl == (sctl = ctl) && (s = tryCompensate(sctl)) >= 0) break; @@ -2312,7 +2330,8 @@ final int helpJoin(ForkJoinTask task, WorkQueue w, boolean internal) { eligible = true; break; } - if (v == 0 || (p = qs[(v - 1) & (n - 1)]) == null || + if (v == 0 || + (p = qs[(v - 1) & (n - 1)]) == null || --steps == 0) { // bound steps eligible = false; break; @@ -2365,8 +2384,6 @@ final int helpComplete(ForkJoinTask task, WorkQueue w, boolean internal) { break; if ((s = task.status) < 0) break; - if ((runState & STOP) != 0) - break; if (!rescan && sctl == (sctl = ctl) && (!internal || (s = tryCompensate(sctl)) >= 0)) break; @@ -2583,31 +2600,34 @@ final ForkJoinTask nextTaskFor(WorkQueue w) { /** * Finds and locks a WorkQueue for an external submitter, or * throws RejectedExecutionException if shutdown or terminating. + * @param r current ThreadLocalRandom.getProbe() value * @param isSubmit false if this is for a common pool fork */ - final WorkQueue submissionQueue(boolean isSubmit) { - int r; - if ((r = ThreadLocalRandom.getProbe()) == 0) { + private WorkQueue submissionQueue(int r) { + if (r == 0) { ThreadLocalRandom.localInit(); // initialize caller's probe r = ThreadLocalRandom.getProbe(); } - for (int id = r << 1;;) { // even indices only - int n, i; WorkQueue[] qs; WorkQueue q; + for (;;) { + int n, i, id; WorkQueue[] qs; WorkQueue q; if ((qs = queues) == null || (n = qs.length) <= 0) break; - else if ((q = qs[i = (n - 1) & id]) == null) { - WorkQueue w = new WorkQueue(null, id | REGISTERED); + if ((q = qs[i = (id = r & SEMASK) & (n - 1)]) == null) { + WorkQueue v, w = new WorkQueue(null, id | REGISTERED); w.array = new ForkJoinTask[INITIAL_QUEUE_CAPACITY]; - acquireRunStateLock(); // install under lock - if (queues == qs && qs[i] == null) // OK even if stopping - qs[i] = w; // else lost race; discard + acquireRunStateLock(); + if ((v = qs[i]) != null) + q = v; // lost race; discard w + else if (queues == qs) + q = qs[i] = w; // else resized releaseRunStateLock(); } - else if (q.getAndSetAccess(1) != 0) // move and restart - id = (r = ThreadLocalRandom.advanceProbe(r)) << 1; - else if (isSubmit && (runState & SHUTDOWN) != 0) { + if (q == null) + ; // retry if resized + else if (q.getAndSetAccess(1) != 0) // move index + r = ThreadLocalRandom.advanceProbe(r); + else if ((runState & SHUTDOWN) != 0) { q.access = 0; // check while q lock held - tryTerminate(false, false); // may trigger termination break; } else @@ -2620,19 +2640,27 @@ else if (isSubmit && (runState & SHUTDOWN) != 0) { * Pushes a submission to the pool, using internal queue if called * from ForkJoinWorkerThread, else external queue. */ - private ForkJoinTask poolSubmit(boolean signalIfEmpty, - boolean external, - ForkJoinTask task) { - Thread t; - U.storeStoreFence(); // ensure safe publication - if (task == null) throw new NullPointerException(); - ForkJoinWorkerThread wt = - ((t = Thread.currentThread()) instanceof ForkJoinWorkerThread) ? - (ForkJoinWorkerThread)t : null; - WorkQueue q = ((!external && wt != null && wt.pool == this) ? - wt.workQueue : submissionQueue(true)); + private void poolSubmit(boolean signalIfEmpty, ForkJoinTask task) { + Thread t; ForkJoinWorkerThread wt; WorkQueue q; + if (((t = Thread.currentThread()) instanceof ForkJoinWorkerThread) && + (wt = (ForkJoinWorkerThread)t).pool == this) { + q = wt.workQueue; + U.storeStoreFence(); // ensure safely publishable + } + else + q = submissionQueue(ThreadLocalRandom.getProbe()); q.push(task, this, signalIfEmpty); - return task; + } + + /** + * Pushes a forked task to an external queue. + */ + final WorkQueue externalSubmissionQueue() { + WorkQueue[] qs; WorkQueue q; int n; + int r = ThreadLocalRandom.getProbe(); + return (((qs = queues) != null && (n = qs.length) > 0 && + (q = qs[r & SEMASK & (n - 1)]) != null&& r != 0 && + q.getAndSetAccess(1) == 0) ? q : submissionQueue(r)); } /** @@ -2641,11 +2669,11 @@ private ForkJoinTask poolSubmit(boolean signalIfEmpty, * null if none. */ static WorkQueue externalQueue(ForkJoinPool p) { - WorkQueue[] qs; - int r = ThreadLocalRandom.getProbe(), n; + WorkQueue[] qs; int n; + int r = ThreadLocalRandom.getProbe(); return (p != null && (qs = p.queues) != null && (n = qs.length) > 0 && r != 0) ? - qs[(n - 1) & (r << 1)] : null; + qs[r & SEMASK & (n - 1)] : null; } /** @@ -2655,13 +2683,6 @@ static WorkQueue commonQueue() { return externalQueue(common); } - /** - * Returns queue for an external thread, if one exists - */ - final WorkQueue externalQueue() { - return externalQueue(this); - } - /** * If the given executor is a ForkJoinPool, poll and execute * AsynchronousCompletionTasks from worker's queue until none are @@ -2674,7 +2695,7 @@ static void helpAsyncBlocker(Executor e, ManagedBlocker blocker) { w = wt.workQueue; } else if (e instanceof ForkJoinPool) - w = ((ForkJoinPool)e).externalQueue(); + w = externalQueue((ForkJoinPool)e); if (w != null) w.helpAsyncBlocker(blocker); } @@ -2749,8 +2770,8 @@ static int getSurplusQueuedTaskCount() { * @return runState (possibly only its status bits) on exit */ private int tryTerminate(boolean now, boolean enable) { - int rs = runState, cfg = config, isShutdown; - if ((rs & STOP) == 0 && (cfg & ISCOMMON) == 0) { + int rs, isShutdown; + if (((rs = runState) & STOP) == 0) { if (now) getAndBitwiseOrRunState(rs = STOP | SHUTDOWN); else { @@ -3032,7 +3053,7 @@ private ForkJoinPool(byte forCommonPoolOnly) { int p = Math.min(pc, MAX_CAP); int size = (p == 0) ? 1 : 1 << (33 - Integer.numberOfLeadingZeros(p-1)); this.parallelism = p; - this.config = ISCOMMON | preset; + this.config = preset; this.bounds = (long)(1 | (maxSpares << SWIDTH)); this.factory = fac; this.ueh = handler; @@ -3081,7 +3102,8 @@ public static ForkJoinPool commonPool() { * scheduled for execution */ public T invoke(ForkJoinTask task) { - poolSubmit(true, false, task); + Objects.requireNonNull(task); + poolSubmit(true, task); return task.join(); } @@ -3094,7 +3116,8 @@ public T invoke(ForkJoinTask task) { * scheduled for execution */ public void execute(ForkJoinTask task) { - poolSubmit(true, false, task); + Objects.requireNonNull(task); + poolSubmit(true, task); } // AbstractExecutorService methods @@ -3107,7 +3130,7 @@ public void execute(ForkJoinTask task) { @Override @SuppressWarnings("unchecked") public void execute(Runnable task) { - poolSubmit(true, false, (task instanceof ForkJoinTask) + poolSubmit(true, (task instanceof ForkJoinTask) ? (ForkJoinTask) task // avoid re-wrap : new ForkJoinTask.RunnableExecuteAction(task)); } @@ -3127,7 +3150,9 @@ public void execute(Runnable task) { * scheduled for execution */ public ForkJoinTask submit(ForkJoinTask task) { - return poolSubmit(true, false, task); + Objects.requireNonNull(task); + poolSubmit(true, task); + return task; } /** @@ -3137,10 +3162,12 @@ public ForkJoinTask submit(ForkJoinTask task) { */ @Override public ForkJoinTask submit(Callable task) { - return poolSubmit(true, false, - (Thread.currentThread() instanceof ForkJoinWorkerThread) ? - new ForkJoinTask.AdaptedCallable(task) : - new ForkJoinTask.AdaptedInterruptibleCallable(task)); + ForkJoinTask t = + (Thread.currentThread() instanceof ForkJoinWorkerThread) ? + new ForkJoinTask.AdaptedCallable(task) : + new ForkJoinTask.AdaptedInterruptibleCallable(task); + poolSubmit(true, t); + return t; } /** @@ -3150,10 +3177,12 @@ public ForkJoinTask submit(Callable task) { */ @Override public ForkJoinTask submit(Runnable task, T result) { - return poolSubmit(true, false, - (Thread.currentThread() instanceof ForkJoinWorkerThread) ? - new ForkJoinTask.AdaptedRunnable(task, result) : - new ForkJoinTask.AdaptedInterruptibleRunnable(task, result)); + ForkJoinTask t = + (Thread.currentThread() instanceof ForkJoinWorkerThread) ? + new ForkJoinTask.AdaptedRunnable(task, result) : + new ForkJoinTask.AdaptedInterruptibleRunnable(task, result); + poolSubmit(true, t); + return t; } /** @@ -3169,11 +3198,10 @@ public ForkJoinTask submit(Runnable task) { ((Thread.currentThread() instanceof ForkJoinWorkerThread) ? new ForkJoinTask.AdaptedRunnable(task, null) : new ForkJoinTask.AdaptedInterruptibleRunnable(task, null)); - return poolSubmit(true, false, f); + poolSubmit(true, f); + return f; } - // Added mainly for possible use in Loom - /** * Submits the given task as if submitted from a non-{@code ForkJoinTask} * client. The task is added to a scheduling queue for submissions to the @@ -3192,7 +3220,9 @@ public ForkJoinTask submit(Runnable task) { * @since 20 */ public ForkJoinTask externalSubmit(ForkJoinTask task) { - return poolSubmit(true, true, task); + Objects.requireNonNull(task); + externalSubmissionQueue().push(task, this, true); + return task; } /** @@ -3212,7 +3242,9 @@ public ForkJoinTask externalSubmit(ForkJoinTask task) { * @since 19 */ public ForkJoinTask lazySubmit(ForkJoinTask task) { - return poolSubmit(false, false, task); + Objects.requireNonNull(task); + poolSubmit(false, task); + return task; } /** @@ -3277,7 +3309,7 @@ public List> invokeAllUninterruptibly(Collection t : tasks) { ForkJoinTask f = ForkJoinTask.adapt(t); futures.add(f); - poolSubmit(true, false, f); + poolSubmit(true, f); } for (int i = futures.size() - 1; i >= 0; --i) ((ForkJoinTask)futures.get(i)).quietlyJoin(); @@ -3300,7 +3332,7 @@ private List> invokeAll(Collection> tasks, for (Callable t : tasks) { ForkJoinTask f = ForkJoinTask.adaptInterruptible(t); futures.add(f); - poolSubmit(true, false, f); + poolSubmit(true, f); } for (int i = futures.size() - 1; i >= 0; --i) ((ForkJoinTask)futures.get(i)) @@ -3580,6 +3612,7 @@ protected int drainTasksTo(Collection> c) { */ public String toString() { // Use a single pass through queues to collect counts + int rs = runState; long st = stealCount; long qt = 0L, ss = 0L; int rc = 0; WorkQueue[] qs; WorkQueue q; @@ -3605,7 +3638,6 @@ public String toString() { int ac = (short)(c >>> RC_SHIFT); if (ac < 0) // ignore transient negative ac = 0; - long rs = runState; String level = ((rs & TERMINATED) != 0 ? "Terminated" : (rs & STOP) != 0 ? "Terminating" : (rs & SHUTDOWN) != 0 ? "Shutting down" : @@ -3638,7 +3670,8 @@ public String toString() { */ public void shutdown() { checkPermission(); - tryTerminate(false, true); + if (workerNamePrefix != null) // not common pool + tryTerminate(false, true); } /** @@ -3661,7 +3694,8 @@ public void shutdown() { */ public List shutdownNow() { checkPermission(); - tryTerminate(true, true); + if (workerNamePrefix != null) // not common pool + tryTerminate(true, true); return Collections.emptyList(); } @@ -3718,7 +3752,7 @@ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { long nanos = unit.toNanos(timeout); CountDownLatch done; - if ((config & ISCOMMON) != 0) { + if (workerNamePrefix == null) { // is common pool if (helpQuiescePool(this, nanos, true) < 0) throw new InterruptedException(); return false; @@ -3772,7 +3806,7 @@ public boolean awaitQuiescence(long timeout, TimeUnit unit) { */ @Override public void close() { - if ((runState & TERMINATED) == 0 && (config & ISCOMMON) == 0) { + if (workerNamePrefix != null && (runState & TERMINATED) == 0) { checkPermission(); CountDownLatch done = null; boolean interrupted = Thread.interrupted(); @@ -3908,24 +3942,24 @@ public static void managedBlock(ManagedBlocker blocker) /** ManagedBlock for ForkJoinWorkerThreads */ private void compensatedBlock(ManagedBlocker blocker) throws InterruptedException { - if (blocker == null) throw new NullPointerException(); + Objects.requireNonNull(blocker); for (;;) { int comp; boolean done; long c = ctl; if (blocker.isReleasable()) break; + if ((runState & STOP) != 0) + throw new InterruptedException(); if ((comp = tryCompensate(c)) >= 0) { - long post = (comp == 0) ? 0L : RC_UNIT; try { done = blocker.block(); } finally { - getAndAddCtl(post); + if (comp > 0) + getAndAddCtl(RC_UNIT); } if (done) break; } - else if ((runState & STOP) != 0) - throw new InterruptedException(); } } @@ -3936,12 +3970,9 @@ else if ((runState & STOP) != 0) * with the value returned by this method to re-adjust the parallelism. */ final long beginCompensatedBlock() { - for (int c;;) { - if ((c = tryCompensate(ctl)) > 0) - return RC_UNIT; - if (c == 0 || (runState & STOP) != 0) - return 0L; - } + int c; + do {} while ((c = tryCompensate(ctl)) < 0); + return (c == 0) ? 0L : RC_UNIT; } /** @@ -3956,7 +3987,7 @@ void endCompensatedBlock(long post) { /** ManagedBlock for external threads */ private static void unmanagedBlock(ManagedBlocker blocker) throws InterruptedException { - if (blocker == null) throw new NullPointerException(); + Objects.requireNonNull(blocker); do {} while (!blocker.isReleasable() && !blocker.block()); } diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index 1f03e2091f248..f88d5e849ad19 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -40,6 +40,7 @@ import java.lang.reflect.Constructor; import java.util.Collection; import java.util.List; +import java.util.Objects; import java.util.RandomAccess; import java.util.concurrent.locks.LockSupport; import jdk.internal.misc.Unsafe; @@ -269,14 +270,14 @@ final boolean casNext(Aux c, Aux v) { // used only in cancellation * control bits occupy only (some of) the upper half (16 bits) of * status field. The lower bits are used for user-defined tags. */ - private static final int DONE = 1 << 31; // must be negative - private static final int ABNORMAL = 1 << 16; - private static final int THROWN = 1 << 17; - private static final int MARKER = 1 << 30; // utility marker - private static final int HAVE_EXCEPTION = DONE | ABNORMAL | THROWN; + static final int DONE = 1 << 31; // must be negative + static final int ABNORMAL = 1 << 16; + static final int THROWN = 1 << 17; + static final int MARKER = 1 << 30; // utility marker + static final int HAVE_EXCEPTION = DONE | ABNORMAL | THROWN; - private static final int UNCOMPENSATE = 1 << 16; // helpJoin sentinel - private static final int SMASK = 0xffff; // short bits for tags + static final int UNCOMPENSATE = 1 << 16; // helpJoin sentinel + static final int SMASK = 0xffff; // short bits for tags // Fields volatile int status; // accessed directly by pool and workers @@ -364,15 +365,15 @@ final boolean trySetThrown(Throwable ex) { /** * Overridable action on setting exception */ - void onFJTExceptionSet(Throwable ex) { + void onAuxExceptionSet(Throwable ex) { } /** - * Tries to set exception, if so invoking onFJTExceptionSet + * Tries to set exception, if so invoking onAuxExceptionSet */ final void trySetException(Throwable ex) { if (trySetThrown(ex)) - onFJTExceptionSet(ex); + onAuxExceptionSet(ex); } /** @@ -409,15 +410,16 @@ else if (casAux(a, next)) */ private int awaitDone(ForkJoinPool pool, int compensation, boolean interruptible, long deadline) { - int s, interrupts = 0; // < 0 : throw; > 0 : re-interrupt - boolean queued = false; - Aux node = null; + int s; if ((s = status) >= 0) { + Aux node = null; try { // spinwait if out of memory node = new Aux(Thread.currentThread(), null); } catch (OutOfMemoryError ex) { } - for (Aux a;;) { // install node + boolean queued = false; + for (;;) { // try to install node + Aux a; if ((s = status) < 0) break; else if (node == null) @@ -428,7 +430,8 @@ else if (((a = aux) == null || a.ex == null) && } if (queued) { // await signal or interrupt LockSupport.setCurrentBlocker(this); - for (long ns;;) { + int interrupts = 0; // < 0 : throw; > 0 : re-interrupt + for (;;) { if ((s = status) < 0) break; if (interrupts < 0) { @@ -444,6 +447,7 @@ else if (((a = aux) == null || a.ex == null) && } } else if (deadline != 0L) { + long ns; if ((ns = deadline - System.nanoTime()) <= 0) { s = 0; break; @@ -670,13 +674,13 @@ public ForkJoinTask() {} public final ForkJoinTask fork() { Thread t; ForkJoinWorkerThread wt; ForkJoinPool p; ForkJoinPool.WorkQueue q; - U.storeStoreFence(); // ensure safe publication if ((t = Thread.currentThread()) instanceof ForkJoinWorkerThread) { p = (wt = (ForkJoinWorkerThread)t).pool; q = wt.workQueue; + U.storeStoreFence(); // ensure safely publishable } else - q = (p = ForkJoinPool.common).submissionQueue(false); + q = (p = ForkJoinPool.common).externalSubmissionQueue(); q.push(this, p, true); return this; } @@ -708,11 +712,8 @@ public final V join() { * @return the computed result */ public final V invoke() { - int s; doExec(); - if ((((s = status) < 0 ? s : awaitDone(false, 0L)) & ABNORMAL) != 0) - reportException(false); - return getRawResult(); + return join(); } /** @@ -1586,7 +1587,7 @@ static final class AdaptedRunnable extends ForkJoinTask @SuppressWarnings("serial") // Conditionally serializable T result; AdaptedRunnable(Runnable runnable, T result) { - if (runnable == null) throw new NullPointerException(); + Objects.requireNonNull(runnable); this.runnable = runnable; this.result = result; // OK to set this even before completion } @@ -1608,7 +1609,7 @@ static final class AdaptedRunnableAction extends ForkJoinTask @SuppressWarnings("serial") // Conditionally serializable final Runnable runnable; AdaptedRunnableAction(Runnable runnable) { - if (runnable == null) throw new NullPointerException(); + Objects.requireNonNull(runnable); this.runnable = runnable; } public final Void getRawResult() { return null; } @@ -1631,7 +1632,7 @@ static final class AdaptedCallable extends ForkJoinTask @SuppressWarnings("serial") // Conditionally serializable T result; AdaptedCallable(Callable callable) { - if (callable == null) throw new NullPointerException(); + Objects.requireNonNull(callable); this.callable = callable; } public final T getRawResult() { return result; } @@ -1714,7 +1715,7 @@ static final class AdaptedInterruptibleCallable extends InterruptibleTask @SuppressWarnings("serial") // Conditionally serializable T result; AdaptedInterruptibleCallable(Callable callable) { - if (callable == null) throw new NullPointerException(); + Objects.requireNonNull(callable); this.callable = callable; } public final T getRawResult() { return result; } @@ -1733,7 +1734,7 @@ static final class AdaptedInterruptibleRunnable extends InterruptibleTask @SuppressWarnings("serial") // Conditionally serializable final T result; AdaptedInterruptibleRunnable(Runnable runnable, T result) { - if (runnable == null) throw new NullPointerException(); + Objects.requireNonNull(runnable); this.runnable = runnable; this.result = result; } @@ -1751,14 +1752,14 @@ static final class RunnableExecuteAction extends InterruptibleTask { @SuppressWarnings("serial") // Conditionally serializable final Runnable runnable; RunnableExecuteAction(Runnable runnable) { - if (runnable == null) throw new NullPointerException(); + Objects.requireNonNull(runnable); this.runnable = runnable; } public final Void getRawResult() { return null; } public final void setRawResult(Void v) { } final Void compute() { runnable.run(); return null; } final Object adaptee() { return runnable; } - void onFJTExceptionSet(Throwable ex) { // if a handler, invoke it + void onAuxExceptionSet(Throwable ex) { // if a handler, invoke it Thread t; java.lang.Thread.UncaughtExceptionHandler h; if ((h = ((t = Thread.currentThread()). getUncaughtExceptionHandler())) != null) { @@ -1846,7 +1847,7 @@ static final class InvokeAnyTask extends InterruptibleTask { final InvokeAnyTask pred; // to traverse on cancellation InvokeAnyTask(Callable callable, InvokeAnyRoot root, InvokeAnyTask pred) { - if (callable == null) throw new NullPointerException(); + Objects.requireNonNull(callable); this.callable = callable; this.root = root; this.pred = pred; From dcfe43ab31a9ae118e25cd9c08c05559b1f3a727 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Wed, 26 Apr 2023 13:00:15 -0400 Subject: [PATCH 18/61] Reduce contention --- .../java/util/concurrent/ForkJoinPool.java | 545 +++++++++--------- .../java/util/concurrent/ForkJoinTask.java | 96 ++- 2 files changed, 308 insertions(+), 333 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index d966c0e6f3167..a881e67478c7e 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -583,16 +583,12 @@ public class ForkJoinPool extends AbstractExecutorService { * method scan) as an encoded sliding window of the last two * sources, and stop signalling when the last two were from the * same source. And similarly not retry when two CAS failures - * were from the same source. Additionally, we allow attempts to - * propagate signals (argument "propagated" in signalWork) to - * return without singalling if another contending call to - * signalWork increased the number of active workers. These - * mechanisms may result in transiently too few workers, but - * once workers poll from a new source, they rapidly reactivate - * others. (An implementation / encoding quirk: Because of the - * offset, unnecessary signals for the maximum-numbered 32767th - * worker, which is unlikely to ever be produced, are not - * suppressed.) + * were from the same source. These mechanisms may result in + * transiently too few workers, but once workers poll from a new + * source, they rapidly reactivate others. (An implementation / + * encoding quirk: Because of the offset, unnecessary signals + * for the maximum-numbered 32767th worker, which is unlikely to + * ever be produced, are not suppressed.) * * * Despite these, signal contention and overhead effects still * occur during ramp-up and ramp-down of small computations. @@ -633,7 +629,7 @@ public class ForkJoinPool extends AbstractExecutorService { * Those in awaitWork are set to small values that only cover * near-miss scenarios for inactivate/activate races. Because idle * workers are often not yet blocked (parked), we use the - * WorkQueue source field to advertise that a waiter actually + * WorkQueue parker field to advertise that a waiter actually * needs unparking upon signal. * * Quiescence. Workers scan looking for work, giving up when they @@ -675,26 +671,25 @@ public class ForkJoinPool extends AbstractExecutorService { * * Termination. A call to shutdownNow invokes tryTerminate to * atomically set a mode bit, performed on the runState lock to - * also disable further queue array updates. However, the process - * of termination is intrinsically non-atomic. The calling thread, - * as well as other workers thereafter terminating help cancel - * queued tasks and interrupt other workers. These actions race - * with unterminated workers. By default, workers check for - * termination only when accessing pool state (usually the - * "queues" array). This may take a while but suffices for - * structured computational tasks. But not necessarily for - * others. Class InterruptibleTask (see below) further arranges - * runState checks before executing task bodies, and ensures - * interrupts while terminating. Even so, there are no guarantees - * after an abrupt shutdown that remaining tasks complete normally - * or exceptionally or are cancelled. Termination may fail to - * complete if running tasks repeatedly ignore both task status - * and interrupts and/or produce more tasks after others that - * could cancel them have exited. Parallelizing termination - * provides multiple attempts to cancel tasks and workers when - * some are only boundedly unresponsive, in addition to speeding - * termination when there are large numbers of tasks that need to - * be cancelled. + * avoid inconsistencies. However, the process of termination is + * intrinsically non-atomic. The calling thread, as well as other + * workers thereafter terminating help cancel queued tasks and + * interrupt other workers. These actions race with unterminated + * workers. By default, workers check for termination only when + * accessing pool state (usually the "queues" array). This may + * take a while but suffices for structured computational tasks. + * But not necessarily for others. Class InterruptibleTask (see + * below) further arranges runState checks before executing task + * bodies, and ensures interrupts while terminating. Even so, + * there are no guarantees after an abrupt shutdown that remaining + * tasks complete normally or exceptionally or are cancelled. + * Termination may fail to complete if running tasks repeatedly + * ignore both task status and interrupts and/or produce more + * tasks after others that could cancel them have exited. + * Parallelizing termination provides multiple attempts to cancel + * tasks and workers when some are only boundedly unresponsive, in + * addition to speeding termination when there are large numbers + * of tasks that need to be cancelled. * * Trimming workers. To release resources after periods of lack of * use, a worker starting to wait when the pool is quiescent will @@ -985,7 +980,7 @@ public class ForkJoinPool extends AbstractExecutorService { * * * New abstract class ForkJoinTask.InterruptibleTask ensures * handling of tasks submitted under the ExecutorService - * API consistent with specs. + * API are consistent with specs. * * Method checkQuiescence() replaces previous quiescence-related * checks, relying on a sequence lock instead of ReentrantLock. * * Termination processing now ensures that internal data @@ -1039,7 +1034,7 @@ public class ForkJoinPool extends AbstractExecutorService { static final int LOCKED = 1 << 3; // lowest seqlock bit // spin/sleep limits for runState locking and elsewhere; powers of 2 - static final int SPIN_WAITS = 1 << 7; // calls to onSpinWait + static final int SPIN_WAITS = 96; // calls to onSpinWait static final int MIN_SLEEP = 1 << 10; // approx 1 usec as nanos static final int MAX_SLEEP = 1 << 20; // approx 1 sec as nanos @@ -1209,6 +1204,7 @@ static final class WorkQueue { int stackPred; // pool stack (ctl) predecessor link int config; // index, mode, REGISTERED bits int base; // index of next slot for poll + volatile Thread parker; // set to owner while parked final ForkJoinWorkerThread owner; // owning thread or null if shared ForkJoinTask[] array; // the queued tasks; power of 2 size @@ -1220,7 +1216,7 @@ static final class WorkQueue { @jdk.internal.vm.annotation.Contended("w") volatile int phase; // versioned, negative if inactive @jdk.internal.vm.annotation.Contended("w") - volatile int source; // source queue id + 1, or < 0 if parked + volatile int source; // source queue id + 1 @jdk.internal.vm.annotation.Contended("w") int nsteals; // number of steals from other queues @@ -1295,7 +1291,7 @@ final void push(ForkJoinTask task, ForkJoinPool pool, signal = false; // not empty } if (signal && pool != null) - pool.signalWork(false); + pool.signalWork(); } } @@ -1428,25 +1424,25 @@ final ForkJoinTask peek() { */ final ForkJoinTask poll(ForkJoinPool pool) { for (int b = base;;) { - int cap, k; ForkJoinTask[] a; + ForkJoinTask[] a; int cap; if ((a = array) == null || (cap = a.length) <= 0) break; // currently impossible - ForkJoinTask t = a[k = (cap - 1) & b], u; + int nb = b + 1, nk = (cap - 1) & nb, k; + ForkJoinTask t = a[k = (cap - 1) & b]; U.loadFence(); - int nb = b + 1, nk = (cap - 1) & nb; if (b == (b = base)) { - if (t == null) - u = a[k]; // reread or CAS - else if ((u = cmpExSlotToNull(a, k, t)) == t) { - base = nb; - U.storeFence(); - if (pool != null && a[nk] != null) - pool.signalWork(true); // propagate - return t; - } - if (u == null && access == 0 && top - b <= 0) - break; // empty - b = base; + if (t == null) + t = a[k]; // reread or CAS + else if (t == (t = cmpExSlotToNull(a, k, t))) { + base = nb; + U.storeFence(); + if (pool != null && a[nk] != null) + pool.signalWork(); // propagate + return t; + } + if (t == null && access == 0 && top - b <= 0) + break; // empty + b = base; } } return null; @@ -1460,25 +1456,22 @@ else if ((u = cmpExSlotToNull(a, k, t)) == t) { private ForkJoinTask tryPoll() { ForkJoinTask[] a; int cap; if ((a = array) != null && (cap = a.length) > 0) { - for (int b = base, k;;) { - ForkJoinTask t = a[k = (cap - 1) & b], u; + for (int b = base;;) { + int nb = b + 1, k; + ForkJoinTask t = a[k = (cap - 1) & b]; U.loadFence(); - int nb = b + 1; - if (b != (b = base)) - ; - else if (t == null) { - if (a[k] == null) + if (b == (b = base)) { + if (t == null) + t = a[k]; + else if (t == (t = cmpExSlotToNull(a, k, t))) { + base = nb; + U.storeFence(); + return t; + } + if (t == null) break; - } - else if ((u = cmpExSlotToNull(a, k, t)) == t) { - base = nb; - U.storeFence(); - return t; - } - else if (u == null) - break; - else b = base; + } } } return null; @@ -1601,31 +1594,31 @@ final int helpComplete(ForkJoinTask task, boolean internal, int limit) { final void helpAsyncBlocker(ManagedBlocker blocker) { if (blocker != null) { for (;;) { - int b = base, cap, k; ForkJoinTask[] a; - if ((a = array) == null || (cap = a.length) <= 0 || - top - b <= 0) + ForkJoinTask[] a = array; + int b = base, cap; + if (a == null || (cap = a.length) <= 0 || top - b <= 0) break; + int nb = b + 1, nk = (cap - 1) & nb, k; ForkJoinTask t = a[k = (cap - 1) & b]; U.loadFence(); - int nb = b + 1, nk = (cap - 1) & nb; if (base != b) ; else if (blocker.isReleasable()) break; else if (a[k] != t) ; - else if (t != null) { - if (!(t instanceof CompletableFuture - .AsynchronousCompletionTask)) + else if (t == null) { + if (a[nk] == null) break; - else if (cmpExSlotToNull(a, k, t) == t) { - base = nb; - U.storeFence(); - t.doExec(); - } } - else if (a[nk] == null) + else if (!(t instanceof CompletableFuture + .AsynchronousCompletionTask)) break; + else if (cmpExSlotToNull(a, k, t) == t) { + base = nb; + U.storeFence(); + t.doExec(); + } } } } @@ -1637,8 +1630,8 @@ else if (a[nk] == null) */ final boolean isApparentlyUnblocked() { Thread wt; Thread.State s; - return ((wt = owner) != null && phase >= 0 && - (config & REGISTERED) != 0 && + return (phase > 0 && (config & REGISTERED) != 0 && + (wt = owner) != null && (s = wt.getState()) != Thread.State.BLOCKED && s != Thread.State.WAITING && s != Thread.State.TIMED_WAITING); @@ -1842,14 +1835,13 @@ final String nextWorkerThreadName() { final void registerWorker(WorkQueue w) { if (w != null) { WorkQueue[] qs; int n; - ThreadLocalRandom.localInit(); - int seed = ThreadLocalRandom.getProbe(); - int id = ((seed << 1) | 1) & SMASK; // initial index guess - w.stackPred = seed; // used by runWorker w.array = new ForkJoinTask[INITIAL_QUEUE_CAPACITY]; + ThreadLocalRandom.localInit(); + int seed = w.stackPred = ThreadLocalRandom.getProbe(); int cfg = w.config | REGISTERED | (config & FIFO); + int id = ((seed << 1) | 1) & SMASK; // initial index guess if ((qs = queues) != null && (n = qs.length) > 0) { - // try to find open slot without lock; rescan below if needed + // scan for open slot without lock; rescan below if needed for (int k = n, m = n - 1; ; id += 2) { if (qs[id &= m] == null) break; @@ -1871,24 +1863,21 @@ final void registerWorker(WorkQueue w) { } } } - w.config = id | cfg; - w.phase = id; // now publishable - if ((rs & STOP) != 0) - ; // skip if stopping - else if (id < n) + w.phase = w.config = id | cfg; // now publishable + if (id < n) qs[id] = w; - else { // expand array + else if ((rs & STOP) == 0) { // expand unless stopping int an = n << 1, am = an - 1; WorkQueue[] as = new WorkQueue[an]; as[id & am] = w; for (int j = 1; j < n; j += 2) as[j] = qs[j]; for (int j = 0; j < n; j += 2) { - WorkQueue q; // shared queues may move + WorkQueue q; // shared queues may move if ((q = qs[j]) != null) as[q.config & am] = q; } - U.storeFence(); // fill before publish + U.storeFence(); // fill before publish queues = as; } } @@ -1914,8 +1903,8 @@ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { cfg = 0; else if (((cfg = w.config) & REGISTERED) != 0) { w.config = cfg & ~REGISTERED; - if (w.phase < 0) // possible if stopped before released - reactivate(w); + if (w.phase < 0) + reactivate(w); // possible if stopped before released if (w.top - w.base > 0) { for (ForkJoinTask t; (t = w.nextLocalTask()) != null; ) ForkJoinTask.cancelIgnoringExceptions(t); @@ -1941,7 +1930,7 @@ else if ((int)c == 0) // was dropped on timeout } releaseRunStateLock(); if (rs == 0 && (cfg & REGISTERED) != 0) - signalWork(false); // may replace unless trimmed or uninitialized + signalWork(); // may replace unless trimmed or uninitialized } if (ex != null) ForkJoinTask.rethrow(ex); @@ -1949,11 +1938,10 @@ else if ((int)c == 0) // was dropped on timeout /** * Releases an idle worker, or creates one if not enough exist. - * @param propagated true if can exit on any signal success */ - final void signalWork(boolean propagated) { + final void signalWork() { int pc = parallelism; - for (long c = ctl, u; ; c = u) { + for (long c = ctl;;) { WorkQueue[] qs = queues; WorkQueue v = null; long ac = (c + RC_UNIT) & RC_MASK, nc; @@ -1966,18 +1954,17 @@ else if ((short)(c >>> TC_SHIFT) < pc) nc = ((c + TC_UNIT) & TC_MASK); else break; - if (c == (u = compareAndExchangeCtl(c, nc | ac))) { + if (c == (c = compareAndExchangeCtl(c, nc | ac))) { if (v == null) createWorker(); else { + Thread t; v.phase = sp; - if (v.source < 0) - U.unpark(v.owner); + if ((t = v.parker) != null) + U.unpark(t); } break; } - else if (propagated && (u & RC_MASK) > (c & RC_MASK)) - break; // another signaller succeeded } } @@ -1987,8 +1974,8 @@ else if (propagated && (u & RC_MASK) > (c & RC_MASK)) * no-create case of signalWork. */ private void reactivate(WorkQueue w) { - for (long c = ctl, u; ; c = u) { - WorkQueue v; + for (long c = ctl;;) { + WorkQueue v; Thread t; WorkQueue[] qs = queues; int sp = (int)c & ACTIVE, i = sp & SMASK; long pc = (UC_MASK & (c + RC_UNIT)) | (c & TC_MASK); @@ -1998,55 +1985,37 @@ private void reactivate(WorkQueue w) { long nc = (v.stackPred & SP_MASK) | pc; if (w != null && w != v && w.phase >= 0) break; - if (c == (u = compareAndExchangeCtl(c, nc))) { + if (c == (c = compareAndExchangeCtl(c, nc))) { v.phase = sp; - if (v.source < 0) - U.unpark(v.owner); + if ((t = v.parker) != null) + U.unpark(t); if (v == w || w == null) break; } } } - /** - * Tries to deactivate worker w; called only on idle timeout. - */ - private boolean tryTrim(WorkQueue w) { - if (w != null) { - int pred = w.stackPred, cfg = w.config | TRIMMED; - long c = ctl; - int sp = (int)c & ACTIVE; - if ((sp & SMASK) == (cfg & SMASK) && - compareAndSetCtl(c, ((pred & SP_MASK) | - (UC_MASK & (c - TC_UNIT))))) { - w.config = cfg; // add sentinel for deregisterWorker - w.phase = sp; - return true; - } - } - return false; - } - /** * Internal version of isQuiescent and related functionality. - * Returns true if terminating or submission queues are empty and - * unlocked, and all workers are inactive. If so, and quiescent - * shutdown is enabled, sets runState mode to STOP. + * @return negative if terminating, 0 if all workers are inactive + * and submission queues are empty and unlocked, else positive */ - private boolean checkQuiescence() { + private int checkQuiescence() { long prevCtl = 0L; boolean scanned = false; outer: for (int prevState = 0, rs = runState; ; prevState = rs) { long c; WorkQueue[] qs; int n; if ((rs & STOP) != 0) // terminating - return true; + return -1; if (((c = ctl) & RC_MASK) != 0L) break; // active if (prevCtl != (prevCtl = c)) scanned = false; // inconsistent - if (scanned) { - if ((rs & SHUTDOWN) == 0 || casRunState(rs, rs | STOP)) - return true; // try to trigger termination + else if (scanned) { + if ((rs & SHUTDOWN) == 0) + return 0; + if (casRunState(rs, rs | STOP)) + return -1; // try to trigger termination scanned = false; // retry } if ((qs = queues) != null && (n = qs.length) > 0) { @@ -2066,7 +2035,7 @@ private boolean checkQuiescence() { if ((rs = runState) == prevState && (rs & LOCKED) == 0) scanned = true; // same unlocked state } - return false; + return 1; } /** @@ -2083,12 +2052,12 @@ final void runWorker(WorkQueue w) { r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift if ((srcs = scan(w, srcs, r)) < 0) { if (rs != (rs = runState)) { - if ((rs & STOP) != 0) // stop or rescan if stale + if ((rs & STOP) != 0) break; - srcs &= ACTIVE; + srcs &= ACTIVE; // stale; rescan } else { - long c; // try to inactivate/enqueue + long c; // try to inactivate & enqueue int p = phase + SS_SEQ, np = p | INACTIVE; long nc = ((p & ACTIVE & SP_MASK) | ((c = ctl) - RC_UNIT) & UC_MASK); @@ -2114,41 +2083,49 @@ else if ((phase = awaitWork(w, nc)) > 0) * returning scan window and retry indicator. * * @param w caller's WorkQueue - * @param srcs encoding for the two previously non-empty scanned queues + * @param prevSrcs encodes up to two previously non-empty scanned queues * @param r random seed * @return the next srcs value to use, with negative (INACTIVE) set if * none found */ - private int scan(WorkQueue w, int srcs, int r) { + private int scan(WorkQueue w, int prevSrcs, int r) { WorkQueue[] qs = queues; int n = (w == null || qs == null) ? 0 : qs.length; - int nsrcs = ((srcs << 16) | 1) & ACTIVE; // base of next srcs window - for (int step = (r >>> 16) | 1, i = n; i > 0; --i, r += step) { - int j, cap, b, k; WorkQueue q; ForkJoinTask[] a; + int next = prevSrcs | INACTIVE; // return value on empty scan + int window = ((prevSrcs << 16) | 1) & ACTIVE; // base of next window + int step = (r >>> 16) | 1; // random stride + outer: for (int i = n; i > 0; --i, r += step) { + int j, cap; WorkQueue q; ForkJoinTask[] a; if ((q = qs[j = r & SMASK & (n - 1)]) != null && (a = q.array) != null && (cap = a.length) > 0) { - ForkJoinTask t = a[k = (cap - 1) & (b = q.base)], u; - U.loadFence(); // to re-read b and t - int nb = b + 1, nk = (cap - 1) & nb, qsrcs = nsrcs + j; - if (q.base != b) - return srcs; // inconsistent - if (t == null) { - if (a[k] != null || a[nk] != null) - return qsrcs; // nonempty; restart - } - else if ((u = WorkQueue.cmpExSlotToNull(a, k, t)) == t) { - q.base = nb; - w.source = qsrcs; - if (qsrcs != srcs && a[nk] != null) - signalWork(true); // propagate at most twice/run - w.topLevelExec(t, q); - return qsrcs; + for (int srcs = window + j, b = q.base, k;;) { + ForkJoinTask t = a[k = (cap - 1) & b]; + U.loadFence(); // re-read b and t + if (b == (b = q.base)) { // else inconsistent; retry + ForkJoinTask u; + int nb = b + 1, nk = (cap - 1) & nb; + if (t == null) + u = a[k]; + else if (t == (u = WorkQueue.cmpExSlotToNull(a, k, t))) { + q.base = nb; + w.source = next = srcs; + if (a[nk] != null && srcs != prevSrcs) + signalWork(); // propagate at most twice/run + w.topLevelExec(t, q); + break outer; + } + if (u == null) { + if (a[nk] != null && next < 0 && + (t == null || srcs != prevSrcs)) + next = srcs; // limit rescans + break; + } + b = q.base; // retry + } } - else if (u != null || (qsrcs != srcs && a[nk] != null)) - return qsrcs; // limit retries if contended } } - return srcs | INACTIVE; + return next; } /** @@ -2158,44 +2135,48 @@ else if (u != null || (qsrcs != srcs && a[nk] != null)) * @return current phase, or negative for exit */ private int awaitWork(WorkQueue w, long queuedCtl) { + boolean idle; int phase, active; + if ((active = (short)(queuedCtl >>> RC_SHIFT)) > 0) + idle = false; // not quiescent + else if (!(idle = (checkQuiescence() == 0))) + reactivate(null); // ensure live if (w == null) - return -1; // currently impossible - boolean idle; // true if pool idle - long deadline = 0L; // for timed wait if idle - int nspins = ((short)(queuedCtl >>> TC_SHIFT) + 0xb) * 3; - if ((short)(queuedCtl >>> RC_SHIFT) > 0) - idle = false; - else if (idle = checkQuiescence()) - deadline = keepAlive + System.currentTimeMillis(); - else - reactivate(null); // ensure live if may be tasks - int phase; // nonnegative on normal return - outer: for (;;) { // await signal or termination - int spins = nspins; // approx #accesses to scan+signal - if ((runState & STOP) != 0) - return -1; - do { // spin to cushion near-misses - if ((phase = w.phase) >= 0) - break outer; - Thread.onSpinWait(); - } while (--spins > 0); - LockSupport.setCurrentBlocker(this); // emulate LockSupport.park - w.source = INACTIVE; // enable unpark - if (w.phase < 0) - U.park(idle, deadline); - w.source = 0; // disable unpark - LockSupport.setCurrentBlocker(null); - if ((phase = w.phase) >= 0) - break; - if (idle) { // check for idle timeout - if (deadline - System.currentTimeMillis() < TIMEOUT_SLOP) { - if (tryTrim(w)) - return -1; - deadline += keepAlive; // not at head; restart timer - } + phase = -1; // currently impossible + else { + if ((phase = w.phase) < 0 & !idle) { + int spins = ((short)(queuedCtl >>> TC_SHIFT) << 1) + active; + do { // spin for approx #accesses to scan+signal + Thread.onSpinWait(); + } while ((phase = w.phase) < 0 && --spins > 0); + } + if (phase < 0) { // emulate LockSupport.park + LockSupport.setCurrentBlocker(this); + long deadline = idle? System.currentTimeMillis() + keepAlive: 0L; + for (;;) { // await signal or termination + if ((runState & STOP) != 0) + break; + w.parker = Thread.currentThread(); + if (w.phase < 0) + U.park(idle, deadline); + w.parker = null; + if ((phase = w.phase) >= 0) + break; + if (idle && // check for idle timeout + deadline - System.currentTimeMillis() < TIMEOUT_SLOP) { + int id = phase & SMASK; + long sp = w.stackPred & SP_MASK; + long c = ctl, nc = sp | (UC_MASK & (c - TC_UNIT)); + if (((int)c & SMASK) == id && compareAndSetCtl(c, nc)) { + w.phase = w.config = id | TRIMMED; + break; // add sentinel for deregisterWorker + } + deadline += keepAlive; // not at head; restart timer + } + Thread.interrupted(); // clear status for next park + } // loop on interrupt or stale unpark + LockSupport.setCurrentBlocker(null); } - Thread.interrupted(); // clear status for next park - } // loop on interrupt or stale unpark + } return phase; } @@ -2247,14 +2228,14 @@ private int tryCompensate(long c) { if ((runState & STOP) != 0) // terminating return 0; else if (sp != 0 && active <= pc) { // activate idle worker - WorkQueue[] qs; WorkQueue v; int i; + WorkQueue[] qs; WorkQueue v; int i; Thread t; if (ctl == c && (qs = queues) != null && qs.length > (i = sp & SMASK) && (v = qs[i]) != null) { long nc = (v.stackPred & SP_MASK) | (UC_MASK & c); if (compareAndSetCtl(c, nc)) { v.phase = sp; - if (v.source < 0) - U.unpark(v.owner); + if ((t = v.parker) != null) + U.unpark(t); return UNCOMPENSATE; } } @@ -2298,64 +2279,66 @@ final void uncompensate() { */ final int helpJoin(ForkJoinTask task, WorkQueue w, boolean internal) { int s = 0; - if (task == null) - return s; - if (w != null) - w.tryRemoveAndExec(task, internal); - if ((s = task.status) >= 0 && internal && w != null) { - int wsrc = w.source & SMASK; - int wid = (w.config & SMASK) + 1, r = wid + 1; - long sctl = 0L; // track stability - outer: for (boolean rescan = true;;) { - WorkQueue[] qs; - if ((s = task.status) < 0) - break; - if (!rescan && sctl == (sctl = ctl) && - (s = tryCompensate(sctl)) >= 0) - break; - rescan = false; - int n = ((qs = queues) == null) ? 0 : qs.length; - scan: for (int i = n >>> 1; i > 0; --i, r += 2) { - int j, cap; WorkQueue q; ForkJoinTask[] a; - if ((q = qs[j = r & SMASK & (n - 1)]) != null && - (a = q.array) != null && (cap = a.length) > 0) { - for (int src = j + 1;;) { - int sq = q.source, b = q.base, k = (cap - 1) & b; - ForkJoinTask t = a[k]; - U.loadFence(); - boolean eligible; // check steal chain - for (int steps = n, v = sq;;) { - WorkQueue p; - if ((v &= SMASK) == wid) { - eligible = true; - break; - } - if (v == 0 || - (p = qs[(v - 1) & (n - 1)]) == null || - --steps == 0) { // bound steps - eligible = false; - break; - } - v = p.source; - } - if ((s = task.status) < 0) - break outer; // validate - int nb = b + 1, qdepth = q.top - b; - if (q.source == sq && q.base == b && a[k] == t) { - if (t == null) { - rescan |= (qdepth > 0); - break; // revisit if nonempty + if (task != null) { + if (w != null) + w.tryRemoveAndExec(task, internal); + if ((s = task.status) >= 0 && internal && w != null) { + int wsrc = w.source & SMASK; + int wid = (w.config & SMASK) + 1, r = wid + 1; + long sctl = 0L; // track stability + outer: for (boolean rescan = true;;) { + WorkQueue[] qs; + if ((s = task.status) < 0) + break; + if (!rescan && sctl == (sctl = ctl) && + (s = tryCompensate(sctl)) >= 0) + break; + rescan = false; + int n = ((qs = queues) == null) ? 0 : qs.length; + scan: for (int i = n >>> 1; i > 0; --i, r += 2) { + int j, cap; WorkQueue q; ForkJoinTask[] a; + if ((q = qs[j = r & SMASK & (n - 1)]) != null && + (a = q.array) != null && (cap = a.length) > 0) { + for (int src = j + 1, sq = q.source, b, k;;) { + ForkJoinTask t = a[k = (cap - 1) & (b = q.base)]; + U.loadFence(); + boolean eligible; // check steal chain + for (int steps = n, v = sq;;) { + WorkQueue p; + if ((v &= SMASK) == wid) { + eligible = true; + break; + } + if (v == 0 || + (p = qs[(v - 1) & (n - 1)]) == null || + --steps == 0) { // bound steps + eligible = false; + break; + } + v = p.source; } - if (t != task && !eligible) - break; - rescan = true; // restart at same index - if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { - q.base = nb; - w.source = src; - t.doExec(); - w.source = wsrc; + if ((s = task.status) < 0) + break outer; // validate + int nb = b + 1, qdepth = q.top - b; + if (sq == (sq = q.source) && + q.base == b && a[k] == t) { + if (t == null) { + if (qdepth > 0) // revisit if nonempty + rescan = true; + break; + } + if (t != task && !eligible) + break; + if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { + q.base = nb; + w.source = src; + t.doExec(); + w.source = wsrc; + rescan = true; // restart at index r + break scan; + } + sq = q.source; } - break scan; } } } @@ -2396,36 +2379,35 @@ final int helpComplete(ForkJoinTask task, WorkQueue w, boolean internal) { for (int b = q.base, k;;) { ForkJoinTask t = a[k = (cap - 1) & b]; U.loadFence(); - int nb = b + 1, nk = (cap - 1) & nb, steps = cap; boolean eligible = false; if (t instanceof CountedCompleter) { CountedCompleter f = (CountedCompleter)t; - do { + for (int steps = cap; steps > 0; --steps) { if (f == task) { eligible = true; break; } - } while ((f = f.completer) != null && - --steps > 0); // bound steps + if ((f = f.completer) == null) + break; + } } - if ((s = task.status) < 0) - break outer; // validate + if ((s = task.status) < 0) // validate + break outer; if (b == (b = q.base) && a[k] == t) { - if (t == null) { - if (!rescan && a[nk] != null) + int nb = b + 1, nk = (cap - 1) & nb; + if (!eligible) { + if (t == null && a[nk] != null) rescan = true; // revisit break; } - if (!eligible) - break; - rescan = true; if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { q.base = nb; U.storeFence(); - locals = true; // process local queue t.doExec(); + locals = rescan = true; + break scan; } - break scan; + b = q.base; } } } @@ -2472,12 +2454,11 @@ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { if ((q = qs[j = r & SMASK & (n - 1)]) != null && q != w) { for (int src = j + 1;;) { ForkJoinTask[] a = q.array; - int b = q.base, cap, k; + int b = q.base, nb = b + 1, cap, k; if (a == null || (cap = a.length) <= 0) break; ForkJoinTask t = a[k = (cap - 1) & b]; U.loadFence(); - int nb = b + 1; if (t != null && phase < 0) // reactivate before taking w.phase = phase = activePhase; if (q.base != b || a[k] != t) @@ -2504,11 +2485,11 @@ else if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { } rs = runState; if (rescan || (rs & LOCKED) != 0 || rs != prevState) - ; // retry + ; // inconsistent else if (!busy) break; else if (phase >= 0) { - waits = 0; // to recheck, then sleep + waits = 0; // recheck, then sleep w.phase = phase = inactivePhase; } else if (System.nanoTime() - startTime > nanos) { @@ -2535,7 +2516,7 @@ else if (waits == 0) // same as spinLockRunState except * @return positive if quiescent, negative if interrupted, else 0 */ private int externalHelpQuiesce(long nanos, boolean interruptible) { - if (!checkQuiescence()) { + if (checkQuiescence() > 0) { long startTime = System.nanoTime(); long maxSleep = Math.min(nanos >>> 8, MAX_SLEEP); for (int waits = 0;;) { @@ -2546,7 +2527,7 @@ else if ((t = pollScan(false)) != null) { waits = 0; t.doExec(); } - else if (checkQuiescence()) + else if (checkQuiescence() <= 0) break; else if (System.nanoTime() - startTime > nanos) return 0; @@ -2647,7 +2628,7 @@ private void poolSubmit(boolean signalIfEmpty, ForkJoinTask task) { q = wt.workQueue; U.storeStoreFence(); // ensure safely publishable } - else + else // find and lock queue q = submissionQueue(ThreadLocalRandom.getProbe()); q.push(task, this, signalIfEmpty); } @@ -2777,7 +2758,7 @@ private int tryTerminate(boolean now, boolean enable) { else { if ((isShutdown = (rs & SHUTDOWN)) == 0 && enable) getAndBitwiseOrRunState(isShutdown = SHUTDOWN); - if (isShutdown != 0 && checkQuiescence()) + if (isShutdown != 0 && checkQuiescence() < 0) rs = STOP | SHUTDOWN; } } @@ -3481,7 +3462,7 @@ public int getActiveThreadCount() { * @return {@code true} if all threads are currently idle */ public boolean isQuiescent() { - return checkQuiescence(); + return checkQuiescence() <= 0; } /** diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index f88d5e849ad19..238186bc84605 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -242,9 +242,9 @@ public abstract class ForkJoinTask implements Future, Serializable { * waiters. Cancelled waiters try to unsplice. */ static final class Aux { - final Thread thread; - final Throwable ex; // null if a waiter - Aux next; // accessed only via memory-acquire chains + Thread thread; // thrower or waiter + final Throwable ex; + Aux next; // accessed only via memory-acquire chains Aux(Thread thread, Throwable ex) { this.thread = thread; this.ex = ex; @@ -273,11 +273,10 @@ final boolean casNext(Aux c, Aux v) { // used only in cancellation static final int DONE = 1 << 31; // must be negative static final int ABNORMAL = 1 << 16; static final int THROWN = 1 << 17; - static final int MARKER = 1 << 30; // utility marker static final int HAVE_EXCEPTION = DONE | ABNORMAL | THROWN; - - static final int UNCOMPENSATE = 1 << 16; // helpJoin sentinel + static final int MARKER = 1 << 30; // utility marker static final int SMASK = 0xffff; // short bits for tags + static final int UNCOMPENSATE = 1 << 16; // helpJoin sentinel // Fields volatile int status; // accessed directly by pool and workers @@ -293,18 +292,24 @@ private int getAndBitwiseOrStatus(int v) { private boolean casStatus(int c, int v) { return U.compareAndSetInt(this, STATUS, c, v); } + + // Support for waiting and signalling + private boolean casAux(Aux c, Aux v) { return U.compareAndSetReference(this, AUX, c, v); } - + private Aux compareAndExchangeAux(Aux c, Aux v) { + return (Aux)U.compareAndExchangeReference(this, AUX, c, v); + } /** Removes and unparks waiters */ private void signalWaiters() { - for (Aux a; (a = aux) != null && a.ex == null; ) { - if (casAux(a, null)) { // detach entire list - for (Thread t; a != null; a = a.next) { - if ((t = a.thread) != Thread.currentThread() && t != null) - LockSupport.unpark(t); // don't self-signal - } + for (Aux a = aux;;) { + if (a == null || a.ex != null) + break; + if (a == (a = compareAndExchangeAux(a, null))) { + do { // detach entire list + LockSupport.unpark(a.thread); + } while ((a = a.next) != null); break; } } @@ -376,29 +381,6 @@ final void trySetException(Throwable ex) { onAuxExceptionSet(ex); } - /** - * Cleans out a wait node after an interrupted or timed out wait. - * Similar to AbstractQueuedSynchronizer cleanup. - */ - private void cancelWait(Aux node) { - for (Aux a; (a = aux) != null && a.ex == null; ) { - for (Aux trail = null;;) { - Aux next = a.next; - if (a == node) { - if (trail != null) - trail.casNext(trail, next); - else if (casAux(a, next)) - return; // cannot be re-encountered - break; // restart - } else { - trail = a; - if ((a = next) == null) - return; - } - } - } - } - /* * Waits for signal, interrupt, timeout, or pool termination. * @@ -434,11 +416,11 @@ else if (((a = aux) == null || a.ex == null) && for (;;) { if ((s = status) < 0) break; - if (interrupts < 0) { + else if (interrupts < 0) { s = ABNORMAL; // interrupted and not done break; } - if (Thread.interrupted()) { + else if (Thread.interrupted()) { if (!ForkJoinPool.poolIsStopping(pool)) interrupts = interruptible ? -1 : 1; else { @@ -457,10 +439,24 @@ else if (deadline != 0L) { else LockSupport.park(); } - if (s >= 0) - cancelWait(node); - else - signalWaiters(); + node.thread = null; // help clean aux; raciness OK + clean: for (Aux a;;) { // remove node if still present + if ((a = aux) == null || a.ex != null) + break; + for (Aux prev = null;;) { + Aux next = a.next; + if (a == node) { + if (prev != null) + prev.casNext(prev, next); + else if (casAux(a, next)) + break clean; + break; // check for failed or stale CAS + } + prev = a; + if ((a = next) == null) + break clean; // not found + } + } LockSupport.setCurrentBlocker(null); if (interrupts > 0) Thread.currentThread().interrupt(); @@ -482,9 +478,8 @@ else if (deadline != 0L) { * @return ABNORMAL if interrupted, else status on exit */ private int awaitDone(boolean interruptible, long deadline) { - int s = 0; ForkJoinWorkerThread wt; ForkJoinPool p; ForkJoinPool.WorkQueue q; - Thread t; boolean internal; + Thread t; boolean internal; int s; if (internal = (t = Thread.currentThread()) instanceof ForkJoinWorkerThread) { p = (wt = (ForkJoinWorkerThread)t).pool; @@ -492,13 +487,12 @@ private int awaitDone(boolean interruptible, long deadline) { } else q = ForkJoinPool.externalQueue(p = ForkJoinPool.common); - if (p != null && - (s = ((this instanceof CountedCompleter) ? - p.helpComplete(this, q, internal) : - (this instanceof InterruptibleTask) && !internal ? status : - p.helpJoin(this, q, internal))) < 0) - return s; - return awaitDone(internal ? p : null, s, interruptible, deadline); + return (((s = (p == null) ? 0 : + ((this instanceof CountedCompleter) ? + p.helpComplete(this, q, internal) : + (this instanceof InterruptibleTask) && !internal ? status : + p.helpJoin(this, q, internal))) < 0)) ? s : + awaitDone(internal ? p : null, s, interruptible, deadline); } /** From 6a116f50b9b6322c8f6f7413fe4e71f90a3a685e Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sat, 3 Jun 2023 08:52:58 -0400 Subject: [PATCH 19/61] Rework versioning --- .../java/util/concurrent/ForkJoinPool.java | 1789 +++++++++-------- .../java/util/concurrent/ForkJoinTask.java | 84 +- .../util/concurrent/ForkJoinWorkerThread.java | 5 +- 3 files changed, 943 insertions(+), 935 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index a881e67478c7e..cc971b97b5eba 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -249,16 +249,11 @@ public class ForkJoinPool extends AbstractExecutorService { * Nardelli, PPoPP 2013 * (http://www.di.ens.fr/~zappa/readings/ppopp13.pdf) for an * analysis of memory ordering requirements in work-stealing - * algorithms similar to the one used here. We also use ordered, - * moded accesses and/or fences for other control, with modes - * reflecting the presence or absence of other contextual sync - * provided by atomic and/or volatile accesses. Some methods (or - * their primary loops) begin with an acquire fence or - * otherwise-unnecessary volatile read that amounts to an - * acquiring read of "this" to cover all fields (which is - * sometimes stronger than necessary, but less brittle). Some - * constructions are intentionally racy because they use read - * values as hints, not for correctness. + * algorithms similar to the one used here. We use per-operation + * ordered writes of various kinds for updates, but usually use + * explicit load fences for reads, to cover access of several + * fields of possibly several objects without further constraining + * read-by-read ordering. * * We also support a user mode in which local task processing is * in FIFO, not LIFO order, simply by using a local version of @@ -267,7 +262,7 @@ public class ForkJoinPool extends AbstractExecutorService { * increased contention among task producers and consumers. Also, * the same data structure (and class) is used for "submission * queues" (described below) holding externally submitted tasks, - * that differ only in that a lock (field "access"; see below) is + * that differ only in that a lock (using field "phase"; see below) is * required by external callers to push and pop tasks. * * Adding tasks then takes the form of a classic array push(task) @@ -278,8 +273,7 @@ public class ForkJoinPool extends AbstractExecutorService { * uses masking, not mod, for indexing a power-of-two-sized array, * enforces memory ordering, supports resizing, and possibly * signals waiting workers to start scanning (described below), - * which requires stronger forms of order accesses (using a form - * of lock release). + * which requires stronger forms of order accesses. * * The pop operation (always performed by owner) is of the form: * if ((task = getAndSet(q.array, (q.top-1) % length, null)) != null) @@ -303,7 +297,7 @@ public class ForkJoinPool extends AbstractExecutorService { * the following rechecks even if the CAS is not attempted. To * more easily distinguish among kinds of CAS failures, we use * the compareAndExchange version, and usually handle null - * returns (emptiness at that slot) separately from others. + * returns (indicating contention) separately from others. * * * q.base may change between reading and using its value to * index the slot. To avoid trying to use the wrong t, the @@ -323,12 +317,9 @@ public class ForkJoinPool extends AbstractExecutorService { * the poll operation. Stalls can be detected by observing that * q.base doesn't change on repeated reads of null t and when * no other alternatives apply, spin-wait for it to settle. To - * reduce producing these kinds of stalls by other stealers (as - * well as quiescence checks etc described below), we encourage - * timely writes to indices using otherwise unnecessarily - * strong writes (including releasing the access lock even in - * cases where it need not be held) or fences when memory - * ordering is not already constrained by context. + * reduce producing these kinds of stalls by other stealers, we + * encourage timely writes to indices using otherwise + * unnecessarily strong writes. * * * The CAS may fail, in which case we may want to retry unless * there is too much contention. One goal is to balance and @@ -346,15 +337,12 @@ public class ForkJoinPool extends AbstractExecutorService { * when deciding whether to try to poll or repoll after a * failure. Both top and base may move independently, and both * lag updates to the underlying array. To reduce memory - * contention, when possible, non-owners avoid reading the - * "top" index at all, and instead use array reads, including - * one-ahead reads to check whether to repoll, relying on the - * fact that a non-empty queue does not have two null slots in - * a row, except in cases (resizes and shifts) that can be - * detected with a secondary recheck. Otherwise, we read the - * access lock (see below) to conservatively (for submission - * queues), or more accurately (for worker queues) determine - * local quiescence. + * contention, non-owners avoid reading the "top" when + * possible, by using one-ahead reads to check whether to + * repoll, relying on the fact that a non-empty queue does not + * have two null slots in a row, except in cases (resizes and + * shifts) that can be detected with a secondary recheck that + * is less likely to conflict with owner writes. * * The poll operations in q.poll(), scan(), helpJoin(), and * elsewhere differ with respect to whether other queues are @@ -382,13 +370,11 @@ public class ForkJoinPool extends AbstractExecutorService { * like workers except that they are restricted to executing local * tasks that they submitted (or when known, subtasks thereof). * Insertion of tasks in shared mode requires a lock. We use only - * a simple spinlock because submitters encountering a busy queue - * move to a different position to use or create other queues. - * They (spin) block when registering new queues, or indirectly - * elsewhere (by revisiting later. The lock needed for external - * queues is generalized (as field "access") for operations on - * internal queues that require a fully-fenced write to deal with - * Dekker-like signalling constructs described below. + * a simple spinlock (as one role of filed "phase") because + * submitters encountering a busy queue move to a different + * position to use or create other queues. They (spin) block when + * registering new queues, or indirectly elsewhere (by revisiting + * later. * * Management * ========== @@ -414,40 +400,46 @@ public class ForkJoinPool extends AbstractExecutorService { * restrict maximum parallelism to (1<<15)-1 (which is far in * excess of normal operating range) to allow ids, counts, and * their negations (used for thresholding) to fit into 16bit - * subfields. Field "parallelism" holds the target parallelism - * (normally corresponding to pool size). It is needed (nearly) - * only in methods updating ctl, so is packed nearby. As of the - * current release, users can dynamically reset target - * parallelism, which is read once per update, so only slowly has - * an effect in creating threads or letting them time out and - * terminate when idle. - * - * Field "runState" holds lifetime status, atomically and - * monotonically setting SHUTDOWN, STOP, and TERMINATED as low - * bits, and otherwise acts as a 29-bit spin-based sequence lock - * for queue array updates (incrementing by LOCKED units per - * operation, rather than the usual even/odd scheme). The seqLock - * properties detect changes and conditionally upgrade to - * coordinate with runState updates. It is typically held for less - * than a dozen instructions unless the queue array is being - * resized, during which contention is rare. To be conservative, - * acquireRunStateLock is implemented as a spin/sleep loop. Here - * and elsewhere spin constants are short enough to apply even on - * systems with few available processors. In addition to checking - * pool status, reads of runState serve as acquire fences before - * reading other non-volatile fields. + * subfields. + * + * Field "runState" and per-WorkQueue field "phase" play similar + * roles, as lockable, versioned counters. Field runState also + * includes monotonic event bits (SHUTDOWN, STOP, and TERMINATED). + * The version tags enable detection of state changes (by + * comparing two reads) modulo bit wraparound. The bit range in + * each case suffices for purposes of determining quiescence, + * termination, avoiding ABA-like errors, and signal control, most + * of which are ultimately based on at most 15bit ranges (due to + * 32767 max total workers). RunState updates do not need to be + * atomic with respect to ctl updates, but because they are not, + * some care is required to avoid stalls. The seqLock properties + * detect changes and conditionally upgrade to coordinate with + * updates. It is typically held for less than a dozen + * instructions unless the queue array is being resized, during + * which contention is rare. To be conservative, lockRunState is + * implemented as a spin/sleep loop. Here and elsewhere spin + * constants are short enough to apply even on systems with few + * available processors. In addition to checking pool status, + * reads of runState sometimes serve as acquire fences before + * reading other fields. + * + * Field "parallelism" holds the target parallelism (normally + * corresponding to pool size). Users can dynamically reset target + * parallelism, but is only accessed when signalling or awaiting + * work, so only slowly has an effect in creating threads or + * letting them time out and terminate when idle. * * Array "queues" holds references to WorkQueues. It is updated - * (only during worker creation and termination) under the - * runState lock. It is otherwise concurrently readable but reads - * for use in scans (see below) are always prefaced by a volatile - * read of runState (or equivalent constructions), ensuring that - * its state is current at the point it is used (which is all we - * require). To simplify index-based operations, the array size is - * always a power of two, and all readers must tolerate null - * slots. Worker queues are at odd indices. Worker ids masked - * with SMASK match their index. Shared (submission) queues are at - * even indices. Grouping them together in this way simplifies and + * (only during worker creation and termination) under the runState + * lock. It is otherwise concurrently readable but reads for use + * in scans (see below) are always prefaced by a volatile read of + * runState (or equivalent constructions), ensuring that its state is + * current at the point it is used (which is all we require). To + * simplify index-based operations, the array size is always a + * power of two, and all readers must tolerate null slots. Worker + * queues are at odd indices. Worker phase ids masked with SMASK + * match their index. Shared (submission) queues are at even + * indices. Grouping them together in this way simplifies and * speeds up task scanning. * * All worker thread creation is on-demand, triggered by task @@ -463,10 +455,8 @@ public class ForkJoinPool extends AbstractExecutorService { * indices, not references. Operations on queues obtained from * these indices remain OK (with at most some unnecessary extra * work) even if an underlying worker failed and was replaced by - * another at the same index. There are two contexts requiring - * extra precautions: In checkQuiescence, the SeqLock is used to - * recheck upon queue array updates. And during termination, - * worker queue array updates are disabled. + * another at the same index. During termination, worker queue + * array updates are disabled. * * Queuing Idle Workers. Unlike HPC work-stealing frameworks, we * cannot let workers spin indefinitely scanning for tasks when @@ -489,8 +479,8 @@ public class ForkJoinPool extends AbstractExecutorService { * and/or running tasks (we cannot accurately determine * which). Unreleased ("available") workers are recorded in the * ctl stack. These workers are made eligible for signalling by - * enqueuing in ctl (see method awaitWork). The "queue" is a form - * of Treiber stack. This is ideal for activating threads in + * enqueuing in ctl (see method runWorker). This "queue" is a + * form of Treiber stack. This is ideal for activating threads in * most-recently used order, and improves performance and * locality, outweighing the disadvantages of being prone to * contention and inability to release a worker unless it is @@ -512,16 +502,14 @@ public class ForkJoinPool extends AbstractExecutorService { * workers. If exceptional, the exception is propagated, generally * to some external caller. * - * WorkQueue field "phase" is used by both workers and the pool to - * manage and track whether a worker is unsignalled (possibly - * blocked waiting for a signal), conveniently using the sign bit - * to check. When a worker is enqueued its phase field is set - * negative. Note that phase field updates lag queue CAS releases; - * seeing a negative phase does not guarantee that the worker is - * available (and so is never checked in this way). When queued, - * the lower 16 bits of its phase must hold its pool index. So we - * place the index there upon initialization and never modify - * these bits. + * WorkQueue field "phase" encodes the queue array id in lower + * bits, and otherwise acts similarly to the pool runState field: + * The "IDLE" bit is clear while active (either a released worker + * or a locked external queue), with other bits serving as a + * version counter to distinguish changes across multiple reads. + * Note that phase field updates lag queue CAS releases; seeing a + * non-idle phase does not guarantee that the worker is available + * (and so is never checked in this way). * * The ctl field also serves as the basis for memory * synchronization surrounding activation. This uses a more @@ -542,12 +530,11 @@ public class ForkJoinPool extends AbstractExecutorService { * * If computations are purely tree structured, it suffices for * every worker to activate another when it pushes a task into * an empty queue, resulting in O(log(#threads)) steps to full - * activation. As discussed above, "empty" must be - * conservatively approximated, sometimes resulting in - * unnecessary signals. Also, to reduce resource usages in - * some cases, at the expense of slower startup in others, - * activation of an idle thread is preferred over creating a - * new one, here and elsewhere. + * activation. Emptiness must be conservatively approximated, + * sometimes resulting in unnecessary signals. Also, to reduce + * resource usages in some cases, at the expense of slower + * startup in others, activation of an idle thread is preferred + * over creating a new one, here and elsewhere. * * * If instead, tasks come in serially from only a single * producer, each worker taking its first (since the last @@ -565,8 +552,7 @@ public class ForkJoinPool extends AbstractExecutorService { * active workers continue scanning after running tasks without * the need to be signalled (which is one reason work stealing * is often faster than alternatives), so additional workers - * aren't needed. But there is no efficient way to detect this - * without adding expensive and unscalable centralized control. + * aren't needed. But there is no efficient way to detect this. * * * For rapidly branching tasks that require full pool resources, * oversignalling is OK, because signalWork will soon have no @@ -575,43 +561,40 @@ public class ForkJoinPool extends AbstractExecutorService { * noticeable slowdowns due to contention and resource * wastage. All remedies are intrinsically heuristic. We use a * strategy that works well in most cases: We track "sources" - * (queue ids offset by constant 1 to avoid zero) of non-empty - * (usually polled) queues while scanning. These are maintained - * in the "source" field of WorkQueues for use in method - * helpJoin and elsewhere (see below). We also maintain them as - * arguments/results of top-level polls (argument "srcs" in - * method scan) as an encoded sliding window of the last two - * sources, and stop signalling when the last two were from the - * same source. And similarly not retry when two CAS failures - * were from the same source. These mechanisms may result in - * transiently too few workers, but once workers poll from a new - * source, they rapidly reactivate others. (An implementation / - * encoding quirk: Because of the offset, unnecessary signals - * for the maximum-numbered 32767th worker, which is unlikely to - * ever be produced, are not suppressed.) + * (queue ids) of non-empty (usually polled) queues while + * scanning. These are maintained in the "source" field of + * WorkQueues for use in method helpJoin and elsewhere (see + * below). We also maintain them as arguments/results of + * top-level polls (argument "window" in method scan) as an + * encoded sliding window of the last two sources, and stop + * signalling when the last two were from the same source. And + * similarly not retry when two CAS failures were from the same + * source. These mechanisms may result in transiently too few + * workers, but once workers poll from a new source, they + * rapidly reactivate others. * * * Despite these, signal contention and overhead effects still * occur during ramp-up and ramp-down of small computations. * * Scanning. Method scan performs top-level scanning for (and * execution of) tasks by polling a pseudo-random permutation of - * the array (by starting at a random index, and using a constant - * cyclically exhaustive stride.) This maintains top-level - * randomized fairness at the cost of poor locaility, but enables - * faster less-fair techniques once a top-level task is found. It - * uses the same basic polling method as WorkQueue.poll(), but - * restarts with a different permutation on each invocation. The - * pseudorandom generator need not have high-quality statistical - * properties in the long term. We use Marsaglia XorShifts, seeded - * with the Weyl sequence from ThreadLocalRandom probes, which are - * cheap and suffice. Scans do not otherwise explicitly take into - * account core affinities, loads, cache localities, etc, However, - * they do exploit temporal locality (which usually approximates - * these) by preferring to re-poll from the same queue (using - * method tryPoll()) after a successful poll before trying others - * (see method topLevelExec), which also reduces bookkeeping and - * scanning overhead. This also reduces fairness, which is - * partially counteracted by giving up on contention. + * the array (by starting at a given index, and using a constant + * cyclically exhaustive stride.) It uses the same basic polling + * method as WorkQueue.poll(), but restarts with a different + * permutation on each invocation. The pseudorandom generator + * need not have high-quality statistical properties in the long + * term. We use Marsaglia XorShifts, seeded with the Weyl sequence + * from ThreadLocalRandom probes, which are cheap and + * suffice. Scans do not otherwise explicitly take into account + * core affinities, loads, cache localities, etc, However, they do + * exploit temporal locality (which usually approximates these) by + * preferring to re-poll from the same queue (either in method + * tryPoll() or scan) after a successful poll before trying others + * (see method topLevelExec), which also reduces bookkeeping, + * cache traffic, and scanning overhead. But it also reduces + * fairness, which is partially counteracted by giving up on + * contention, as well as resetting origins to random values upon + * any runState change. * * Deactivation. When method scan indicates that no tasks are * found by a worker, it tries to deactivate (in runWorker). Note @@ -636,53 +619,49 @@ public class ForkJoinPool extends AbstractExecutorService { * don't find any, without being sure that none are available. * However, some required functionality relies on consensus about * quiescence (also termination, discussed below). The count - * fields in ctl allow efficient (modulo contention) and accurate - * discovery of states in which all workers are idle. However, - * because external (asynchronous) submitters are not part of this - * vote, these mechanisms themselves do not guarantee that the - * pool is in a quiescent state with respect to methods - * isQuiescent, shutdown (which begins termination when - * quiescent), helpQuiesce, and indirectly others including - * tryCompensate. Method checkQuiescence() is used in all of these - * contexts. It provides checks that all workers are idle and - * there are no submissions that they could poll if they were not - * idle, retrying on inconsistent reads of queues and using the - * SeqLock to retry on queue array updates. (It also reports - * quiescence if the pool is terminating.) A true report means - * only that there was a moment at which quiescence held; further - * actions based on this may be racy except when using seq lock - * upgrade to trigger quiescent termination. False negatives are - * inevitable (for example when queues indices lag updates, as - * described above), which is accommodated when (tentatively) idle - * by scanning for work etc, and then re-invoking. This includes - * cases in which the final unparked thread (in awaitWork) uses - * checkQuiescence() to check for tasks that could have been added - * during a race window that would not be accompanied by a signal, - * in which case re-activating itself (or any other worker) to - * rescan. Method helpQuiesce acts similarly but cannot rely on - * ctl counts to determine that all workers are inactive because - * the caller and any others executing helpQuiesce are not - * included in counts. + * fields in ctl allow accurate discovery of states in which all + * workers are idle. However, because external (asynchronous) + * submitters are not part of this vote, these mechanisms + * themselves do not guarantee that the pool is in a quiescent + * state with respect to methods isQuiescent, shutdown (which + * begins termination when quiescent), helpQuiesce, and indirectly + * others including tryCompensate. Method isQuiescent() is used in + * all of these contexts. It provides checks that all workers are + * idle and there are no submissions that they could poll if they + * were not idle, retrying on inconsistent reads of queues and + * using the runState seqLock to retry on queue array updates. (It + * also reports quiescence if the pool is terminating.) A true + * report means only that there was a moment at which quiescence + * held. False negatives are inevitable (for example when queues + * indices lag updates, as described above), which is accommodated + * when (tentatively) idle by scanning for work etc, and then + * re-invoking. This includes cases in which the final unparked + * thread (in awaitWork) uses isQuiescent() to check for tasks + * that could have been added during a race window that would not + * be accompanied by a signal, in which case re-activating itself + * (or any other worker) to rescan. Method helpQuiesce acts + * similarly but cannot rely on ctl counts to determine that all + * workers are inactive because the caller and any others + * executing helpQuiesce are not included in counts. * * The quiescence check in awaitWork also serves as a safeguard * when all other workers gave up prematurely and inactivated (due - * to excessive contention or array resizing) which will cause - * this thread to rescan and wake up others. + * to excessive contention) which will cause this thread to rescan + * and wake up others. * * Termination. A call to shutdownNow invokes tryTerminate to - * atomically set a mode bit, performed on the runState lock to - * avoid inconsistencies. However, the process of termination is - * intrinsically non-atomic. The calling thread, as well as other - * workers thereafter terminating help cancel queued tasks and - * interrupt other workers. These actions race with unterminated - * workers. By default, workers check for termination only when - * accessing pool state (usually the "queues" array). This may - * take a while but suffices for structured computational tasks. - * But not necessarily for others. Class InterruptibleTask (see - * below) further arranges runState checks before executing task - * bodies, and ensures interrupts while terminating. Even so, - * there are no guarantees after an abrupt shutdown that remaining - * tasks complete normally or exceptionally or are cancelled. + * atomically set a runState mode bit. However, the process of + * termination is intrinsically non-atomic. The calling thread, as + * well as other workers thereafter terminating help cancel queued + * tasks and interrupt other workers. These actions race with + * unterminated workers. By default, workers check for + * termination only when accessing pool state. This may take a + * while but suffices for structured computational tasks. But not + * necessarily for others. Class InterruptibleTask (see below) + * further arranges runState checks before executing task bodies, and + * ensures interrupts while terminating. Even so, there are no + * guarantees after an abrupt shutdown that remaining tasks + * complete normally or exceptionally or are cancelled. * Termination may fail to complete if running tasks repeatedly * ignore both task status and interrupts and/or produce more * tasks after others that could cancel them have exited. @@ -704,9 +683,9 @@ public class ForkJoinPool extends AbstractExecutorService { * task) avoid context switching or adding worker threads when one * task would otherwise be blocked waiting for completion of * another, basically, just by running that task or one of its - * subtasks if not already taken. As described below, these - * mechanics are disabled for InterruptibleTasks, that guarantee - * that callers do not executed submitted tasks. + * subtasks if not already taken. These mechanics are disabled for + * InterruptibleTasks, that guarantee that callers do not executed + * submitted tasks. * * The basic structure of joining is an extended spin/block scheme * in which workers check for task completions status between @@ -734,19 +713,19 @@ public class ForkJoinPool extends AbstractExecutorService { * * The two methods for finding and executing subtasks vary in * details. The algorithm in helpJoin entails a form of "linear - * helping". Each worker records (in field "source") an offset - * index to the queue from which it last stole a - * task. (Implementation note: to distinguish lack of source from - * index 0 or other sentinels, source fields are offset by - * constant 1.) The scan in method helpJoin uses these markers - * to try to find a worker to help (i.e., steal back a task from - * and execute it) that could makje progress toward completion of - * the actively joined task. Thus, the joiner executes a task - * that would be on its own local deque if the to-be-joined task - * had not been stolen. This is a conservative variant of the - * approach described in Wagner & Calder "Leapfrogging: a portable - * technique for implementing efficient futures" SIGPLAN Notices, - * 1993 (http://portal.acm.org/citation.cfm?id=155354). It differs + * helping". Each worker records (in field "source") the index of + * the internal queue from which it last stole a task. (Note: + * because chains cannot include even-numbered external queues, + * they are ignored, and 0 is an OK default.) The scan in method + * helpJoin uses these markers to try to find a worker to help + * (i.e., steal back a task from and execute it) that could makje + * progress toward completion of the actively joined task. Thus, + * the joiner executes a task that would be on its own local deque + * if the to-be-joined task had not been stolen. This is a + * conservative variant of the approach described in Wagner & + * Calder "Leapfrogging: a portable technique for implementing + * efficient futures" SIGPLAN Notices, 1993 + * (http://portal.acm.org/citation.cfm?id=155354). It differs * mainly in that we only record queues, not full dependency * links. This requires a linear scan of the queues to locate * stealers, but isolates cost to when it is needed, rather than @@ -815,9 +794,10 @@ public class ForkJoinPool extends AbstractExecutorService { * some System property parsing and with security processing that * takes far longer than the actual construction when * SecurityManagers are used or properties are set. The common - * pool is distinguished internally by having a null - * workerNamePrefix, along with PRESET_SIZE set if parallelism was - * configured by system property. + * pool is distinguished by having a null workerNamePrefix (which + * is an odd convention, but avoids the need to decode status in + * factory classes). It also has PRESET_SIZE config set if + * parallelism was configured by system property. * * When external threads use the common pool, they can perform * subtask processing (see helpComplete and related methods) upon @@ -891,31 +871,31 @@ public class ForkJoinPool extends AbstractExecutorService { * ================ * * Performance is very sensitive to placement of instances of - * ForkJoinPool and WorkQueues and their queue arrays, as well the - * placement of their fields. Caches misses and contention due to - * false-sharing have been observed to slow down some programs by - * more than a factor of four. There is no perfect solution, in - * part because isolating more fields also generates more cache - * misses in more common cases (because some fields snd slots are - * usually read at the same time), and the main means of placing - * memory, the @Contended annotation provides only rough control - * (for good reason). We isolate the ForkJoinPool.ctl field as - * well the set of WorkQueue fields that otherwise cause the most - * false-sharing misses with respect to other fields. Also, - * ForkJoinPool fields are ordered such that fields less prone to - * contention effects are first, offsetting those that otherwise - * would be, while also reducing total footprint vs using - * multiple @Contended regions, which tends to slow down - * less-contended applications. These arrangements mainly reduce - * cache traffic by scanners, which speeds up finding tasks to - * run. Initial sizing and resizing of WorkQueue arrays is an - * even more delicate tradeoff because the best strategy may vary - * across garbage collectors. Small arrays are better for locality - * and reduce GC scan time, but large arrays reduce both direct - * false-sharing and indirect cases due to GC bookkeeping + * ForkJoinPool and WorkQueues and their queue arrays, as well as + * the placement of their fields. Caches misses and contention due + * to false-sharing have been observed to slow down some programs + * by more than a factor of four. Effects may vary across initial + * memory configuarations, applications, and different garbage + * collectors and GC settings, so there is no perfect solution. + * Too much isolation may generate more cache misses in common + * cases (because some fields snd slots are usually read at the + * same time). The @Contended annotation provides only rough + * control (for good reason). Similarly for relying on fields + * being placed in size-sorted declaration order. + * + * For class ForkJoinPool, it is usually more effective to order + * fields such that the most commonly accessed fields are unlikely + * to share cache lines with adjacent objects under JVM layout + * rules. For class WorkQueue, an embedded @Contended region + * segregates fields most heavily updated by owners from those + * most commonly read by stealers or other management. Initial + * sizing and resizing of WorkQueue arrays is an even more + * delicate tradeoff because the best strategy systematically + * varies across garbage collectors. Small arrays are better for + * locality and reduce GC scan time, but large arrays reduce both + * direct false-sharing and indirect cases due to GC bookkeeping * (cardmarks etc), and reduce the number of resizes, which are - * not especially fast because they require atomic transfers, and - * may cause other scanning workers to stall or give up. + * not especially fast because they require atomic transfers. * Currently, arrays are initialized to be fairly small but early * resizes rapidly increase size by more than a factor of two * until very large. (Maintenance note: any changes in fields, @@ -933,7 +913,9 @@ public class ForkJoinPool extends AbstractExecutorService { * other jdk components that require early parallelism. This can * be awkward and ugly, but also reflects the need to control * outcomes across the unusual cases that arise in very racy code - * with very few invariants. All fields are read into locals + * with very few invariants. All atomic task slot updates use + * Unsafe operations requiring offset positions, not indices, as + * computed by method slotOffset. All fields are read into locals * before use, and null-checked if they are references, even if * they can never be null under current usages. Usually, * computations (held in local variables) are defined as soon as @@ -981,8 +963,9 @@ public class ForkJoinPool extends AbstractExecutorService { * * New abstract class ForkJoinTask.InterruptibleTask ensures * handling of tasks submitted under the ExecutorService * API are consistent with specs. - * * Method checkQuiescence() replaces previous quiescence-related - * checks, relying on a sequence lock instead of ReentrantLock. + * * Method isQuiescent() replaces previous quiescence-related + * checks, relying on versioning and sequence locking instead + * of ReentrantLock. * * Termination processing now ensures that internal data * structures are maintained consistently enough while stopping * to interrupt all workers and cancel all tasks. It also uses a @@ -1009,7 +992,7 @@ public class ForkJoinPool extends AbstractExecutorService { * The default value for common pool maxSpares. Overridable using * the "java.util.concurrent.ForkJoinPool.common.maximumSpares" * system property. The default value is far in excess of normal - * requirements, but also far short of MAX_CAP and typical OS + * requirements, but also far short of maximum capacity and typical OS * thread limits, so allows JVMs to catch misuse/abuse before * running out of resources needed to do so. */ @@ -1021,30 +1004,39 @@ public class ForkJoinPool extends AbstractExecutorService { */ static final int INITIAL_QUEUE_CAPACITY = 1 << 6; - // Bounds - static final int SWIDTH = 16; // width of short - static final int SMASK = 0xffff; // short bits - static final int SEMASK = 0xfffe; // short even bits - static final int MAX_CAP = 0x7fff; // max worker index + // conversions among short, int, long + static final int SWIDTH = 16; // width of short + static final int SMASK = 0xffff; // (unsigned) short bits + static final long LMASK = 0xffffffffL; // lower 32 bits of long + static final long UMASK = ~LMASK; // upper 32 bits + + // masks and sentinels for queue indices + static final int MAX_CAP = 0x7fff; // max # workers + static final int EXTERNAL_ID_MASK = 0x3ffe; // max external queue id + static final int INVALID_ID = 0x4000; // unused external queue id // pool.runState bits - static final int STOP = 1 << 0; - static final int SHUTDOWN = 1 << 1; - static final int TERMINATED = 1 << 2; - static final int LOCKED = 1 << 3; // lowest seqlock bit - - // spin/sleep limits for runState locking and elsewhere; powers of 2 - static final int SPIN_WAITS = 96; // calls to onSpinWait - static final int MIN_SLEEP = 1 << 10; // approx 1 usec as nanos - static final int MAX_SLEEP = 1 << 20; // approx 1 sec as nanos - - // {pool, workQueue} bits and sentinels - static final int FIFO = 1 << 16; // fifo queue or access mode - static final int REGISTERED = 1 << 17; // true when alive - static final int CLEAR_TLS = 1 << 18; // set for Innocuous workers - static final int TRIMMED = 1 << 19; // timed out while idle - static final int PRESET_SIZE = 1 << 21; // size was set by property - static final int UNCOMPENSATE = 1 << 16; // tryCompensate return + static final int STOP = 1 << 0; // terminating + static final int SHUTDOWN = 1 << 1; // terminate when quiescent + static final int TERMINATED = 1 << 2; // only set if STOP also set + static final int RS_LOCK = 1 << 3; // lowest seqlock bit + static final int RS_EPOCH = 1 << 4; // version counter tag + + // spin/sleep limits for runState locking and elsewhere + static final int SPIN_WAITS = 1 << 7; // max calls to onSpinWait + static final int MIN_SLEEP = 1 << 10; // approx 1 usec as nanos + static final int MAX_SLEEP = 1 << 20; // approx 1 sec as nanos + + // {pool, workQueue} config bits + static final int FIFO = 1 << 0; // fifo queue or access mode + static final int CLEAR_TLS = 1 << 1; // set for Innocuous workers + static final int PRESET_SIZE = 1 << 2; // size was set by property + + // others + static final int TRIMMED = 1 << 31; // timed out while idle + static final int UNCOMPENSATE = 1 << 16; // tryCompensate return + static final int IDLE = 1 << 16; // phase seqlock/version count + static final long RESCAN = 1L << 63; // window retry indicator /* * Bits and masks for ctl and bounds are packed with 4 16 bit subfields: @@ -1064,25 +1056,34 @@ public class ForkJoinPool extends AbstractExecutorService { * Other updates of multiple subfields require CAS. */ - // Lower and upper word masks - static final long SP_MASK = 0xffffffffL; - static final long UC_MASK = ~SP_MASK; // Release counts - static final int RC_SHIFT = 48; - static final long RC_UNIT = 0x0001L << RC_SHIFT; - static final long RC_MASK = 0xffffL << RC_SHIFT; + static final int RC_SHIFT = 48; + static final long RC_UNIT = 0x0001L << RC_SHIFT; + static final long RC_MASK = 0xffffL << RC_SHIFT; // Total counts - static final int TC_SHIFT = 32; - static final long TC_UNIT = 0x0001L << TC_SHIFT; - static final long TC_MASK = 0xffffL << TC_SHIFT; + static final int TC_SHIFT = 32; + static final long TC_UNIT = 0x0001L << TC_SHIFT; + static final long TC_MASK = 0xffffL << TC_SHIFT; - // sp bits (also used for source field) - static final int SS_SEQ = 1 << 16; // version count - static final int INACTIVE = 1 << 31; // phase/source bit if idle - static final int ACTIVE = ~INACTIVE; // initial value for srcs + /* + * All atomic operations on task arrays (queues) use Unsafe + * operations that take array offsets versus indices, based on + * array base and shift constants established during static + * initialization. + */ + static final long ABASE; + static final int ASHIFT; // Static utilities + /** + * Returns the array offset corresponding to the given index for + * Unsafe task queue operations + */ + static long slotOffset(int index) { + return ((long)index << ASHIFT) + ABASE; + } + /** * If there is a security manager, makes sure caller has * permission to modify threads. @@ -1201,70 +1202,77 @@ public ForkJoinWorkerThread run() { * submission. See above for descriptions and algorithms. */ static final class WorkQueue { - int stackPred; // pool stack (ctl) predecessor link - int config; // index, mode, REGISTERED bits - int base; // index of next slot for poll - volatile Thread parker; // set to owner while parked - final ForkJoinWorkerThread owner; // owning thread or null if shared + // fields declared in order of their likely layout on most VMs + volatile ForkJoinWorkerThread owner; // null if shared or terminated + volatile Thread parker; // set when parking in awaitWork ForkJoinTask[] array; // the queued tasks; power of 2 size + int base; // index of next slot for poll + final int config; // mode bits // fields otherwise causing more unnecessary false-sharing cache misses @jdk.internal.vm.annotation.Contended("w") int top; // index of next slot for push @jdk.internal.vm.annotation.Contended("w") - volatile int access; // values 0, 1 (locked) + volatile int phase; // versioned active status @jdk.internal.vm.annotation.Contended("w") - volatile int phase; // versioned, negative if inactive + int stackPred; // pool stack (ctl) predecessor link @jdk.internal.vm.annotation.Contended("w") - volatile int source; // source queue id + 1 + volatile int source; // source queue id (or TRIMMED if trimmed) @jdk.internal.vm.annotation.Contended("w") int nsteals; // number of steals from other queues // Support for atomic operations private static final Unsafe U; - private static final long ACCESS; - private static final long ABASE; - private static final int ASHIFT; + private static final long PHASE; + private static final long BASE; + private static final long TOP; + private static final long SOURCE; + private static final long ARRAY; - static ForkJoinTask getAndClearSlot(ForkJoinTask[] a, int i) { - return (ForkJoinTask) - U.getAndSetReference(a, ((long)i << ASHIFT) + ABASE, null); + final void updateBase(int v) { + U.putIntVolatile(this, BASE, v); + } + final void updateTop(int v) { + U.putIntOpaque(this, TOP, v); + } + final void forgetSource() { + U.putIntOpaque(this, SOURCE, 0); } - static ForkJoinTask cmpExSlotToNull(ForkJoinTask[] a, int i, - ForkJoinTask c) { - return (ForkJoinTask) - U.compareAndExchangeReference(a, ((long)i << ASHIFT) + ABASE, - c, null); + final void updateArray(ForkJoinTask[] a) { + U.putReferenceVolatile(this, ARRAY, a); } - final int getAndSetAccess(int v) { - return U.getAndSetInt(this, ACCESS, v); + final void unlockPhase() { + U.getAndAddInt(this, PHASE, IDLE); } - final void releaseAccess() { // unlock - U.putIntRelease(this, ACCESS, 0); + final boolean tryLockPhase() { // seqlock acquire + int p; + return (((p = phase) & IDLE) != 0 && + U.compareAndSetInt(this, PHASE, p, p + IDLE)); } /** * Constructor. For internal queues, most fields are initialized * upon thread start in pool.registerWorker. */ - WorkQueue(ForkJoinWorkerThread owner, int config) { + WorkQueue(ForkJoinWorkerThread owner, int id, int cfg, + boolean clearThreadLocals) { + top = base = 1; + this.config = (clearThreadLocals) ? cfg | CLEAR_TLS : cfg; + this.phase = id; this.owner = owner; - this.config = config; - base = top = 1; } /** * Returns an exportable index (used by ForkJoinWorkerThread). - */ - final int getPoolIndex() { - return (config & 0xffff) >>> 1; // ignore odd/even tag bit + */final int getPoolIndex() { + return (phase & 0xffff) >>> 1; // ignore odd/even tag bit } /** * Returns the approximate number of tasks in the queue. */ final int queueSize() { - int unused = access; // for ordering effect + int unused = phase; // for ordering effect return Math.max(top - base, 0); // ignore transient negative } @@ -1272,26 +1280,27 @@ final int queueSize() { * Pushes a task. Called only by owner or if already locked * * @param task the task. Caller must ensure non-null. - * @param pool the pool. Must be non-null unless terminating. - * @param signal true if signal if queue may have been or appeared empty + * @param pool the pool to signal if was previously empty, else null + * @param internal if caller owns this queue * @throws RejectedExecutionException if array cannot be resized */ final void push(ForkJoinTask task, ForkJoinPool pool, - boolean signal) { - ForkJoinTask[] a = array; - int b = base, s = top++, cap, m; - if (a != null && (cap = a.length) > 0) { - if ((m = (cap - 1)) == s - b) - growAndPush(task, a, s); // may have appeared empty + boolean internal) { + int s = top++, cap, m; ForkJoinTask[] a; + if ((a = array) != null && (cap = a.length) > 0) { + if ((m = cap - 1) == s - base) + growAndPush(task, a, s, internal); else { - int prev = m & (s - 1); - a[m & s] = task; - getAndSetAccess(0); // for memory effects if internal - if (a[prev] != null) - signal = false; // not empty + long pos = slotOffset(m & s); + if (internal) + U.getAndSetReference(a, pos, task); // fully fenced + else { + U.putReference(a, pos, task); // inside lock + unlockPhase(); + } + if (a[m & (s - 1)] == null && pool != null) + pool.signalWork(); } - if (signal && pool != null) - pool.signalWork(); } } @@ -1301,7 +1310,7 @@ final void push(ForkJoinTask task, ForkJoinPool pool, * @param s the old top value */ private void growAndPush(ForkJoinTask task, ForkJoinTask[] a, - int s) { + int s, boolean internal) { int cap; // rapidly grow until large if (a != null && (cap = a.length) > 0) { int newCap = (cap < 1 << 24) ? cap << 2 : cap << 1; @@ -1316,16 +1325,18 @@ private void growAndPush(ForkJoinTask task, ForkJoinTask[] a, do { // poll old, push to new; exit if lose to pollers newArray[k & newMask] = task; } while (--k != b && - (task = getAndClearSlot(a, k & m)) != null); - U.storeFence(); - array = newArray; - access = 0; + (task = (ForkJoinTask)U.getAndSetReference( + a, slotOffset(k & m), null)) != null); + updateArray(newArray); + if (!internal) + unlockPhase(); return; } } } top = s; // revert on failure - access = 0; + if (!internal) + unlockPhase(); throw new RejectedExecutionException("Queue capacity exceeded"); } @@ -1335,28 +1346,28 @@ private void growAndPush(ForkJoinTask task, ForkJoinTask[] a, * @param fifo nonzero if FIFO mode */ private ForkJoinTask nextLocalTask(int fifo) { - ForkJoinTask[] a = array; - int b = base, p = top, s = p - 1, nb, cap; ForkJoinTask t = null; + ForkJoinTask[] a = array; + int b = base, p = top, s = p - 1, cap; if (p - b > 0 && a != null && (cap = a.length) > 0) { + int m = cap - 1, nb; do { if (fifo == 0 || (nb = b + 1) == p) { - if ((t = getAndClearSlot(a, (cap - 1) & s)) != null) - top = s; - break; // lost race for only task + if ((t = (ForkJoinTask)U.getAndSetReference( + a, slotOffset(m & s), null)) != null) + updateTop(s); // else lost race for only task + break; } - else if ((t = getAndClearSlot(a, (cap - 1) & b)) != null) { - base = nb; + if ((t = (ForkJoinTask)U.getAndSetReference( + a, slotOffset(m & b), null)) != null) { + updateBase(nb); break; } - else { - while (b == (b = base)) { - U.loadFence(); - Thread.onSpinWait(); // spin to reduce memory traffic - } + while (b == (b = base)) { + U.loadFence(); + Thread.onSpinWait(); // spin to reduce memory traffic } } while (p - b > 0); - U.storeStoreFence(); // for timely index updates } return t; } @@ -1371,30 +1382,25 @@ final ForkJoinTask nextLocalTask() { /** * Pops the given task only if it is at the current top. + * @param task the task. Caller must ensure non-null. + * @param internal if caller owns this queue */ final boolean tryUnpush(ForkJoinTask task, boolean internal) { + boolean taken; ForkJoinTask[] a = array; - int b = base, p = top, s, cap, k; - if (task == null || b == p || a == null || (cap = a.length) <= 0 || - a[k = (cap - 1) & (s = p - 1)] != task) - return false; - if (!internal) { - if (getAndSetAccess(1) != 0) - return false; - else if (top != p || cmpExSlotToNull(a, k, task) != task) { - releaseAccess(); - return false; - } - top = s; - releaseAccess(); - } - else { - if (getAndClearSlot(a, k) == null) - return false; - top = s; - U.storeFence(); + int p = top, s = p - 1, cap, k; + if (a != null && (cap = a.length) > 0 && + a[k = (cap - 1) & s] == task && + (internal || tryLockPhase())) { + if (taken = (top == p && U.compareAndSetReference( + a, slotOffset(k), task, null))) + updateTop(s); + if (!internal) + unlockPhase(); } - return true; + else + taken = false; + return taken; } /** @@ -1423,55 +1429,47 @@ final ForkJoinTask peek() { * @param pool if nonnull, pool to signal if more tasks exist */ final ForkJoinTask poll(ForkJoinPool pool) { - for (int b = base;;) { - ForkJoinTask[] a; int cap; - if ((a = array) == null || (cap = a.length) <= 0) - break; // currently impossible - int nb = b + 1, nk = (cap - 1) & nb, k; - ForkJoinTask t = a[k = (cap - 1) & b]; + for (;;) { + ForkJoinTask[] a = array; + int b = base, cap, k; + if (a == null || (cap = a.length) <= 0) + break; + ForkJoinTask t = a[k = b & (cap - 1)]; U.loadFence(); - if (b == (b = base)) { + if (base == b) { + Object o; + int nb = b + 1, nk = nb & (cap - 1); if (t == null) - t = a[k]; // reread or CAS - else if (t == (t = cmpExSlotToNull(a, k, t))) { - base = nb; - U.storeFence(); + o = a[k]; + else if (t == (o = U.compareAndExchangeReference( + a, slotOffset(k), t, null))) { + updateBase(nb); if (pool != null && a[nk] != null) pool.signalWork(); // propagate return t; } - if (t == null && access == 0 && top - b <= 0) + if (o == null && a[nk] == null && array == a && + (phase & (IDLE | 1)) != 0 && top - base <= 0) break; // empty - b = base; } } return null; } /** - * Tries to poll next task in FIFO order, failing on - * contention or stalls. Used only by topLevelExec to repoll - * from the queue obtained from pool.scan. + * Tries to poll next task in FIFO order, failing without + * retries on contention or stalls. Used only by topLevelExec + * to repoll from the queue obtained from pool.scan. */ private ForkJoinTask tryPoll() { - ForkJoinTask[] a; int cap; - if ((a = array) != null && (cap = a.length) > 0) { - for (int b = base;;) { - int nb = b + 1, k; - ForkJoinTask t = a[k = (cap - 1) & b]; - U.loadFence(); - if (b == (b = base)) { - if (t == null) - t = a[k]; - else if (t == (t = cmpExSlotToNull(a, k, t))) { - base = nb; - U.storeFence(); - return t; - } - if (t == null) - break; - b = base; - } + ForkJoinTask t; ForkJoinTask[] a; int b, cap, k; + if ((a = array) != null && (cap = a.length) > 0 && + (t = a[k = (b = base) & (cap - 1)]) != null) { + U.loadFence(); + if (base == b && + U.compareAndSetReference(a, slotOffset(k), t, null)) { + updateBase(b + 1); + return t; } } return null; @@ -1484,18 +1482,20 @@ else if (t == (t = cmpExSlotToNull(a, k, t))) { * remaining local tasks and/or others available from the * given queue, if any. */ - final void topLevelExec(ForkJoinTask task, WorkQueue src) { - int cfg = config, fifo = cfg & FIFO, nstolen = 1; + final void topLevelExec(ForkJoinTask task, WorkQueue src, int srcId) { + int cfg = config, fifo = cfg & FIFO, nstolen = nsteals + 1; + if ((srcId & 1) != 0) // don't record external sources + source = srcId; + if ((cfg & CLEAR_TLS) != 0) + ThreadLocalRandom.eraseThreadLocals(Thread.currentThread()); while (task != null) { task.doExec(); if ((task = nextLocalTask(fifo)) == null && src != null && (task = src.tryPoll()) != null) ++nstolen; } - nsteals += nstolen; - source = 0; - if ((cfg & CLEAR_TLS) != 0) - ThreadLocalRandom.eraseThreadLocals(Thread.currentThread()); + nsteals = nstolen; + forgetSource(); } /** @@ -1503,36 +1503,41 @@ final void topLevelExec(ForkJoinTask task, WorkQueue src) { * runs task if present. */ final void tryRemoveAndExec(ForkJoinTask task, boolean internal) { + boolean taken = false; ForkJoinTask[] a = array; int b = base, p = top, s = p - 1, d = p - b, cap; - if (task != null && d > 0 && a != null && (cap = a.length) > 0) { + if (task != null && a != null && (cap = a.length) > 0 && d > 0) { for (int i = s; ; --i) { ForkJoinTask t; int k; if ((t = a[k = i & (cap - 1)]) == task) { - if (!internal) { - if (getAndSetAccess(1) != 0) - break; // fail if locked - if (top != p || cmpExSlotToNull(a, k, t) != t) { - releaseAccess(); - break; + long pos = slotOffset(k); + if (!internal && !tryLockPhase()) + break; // fail if locked + if (top == p && + U.compareAndSetReference(a, pos, task, null)) { + if (i == s) // act as pop + updateTop(s); + else if (i == base) // act as poll + updateBase(i + 1); + else { // swap with top + U.putReferenceVolatile( + a, pos, (ForkJoinTask) + U.getAndSetReference( + a, slotOffset(s & (cap - 1)), null)); + updateTop(s); } + taken = true; } - else if (getAndClearSlot(a, k) == null) - break; // missed - if (i == s) // act as pop - top = s; - else if (i == base) // act as poll - base = i + 1; - else // swap top - a[k] = getAndClearSlot(a, (top = s) & (cap - 1)); - releaseAccess(); - task.doExec(); + if (!internal) + unlockPhase(); break; } else if (t == null || --d == 0) break; } } + if (taken) + task.doExec(); } /** @@ -1544,15 +1549,20 @@ else if (t == null || --d == 0) * @return task status if known done; else 0 */ final int helpComplete(ForkJoinTask task, boolean internal, int limit) { + int status = 0; if (task != null) { outer: for (;;) { - ForkJoinTask[] a; ForkJoinTask t; - int status, p, s, cap, k; - if ((status = task.status) < 0) - return status; - if ((a = array) == null || (cap = a.length) <= 0 || - (t = a[k = (cap - 1) & (s = (p = top) - 1)]) == null || - !(t instanceof CountedCompleter)) + ForkJoinTask[] a; ForkJoinTask t; boolean taken; + int stat, p, s, cap, k; + if ((stat = task.status) < 0) { + status = stat; + break; + } + if ((a = array) == null || (cap = a.length) <= 0) + break; + if ((t = a[k = (cap - 1) & (s = (p = top) - 1)]) == null) + break; + if (!(t instanceof CountedCompleter)) break; CountedCompleter f = (CountedCompleter)t; for (int steps = cap;;) { // bound path @@ -1561,28 +1571,22 @@ final int helpComplete(ForkJoinTask task, boolean internal, int limit) { if ((f = f.completer) == null || --steps == 0) break outer; } - if (!internal) { - if (getAndSetAccess(1) != 0) - break; // fail if locked - if (top != p || cmpExSlotToNull(a, k, t) != t) { - releaseAccess(); - break; // missed - } - top = s; - releaseAccess(); - } - else { - if (getAndClearSlot(a, k) == null) - break; - top = s; - U.storeFence(); - } + if (!internal && !tryLockPhase()) + break; + if (taken = + (top == p && + U.compareAndSetReference(a, slotOffset(k), t, null))) + updateTop(s); + if (!internal) + unlockPhase(); + if (!taken) + break; t.doExec(); if (limit != 0 && --limit == 0) break; } } - return 0; + return status; } /** @@ -1594,29 +1598,23 @@ final int helpComplete(ForkJoinTask task, boolean internal, int limit) { final void helpAsyncBlocker(ManagedBlocker blocker) { if (blocker != null) { for (;;) { - ForkJoinTask[] a = array; - int b = base, cap; - if (a == null || (cap = a.length) <= 0 || top - b <= 0) + ForkJoinTask[] a; int b, cap, k; + if ((a = array) == null || (cap = a.length) <= 0) break; - int nb = b + 1, nk = (cap - 1) & nb, k; - ForkJoinTask t = a[k = (cap - 1) & b]; + ForkJoinTask t = a[k = (b = base) & (cap - 1)]; U.loadFence(); - if (base != b) - ; - else if (blocker.isReleasable()) - break; - else if (a[k] != t) - ; - else if (t == null) { - if (a[nk] == null) + if (t == null) { + if (top - b <= 0) break; } else if (!(t instanceof CompletableFuture .AsynchronousCompletionTask)) break; - else if (cmpExSlotToNull(a, k, t) == t) { - base = nb; - U.storeFence(); + if (blocker.isReleasable()) + break; + if (base == b && t != null && + U.compareAndSetReference(a, slotOffset(k), t, null)) { + updateBase(b + 1); t.doExec(); } } @@ -1630,30 +1628,20 @@ else if (cmpExSlotToNull(a, k, t) == t) { */ final boolean isApparentlyUnblocked() { Thread wt; Thread.State s; - return (phase > 0 && (config & REGISTERED) != 0 && - (wt = owner) != null && + return ((wt = owner) != null && (phase & IDLE) != 0 && (s = wt.getState()) != Thread.State.BLOCKED && s != Thread.State.WAITING && s != Thread.State.TIMED_WAITING); } - /** - * Called in constructors if ThreadLocals not preserved - */ - final void setClearThreadLocals() { - config |= CLEAR_TLS; - } - static { U = Unsafe.getUnsafe(); Class klass = WorkQueue.class; - ACCESS = U.objectFieldOffset(klass, "access"); - Class aklass = ForkJoinTask[].class; - ABASE = U.arrayBaseOffset(aklass); - int scale = U.arrayIndexScale(aklass); - ASHIFT = 31 - Integer.numberOfLeadingZeros(scale); - if ((scale & (scale - 1)) != 0) - throw new Error("array index scale not a power of two"); + PHASE = U.objectFieldOffset(klass, "phase"); + BASE = U.objectFieldOffset(klass, "base"); + TOP = U.objectFieldOffset(klass, "top"); + SOURCE = U.objectFieldOffset(klass, "source"); + ARRAY = U.objectFieldOffset(klass, "array"); } } @@ -1685,27 +1673,21 @@ final void setClearThreadLocals() { */ static volatile RuntimePermission modifyThreadPermission; - - // Instance fields - volatile long stealCount; // collects worker nsteals - volatile long threadIds; // for worker thread names - final long bounds; // min, max threads packed as shorts - final long keepAlive; // milliseconds before dropping if idle - final int config; // static configuration bits - volatile int runState; // SHUTDOWN, STOP, TERMINATED, seq bits - - final Predicate saturate; + // fields declared in order of their likely layout on most VMs volatile CountDownLatch termination; // lazily constructed + final Predicate saturate; final ForkJoinWorkerThreadFactory factory; final UncaughtExceptionHandler ueh; // per-worker UEH final SharedThreadContainer container; final String workerNamePrefix; // null for common pool WorkQueue[] queues; // main registry - - @jdk.internal.vm.annotation.Contended("fjpctl") // segregate + final long keepAlive; // milliseconds before dropping if idle + final long config; // static configuration bits + volatile long stealCount; // collects worker nsteals + volatile long threadIds; // for worker thread names volatile long ctl; // main pool control - @jdk.internal.vm.annotation.Contended("fjpctl") // colocate int parallelism; // target number of workers + volatile int runState; // versioned, lockable // Support for atomic operations private static final Unsafe U; @@ -1750,20 +1732,20 @@ private int getAndBitwiseOrRunState(int v) { // for status bits private boolean casRunState(int c, int v) { return U.compareAndSetInt(this, RUNSTATE, c, v); } - private void releaseRunStateLock() { // increment lock bit - U.getAndAddInt(this, RUNSTATE, LOCKED); + private void unlockRunState() { // increment lock bit + U.getAndAddInt(this, RUNSTATE, RS_LOCK); } - private int acquireRunStateLock() { // lock and return current state - int s, u; // locked when LOCKED set - if (((s = runState) & LOCKED) == 0 && casRunState(s, u = s + LOCKED)) + private int lockRunState() { // lock and return current state + int s, u; // locked when RS_LOCK set + if (((s = runState) & RS_LOCK) == 0 && casRunState(s, u = s + RS_LOCK)) return u; else return spinLockRunState(); } private int spinLockRunState() { // spin/sleep for (int waits = 0, s, u;;) { - if (((s = runState) & LOCKED) == 0) { - if (casRunState(s, u = s + LOCKED)) + if (((s = runState) & RS_LOCK) == 0) { + if (casRunState(s, u = s + RS_LOCK)) return u; waits = 0; } @@ -1795,13 +1777,12 @@ static boolean poolIsStopping(ForkJoinPool p) { // Used by ForkJoinTask * @return true if successful */ private boolean createWorker() { - int rs = runState; ForkJoinWorkerThreadFactory fac = factory; SharedThreadContainer ctr = container; Throwable ex = null; ForkJoinWorkerThread wt = null; try { - if ((rs & STOP) == 0 && // avoid construction if terminating + if ((runState & STOP) == 0 && // avoid construction if terminating fac != null && (wt = fac.newThread(this)) != null) { if (ctr != null) ctr.start(wt); @@ -1834,56 +1815,42 @@ final String nextWorkerThreadName() { */ final void registerWorker(WorkQueue w) { if (w != null) { - WorkQueue[] qs; int n; w.array = new ForkJoinTask[INITIAL_QUEUE_CAPACITY]; ThreadLocalRandom.localInit(); int seed = w.stackPred = ThreadLocalRandom.getProbe(); - int cfg = w.config | REGISTERED | (config & FIFO); - int id = ((seed << 1) | 1) & SMASK; // initial index guess - if ((qs = queues) != null && (n = qs.length) > 0) { - // scan for open slot without lock; rescan below if needed - for (int k = n, m = n - 1; ; id += 2) { - if (qs[id &= m] == null) - break; - if ((k -= 2) <= 0) { - id |= n; - break; - } - } - int rs = acquireRunStateLock(); - try { - if ((qs = queues) != null && (n = qs.length) > 0) { // reread - if (id >= n || qs[id] != null) { // rescan - for (int k = n, m = n - 1; ; id += 2) { - if (qs[id &= m] == null) - break; - if ((k -= 2) <= 0) { - id |= n; - break; - } - } + int id = ((seed << 1) | 1) & SMASK; // base of linear-probe-like scan + int stop = lockRunState() & STOP; + try { + WorkQueue[] qs; int n; + if ((qs = queues) != null && (n = qs.length) > 0) { + for (int k = n, m = n - 1; ; id += 2) { + if (qs[id &= m] == null) + break; + if ((k -= 2) <= 0) { + id |= n; + break; } - w.phase = w.config = id | cfg; // now publishable - if (id < n) - qs[id] = w; - else if ((rs & STOP) == 0) { // expand unless stopping - int an = n << 1, am = an - 1; - WorkQueue[] as = new WorkQueue[an]; - as[id & am] = w; - for (int j = 1; j < n; j += 2) - as[j] = qs[j]; - for (int j = 0; j < n; j += 2) { - WorkQueue q; // shared queues may move - if ((q = qs[j]) != null) - as[q.config & am] = q; - } - U.storeFence(); // fill before publish - queues = as; + } + w.phase = id; // now publishable + if (id < n) + qs[id] = w; + else if (stop == 0) { // expand unless stopping + int an = n << 1, am = an - 1; + WorkQueue[] as = new WorkQueue[an]; + as[id & am] = w; + for (int j = 1; j < n; j += 2) + as[j] = qs[j]; + for (int j = 0; j < n; j += 2) { + WorkQueue q; // shared queues may move + if ((q = qs[j]) != null) + as[q.phase & EXTERNAL_ID_MASK & am] = q; } + U.storeFence(); // fill before publish + queues = as; } - } finally { - releaseRunStateLock(); } + } finally { + unlockRunState(); } } } @@ -1898,38 +1865,43 @@ else if ((rs & STOP) == 0) { // expand unless stopping * @param ex the exception causing failure, or null if none */ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { - WorkQueue w; int cfg; - if ((w = (wt == null) ? null : wt.workQueue) == null) - cfg = 0; - else if (((cfg = w.config) & REGISTERED) != 0) { - w.config = cfg & ~REGISTERED; - if (w.phase < 0) - reactivate(w); // possible if stopped before released - if (w.top - w.base > 0) { - for (ForkJoinTask t; (t = w.nextLocalTask()) != null; ) - ForkJoinTask.cancelIgnoringExceptions(t); + WorkQueue w = null; + int src = 0, phase = 0; + boolean replaceable = false; + if (wt != null && (w = wt.workQueue) != null) { + phase = w.phase; + src = w.source; + w.owner = null; // disable signals + if (phase != 0) { // else failed to start + replaceable = true; + if ((phase & IDLE) != 0) + reactivate(w); // pool stopped before released + if (w.top - w.base > 0) { + for (ForkJoinTask t; (t = w.nextLocalTask()) != null; ) + ForkJoinTask.cancelIgnoringExceptions(t); + } } } long c = ctl; - if ((cfg & TRIMMED) == 0) // decrement counts + if (src != TRIMMED) // decrement counts do {} while (c != (c = compareAndExchangeCtl( c, ((RC_MASK & (c - RC_UNIT)) | (TC_MASK & (c - TC_UNIT)) | - (SP_MASK & c))))); + (LMASK & c))))); else if ((int)c == 0) // was dropped on timeout - cfg &= ~REGISTERED; // suppress replacement if last + replaceable = false; if ((tryTerminate(false, false) & STOP) == 0 && w != null) { WorkQueue[] qs; int n, i; // remove index unless terminating long ns = w.nsteals & 0xffffffffL; - int rs = acquireRunStateLock() & STOP; - if (rs == 0 && (qs = queues) != null && (n = qs.length) > 0 && - qs[i = cfg & (n - 1)] == w) { + int stop = lockRunState() & STOP; + if (stop == 0 && (qs = queues) != null && (n = qs.length) > 0 && + qs[i = phase & SMASK & (n - 1)] == w) { qs[i] = null; stealCount += ns; // accumulate steals } - releaseRunStateLock(); - if (rs == 0 && (cfg & REGISTERED) != 0) + unlockRunState(); + if (stop == 0 && replaceable) signalWork(); // may replace unless trimmed or uninitialized } if (ex != null) @@ -1943,17 +1915,23 @@ final void signalWork() { int pc = parallelism; for (long c = ctl;;) { WorkQueue[] qs = queues; - WorkQueue v = null; long ac = (c + RC_UNIT) & RC_MASK, nc; - int sp = (int)c & ACTIVE, i = sp & SMASK; - if ((short)(c >>> RC_SHIFT) >= pc || qs == null || qs.length <= i) + int sp = (int)c, i = sp & SMASK; + if (qs == null || qs.length <= i) break; - if (sp != 0 && (v = qs[i]) != null) - nc = (v.stackPred & SP_MASK) | (c & TC_MASK); - else if ((short)(c >>> TC_SHIFT) < pc) - nc = ((c + TC_UNIT) & TC_MASK); - else + WorkQueue v = null, w = qs[i]; + if ((short)(c >>> RC_SHIFT) >= pc) break; + if (sp == 0) { + if ((short)(c >>> TC_SHIFT) >= pc) + break; + nc = ((c + TC_UNIT) & TC_MASK); + } + else { + if ((v = w) == null) + break; + nc = (v.stackPred & LMASK) | (c & TC_MASK); + } if (c == (c = compareAndExchangeCtl(c, nc | ac))) { if (v == null) createWorker(); @@ -1969,27 +1947,23 @@ else if ((short)(c >>> TC_SHIFT) < pc) } /** - * Reactivates the given worker (and possibly others if not top of - * ctl stack), or any worker if null and idle. A variant of the - * no-create case of signalWork. + * Reactivates the given worker, and possibly others if not top of + * ctl stack. Needed during shutdown to ensure release on termination. */ private void reactivate(WorkQueue w) { for (long c = ctl;;) { - WorkQueue v; Thread t; - WorkQueue[] qs = queues; - int sp = (int)c & ACTIVE, i = sp & SMASK; - long pc = (UC_MASK & (c + RC_UNIT)) | (c & TC_MASK); - if ((w == null && (c & RC_MASK) > 0L) || sp == 0 || - qs == null || qs.length <= i || (v = qs[i]) == null) - break; - long nc = (v.stackPred & SP_MASK) | pc; - if (w != null && w != v && w.phase >= 0) + WorkQueue[] qs; WorkQueue v; int sp, i; Thread t; + if ((qs = queues) == null || (sp = (int)c) == 0 || + qs.length <= (i = sp & SMASK) || (v = qs[i]) == null || + (v != w && w != null && (w.phase & IDLE) == 0)) break; - if (c == (c = compareAndExchangeCtl(c, nc))) { + if (c == (c = compareAndExchangeCtl( + c, ((UMASK & (c + RC_UNIT)) | (c & TC_MASK) | + (v.stackPred & LMASK))))) { v.phase = sp; if ((t = v.parker) != null) U.unpark(t); - if (v == w || w == null) + if (v == w) break; } } @@ -1997,45 +1971,61 @@ private void reactivate(WorkQueue w) { /** * Internal version of isQuiescent and related functionality. - * @return negative if terminating, 0 if all workers are inactive - * and submission queues are empty and unlocked, else positive - */ - private int checkQuiescence() { - long prevCtl = 0L; - boolean scanned = false; - outer: for (int prevState = 0, rs = runState; ; prevState = rs) { - long c; WorkQueue[] qs; int n; - if ((rs & STOP) != 0) // terminating - return -1; - if (((c = ctl) & RC_MASK) != 0L) - break; // active - if (prevCtl != (prevCtl = c)) - scanned = false; // inconsistent - else if (scanned) { - if ((rs & SHUTDOWN) == 0) - return 0; - if (casRunState(rs, rs | STOP)) - return -1; // try to trigger termination - scanned = false; // retry + * @return true if terminating or all workers are inactive and + * submission queues are empty and unlocked, else ensure at least + * one worker is active (if any exist) + * @param transition trigger termination or advance generation if + * quiescent + */ + private boolean isQuiescent(boolean transition) { + long phaseSum = 0L; + boolean swept = false, possibleSubmission = false; + outer: for (int e = 0;;) { + int prevRunState = e; + long c = ctl; + if (((e = runState) & STOP) != 0) + return true; // terminating + else if ((c & RC_MASK) > 0L) + break; // at least one active + else if (possibleSubmission) { // cover signal race + WorkQueue[] qs; int sp, j; WorkQueue v; Thread t; + possibleSubmission = swept = false; + if ((sp = (int)c) == 0 || (qs = queues) == null || + qs.length <= (j = sp & SMASK) || (v = qs[j]) == null) + break; // no inactive workers + if (compareAndSetCtl(c, (v.stackPred & LMASK) | + (UMASK & (c + RC_UNIT)) | (c & TC_MASK))) { + v.phase = sp; + if ((t = v.parker) != null) + U.unpark(t); // reactivate + } } - if ((qs = queues) != null && (n = qs.length) > 0) { - for (int i = 0;;) { // alternates even for external, odd internal - WorkQueue q, w; - if ((q = qs[i]) != null && - (q.access != 0 || q.top - q.base > 0)) - break outer; // possibly nonempty - if (++i == n) - break; - if ((w = qs[i]) != null && w.phase > 0) - break outer; // active worker - if (++i == n) - break; + else if (ctl != c) // re-snapshot + ; + else if ((e == prevRunState && (e & RS_LOCK) == 0 && swept) && + (!transition || + casRunState(e, ((e & SHUTDOWN) == 0 ? + e + RS_EPOCH : // advance + e | STOP)))) // terminate + return true; + else { + long sum = 0L; + WorkQueue[] qs = queues; + int n = (qs == null) ? 0 : qs.length; + for (int i = 0; i < n; i += 2) { // scan external queues + WorkQueue q; int p; + if ((q = qs[i]) != null) { + if (((p = q.phase) & IDLE) == 0 || q.top - q.base > 0) { + possibleSubmission = true; + continue outer; // recheck counts + } + sum += p; + } } + swept = (phaseSum == (phaseSum = sum)); } - if ((rs = runState) == prevState && (rs & LOCKED) == 0) - scanned = true; // same unlocked state } - return 1; + return false; } /** @@ -2045,82 +2035,101 @@ else if (scanned) { * @param w caller's WorkQueue (may be null on failed initialization) */ final void runWorker(WorkQueue w) { - int rs = runState; - if (w != null && (rs & STOP) == 0) { - int phase = w.phase, r = w.stackPred; // use seed from registerWorker - for (int srcs = ACTIVE;;) { - r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift - if ((srcs = scan(w, srcs, r)) < 0) { - if (rs != (rs = runState)) { - if ((rs & STOP) != 0) - break; - srcs &= ACTIVE; // stale; rescan - } - else { - long c; // try to inactivate & enqueue - int p = phase + SS_SEQ, np = p | INACTIVE; - long nc = ((p & ACTIVE & SP_MASK) | - ((c = ctl) - RC_UNIT) & UC_MASK); - w.stackPred = (int)c; // set ctl stack link - w.phase = np; - if (!compareAndSetCtl(c, nc)) { - w.phase = phase; // back out on ctl contention - srcs &= ACTIVE; - } - else if ((phase = awaitWork(w, nc)) > 0) - srcs = ACTIVE; // restart - else - break; // terminated + if (w != null) { + int phase = w.phase, r = w.stackPred; // seed from registerWorker + int e = runState; // reset or exit when changed + long next = 0L; + for (boolean stateChange = true;;) { + int window; // encodes origin and previous non-empty queue + if (stateChange) { + stateChange = false; + if ((e & STOP) != 0) // terminating + break; // else use random origin + window = (INVALID_ID << 16) | (r >>> 16); + } + else + window = (int)next; // continue from last scan + if ((next = scan(w, window, (r << 1) | 1)) < 0L) + ; // continue scanning + else if (e != (e = runState)) + stateChange = true; + else { // try to inactivate & enqueue + boolean enqueued = true; + int idlePhase = phase + IDLE; + long np = (phase + (IDLE << 1)) & LMASK, pc = ctl; + long qc = ((pc - RC_UNIT) & UMASK) | np; + w.stackPred = (int)pc; // set ctl stack link + w.phase = idlePhase; // try to enqueue + if (pc != (pc = compareAndExchangeCtl(pc, qc))) { + qc = ((pc - RC_UNIT) & UMASK) | np; + w.stackPred = (int)pc; // retry once + if (pc != compareAndExchangeCtl(pc, qc)) + enqueued = false; } + if (!enqueued) + w.phase = phase; // back out on contention + else if ((phase = awaitWork(w, qc, idlePhase)) == idlePhase) + break; // worker exit + else if (e != (e = runState)) // recheck after blocking + stateChange = true; } + r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift } } } /** * Scans for and if found executes top-level tasks: Tries to poll - * each queue starting at a random index with random stride, + * each queue starting at initial index with random stride, * returning scan window and retry indicator. * * @param w caller's WorkQueue - * @param prevSrcs encodes up to two previously non-empty scanned queues - * @param r random seed - * @return the next srcs value to use, with negative (INACTIVE) set if - * none found + * @param window encodes up to two previously non-empty scanned queues + * @param step random array stride + * @return the next window value to use, with highest bit set for rescan */ - private int scan(WorkQueue w, int prevSrcs, int r) { + private long scan(WorkQueue w, int window, int step) { WorkQueue[] qs = queues; - int n = (w == null || qs == null) ? 0 : qs.length; - int next = prevSrcs | INACTIVE; // return value on empty scan - int window = ((prevSrcs << 16) | 1) & ACTIVE; // base of next window - int step = (r >>> 16) | 1; // random stride - outer: for (int i = n; i > 0; --i, r += step) { + int n = (qs == null) ? 0 : qs.length; + long next = window & LMASK; // default for empty scan + outer: for (int i = window, l = n; l > 0; --l, i += step) { int j, cap; WorkQueue q; ForkJoinTask[] a; - if ((q = qs[j = r & SMASK & (n - 1)]) != null && + if ((q = qs[j = i & SMASK & (n - 1)]) != null && (a = q.array) != null && (cap = a.length) > 0) { - for (int srcs = window + j, b = q.base, k;;) { - ForkJoinTask t = a[k = (cap - 1) & b]; - U.loadFence(); // re-read b and t - if (b == (b = q.base)) { // else inconsistent; retry - ForkJoinTask u; - int nb = b + 1, nk = (cap - 1) & nb; - if (t == null) - u = a[k]; - else if (t == (u = WorkQueue.cmpExSlotToNull(a, k, t))) { - q.base = nb; - w.source = next = srcs; - if (a[nk] != null && srcs != prevSrcs) - signalWork(); // propagate at most twice/run - w.topLevelExec(t, q); + for (int b, k;;) { + ForkJoinTask t = a[k = (b = q.base) & (cap - 1)]; + U.loadFence(); // re-read b and t + if (q.base == b) { // else inconsistent; retry + Object o; // to check identities + int nb = b + 1, nk = nb & (cap - 1); + if (t == null) { + if (a[k] == null) { // revisit if nonempty + if ((next & RESCAN) == 0L && + (a[nk] != null || q.top - b > 0)) + next |= RESCAN; + break; + } + } + else if (t == (o = U.compareAndExchangeReference( + a, slotOffset(k), t, null))) { + q.updateBase(nb); + if (window != (window = (window << 16) | j) && + a[nk] != null) + signalWork(); // propagate at most twice/run + next = (window & LMASK) | RESCAN; + if (w != null) // always true + w.topLevelExec(t, q, j); break outer; } - if (u == null) { - if (a[nk] != null && next < 0 && - (t == null || srcs != prevSrcs)) - next = srcs; // limit rescans + else if (o == null) { // contended; limit rescans + int qw = (window << 16) | j; + if ((next & RESCAN) == 0L && + (q.array != a || // resized + (qw != window && + (a[nk] != null || q.top - q.base > 0)))) + next = (qw & LMASK) | RESCAN; break; } - b = q.base; // retry } } } @@ -2131,53 +2140,51 @@ else if (t == (u = WorkQueue.cmpExSlotToNull(a, k, t))) { /** * Awaits signal or termination. * - * @param queuedCtl ctl value at point of inactivation - * @return current phase, or negative for exit - */ - private int awaitWork(WorkQueue w, long queuedCtl) { - boolean idle; int phase, active; - if ((active = (short)(queuedCtl >>> RC_SHIFT)) > 0) - idle = false; // not quiescent - else if (!(idle = (checkQuiescence() == 0))) - reactivate(null); // ensure live - if (w == null) - phase = -1; // currently impossible - else { - if ((phase = w.phase) < 0 & !idle) { - int spins = ((short)(queuedCtl >>> TC_SHIFT) << 1) + active; - do { // spin for approx #accesses to scan+signal - Thread.onSpinWait(); - } while ((phase = w.phase) < 0 && --spins > 0); - } - if (phase < 0) { // emulate LockSupport.park + * @param queuedCtl ctl at point of inactivation + * @param idlePhase w's phase while idle + * @return current phase, same as idlePhase for exit + */ + private int awaitWork(WorkQueue w, long queuedCtl, int idlePhase) { + int p = idlePhase; + boolean quiescent = (queuedCtl & RC_MASK) <= 0L && isQuiescent(true); + if (w != null && (p = w.phase) == idlePhase && (runState & STOP) == 0) { + long deadline = (quiescent ? // timeout for trim + keepAlive + System.currentTimeMillis() : 0L); + int spins = ((((int)(queuedCtl >>> TC_SHIFT)) & SMASK) << 1) + 3; + do { // spin for approx #accesses to scan+signal + Thread.onSpinWait(); + } while ((p = w.phase) == idlePhase && --spins >= 0); + + if (spins < 0) { // emulate LockSupport.park LockSupport.setCurrentBlocker(this); - long deadline = idle? System.currentTimeMillis() + keepAlive: 0L; - for (;;) { // await signal or termination - if ((runState & STOP) != 0) + w.parker = Thread.currentThread(); + for (;;) { + if ((p = w.phase) != idlePhase) break; - w.parker = Thread.currentThread(); - if (w.phase < 0) - U.park(idle, deadline); - w.parker = null; - if ((phase = w.phase) >= 0) + U.park(quiescent, deadline); + if ((p = w.phase) != idlePhase) break; - if (idle && // check for idle timeout + Thread.interrupted(); // clear status for next park + if ((runState & STOP) != 0) + break; + if (quiescent && // trim on timeout deadline - System.currentTimeMillis() < TIMEOUT_SLOP) { - int id = phase & SMASK; - long sp = w.stackPred & SP_MASK; - long c = ctl, nc = sp | (UC_MASK & (c - TC_UNIT)); + int id = idlePhase & SMASK; + long sp = w.stackPred & LMASK; + long c = ctl, nc = sp | (UMASK & (c - TC_UNIT)); if (((int)c & SMASK) == id && compareAndSetCtl(c, nc)) { - w.phase = w.config = id | TRIMMED; - break; // add sentinel for deregisterWorker + w.source = TRIMMED; // sentinel for deregisterWorker + w.phase = idlePhase + IDLE; + break; } - deadline += keepAlive; // not at head; restart timer + deadline += keepAlive; // not at head; restart timer } - Thread.interrupted(); // clear status for next park - } // loop on interrupt or stale unpark + } + w.parker = null; LockSupport.setCurrentBlocker(null); } } - return phase; + return p; } /** @@ -2218,45 +2225,44 @@ private ForkJoinTask pollScan(boolean submissionsOnly) { */ private int tryCompensate(long c) { Predicate sat; - long b = bounds; // unpack fields - int pc = parallelism; - int minActive = (short)(b & SMASK), - maxTotal = (short)(b >>> SWIDTH) + pc, + long b = config; + int pc = parallelism, // unpack fields + minActive = (short)(b >>> RC_SHIFT), + maxTotal = (short)(b >>> TC_SHIFT) + pc, active = (short)(c >>> RC_SHIFT), total = (short)(c >>> TC_SHIFT), - sp = (int)c & ACTIVE; - if ((runState & STOP) != 0) // terminating - return 0; - else if (sp != 0 && active <= pc) { // activate idle worker + sp = (int)c, + stat = -1; // default retry return + if (sp != 0 && active <= pc) { // activate idle worker WorkQueue[] qs; WorkQueue v; int i; Thread t; - if (ctl == c && (qs = queues) != null && - qs.length > (i = sp & SMASK) && (v = qs[i]) != null) { - long nc = (v.stackPred & SP_MASK) | (UC_MASK & c); - if (compareAndSetCtl(c, nc)) { - v.phase = sp; - if ((t = v.parker) != null) - U.unpark(t); - return UNCOMPENSATE; - } + if ((qs = queues) != null && qs.length > (i = sp & SMASK) && + (v = qs[i]) != null && + compareAndSetCtl(c, (c & UMASK) | (v.stackPred & LMASK))) { + v.phase = sp; + if ((t = v.parker) != null) + U.unpark(t); + stat = UNCOMPENSATE; } - return -1; // inconsistent; retry } - else if (active > minActive && total >= pc) { // reduce active workers - long nc = ((RC_MASK & (c - RC_UNIT)) | (~RC_MASK & c)); - return compareAndSetCtl(c, nc) ? UNCOMPENSATE : -1; + else if (active > minActive && total >= pc) { // reduce active workers + if (compareAndSetCtl(c, ((c - RC_UNIT) & RC_MASK) | (c & ~RC_MASK))) + stat = UNCOMPENSATE; } - else if (total < maxTotal && total < MAX_CAP) { + else if (total < maxTotal && total < MAX_CAP) { // try to expand pool long nc = ((c + TC_UNIT) & TC_MASK) | (c & ~TC_MASK); - return (!compareAndSetCtl(c, nc) ? -1 : // expand pool - createWorker() ? UNCOMPENSATE : 0); + if ((runState & STOP) != 0) // terminating + stat = 0; + else if (compareAndSetCtl(c, nc)) + stat = createWorker() ? UNCOMPENSATE : 0; } - else if (!compareAndSetCtl(c, c)) // validate - return -1; + else if (!compareAndSetCtl(c, c)) // validate + ; else if ((sat = saturate) != null && sat.test(this)) - return 0; + stat = 0; else throw new RejectedExecutionException( "Thread limit exceeded replacing blocked worker"); + return stat; } /** @@ -2277,67 +2283,70 @@ final void uncompensate() { * @param internal true if w is owned by a ForkJoinWorkerThread * @return task status on exit, or UNCOMPENSATE for compensated blocking */ + final int helpJoin(ForkJoinTask task, WorkQueue w, boolean internal) { + if (w != null) + w.tryRemoveAndExec(task, internal); int s = 0; - if (task != null) { - if (w != null) - w.tryRemoveAndExec(task, internal); - if ((s = task.status) >= 0 && internal && w != null) { - int wsrc = w.source & SMASK; - int wid = (w.config & SMASK) + 1, r = wid + 1; - long sctl = 0L; // track stability - outer: for (boolean rescan = true;;) { - WorkQueue[] qs; - if ((s = task.status) < 0) + if (task != null && (s = task.status) >= 0 && internal && w != null) { + int wid = w.phase & SMASK, r = wid + 2, wsrc = w.source; + long sctl = 0L; // track stability + outer: for (boolean rescan = true;;) { + if ((s = task.status) < 0) + break; + if (!rescan) { + if ((runState & STOP) != 0) break; - if (!rescan && sctl == (sctl = ctl) && - (s = tryCompensate(sctl)) >= 0) + if (sctl == (sctl = ctl) && (s = tryCompensate(sctl)) >= 0) break; - rescan = false; - int n = ((qs = queues) == null) ? 0 : qs.length; - scan: for (int i = n >>> 1; i > 0; --i, r += 2) { - int j, cap; WorkQueue q; ForkJoinTask[] a; - if ((q = qs[j = r & SMASK & (n - 1)]) != null && - (a = q.array) != null && (cap = a.length) > 0) { - for (int src = j + 1, sq = q.source, b, k;;) { - ForkJoinTask t = a[k = (cap - 1) & (b = q.base)]; - U.loadFence(); - boolean eligible; // check steal chain - for (int steps = n, v = sq;;) { + } + rescan = false; + WorkQueue[] qs = queues; + int n = (qs == null) ? 0 : qs.length; + scan: for (int l = n >>> 1; l > 0; --l, r += 2) { + int j; WorkQueue q; + if ((q = qs[j = r & SMASK & (n - 1)]) != null) { + for (;;) { + int sq = q.source, b, cap, k; ForkJoinTask[] a; + if ((a = q.array) == null || (cap = a.length) <= 0) + break; + ForkJoinTask t = a[k = (b = q.base) & (cap - 1)]; + U.loadFence(); + boolean eligible = false; + if (t == task) + eligible = true; + else if (t != null) { // check steal chain + for (int v = sq, d = cap;;) { WorkQueue p; - if ((v &= SMASK) == wid) { + if (v == wid) { eligible = true; break; } - if (v == 0 || - (p = qs[(v - 1) & (n - 1)]) == null || - --steps == 0) { // bound steps - eligible = false; - break; - } + if ((v & 1) == 0 || // external or none + --d < 0 || // bound depth + (p = qs[v & (n - 1)]) == null) + break; // stale v = p.source; } - if ((s = task.status) < 0) - break outer; // validate - int nb = b + 1, qdepth = q.top - b; - if (sq == (sq = q.source) && - q.base == b && a[k] == t) { - if (t == null) { - if (qdepth > 0) // revisit if nonempty - rescan = true; - break; - } - if (t != task && !eligible) - break; - if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { - q.base = nb; - w.source = src; - t.doExec(); - w.source = wsrc; - rescan = true; // restart at index r - break scan; - } - sq = q.source; + } + if ((s = task.status) < 0) + break outer; // validate + if (q.source == sq && q.base == b && a[k] == t) { + int nb = b + 1, nk = nb & (cap - 1); + if (!eligible) { // revisit if nonempty + if (!rescan && t == null && + (a[nk] != null || q.top - b > 0)) + rescan = true; + break; + } + if (U.compareAndSetReference( + a, slotOffset(k), t, null)) { + q.updateBase(nb); + w.source = j; + t.doExec(); + w.source = wsrc; + rescan = true; // restart at index r + break scan; } } } @@ -2359,25 +2368,31 @@ final int helpJoin(ForkJoinTask task, WorkQueue w, boolean internal) { final int helpComplete(ForkJoinTask task, WorkQueue w, boolean internal) { int s = 0; if (task != null && (s = task.status) >= 0 && w != null) { - int r = w.config + 1; // for indexing + int r = w.phase + 1; // for indexing long sctl = 0L; // track stability outer: for (boolean rescan = true, locals = true;;) { - WorkQueue[] qs; if (locals && (s = w.helpComplete(task, internal, 0)) < 0) break; if ((s = task.status) < 0) break; - if (!rescan && sctl == (sctl = ctl) && - (!internal || (s = tryCompensate(sctl)) >= 0)) - break; + if (!rescan) { + if ((runState & STOP) != 0) + break; + if (sctl == (sctl = ctl) && + (!internal || (s = tryCompensate(sctl)) >= 0)) + break; + } rescan = locals = false; - int n = ((qs = queues) == null) ? 0 : qs.length; - scan: for (int i = n; i > 0; --i, ++r) { - int j, cap; WorkQueue q; ForkJoinTask[] a; - if ((q = qs[j = r & SMASK & (n - 1)]) != null && - (a = q.array) != null && (cap = a.length) > 0) { - for (int b = q.base, k;;) { - ForkJoinTask t = a[k = (cap - 1) & b]; + WorkQueue[] qs = queues; + int n = (qs == null) ? 0 : qs.length; + scan: for (int l = n; l > 0; --l, ++r) { + int j; WorkQueue q; + if ((q = qs[j = r & SMASK & (n - 1)]) != null) { + for (;;) { + ForkJoinTask[] a; int b, cap, k; + if ((a = q.array) == null || (cap = a.length) <= 0) + break; + ForkJoinTask t = a[k = (b = q.base) & (cap - 1)]; U.loadFence(); boolean eligible = false; if (t instanceof CountedCompleter) { @@ -2393,21 +2408,23 @@ final int helpComplete(ForkJoinTask task, WorkQueue w, boolean internal) { } if ((s = task.status) < 0) // validate break outer; - if (b == (b = q.base) && a[k] == t) { - int nb = b + 1, nk = (cap - 1) & nb; - if (!eligible) { - if (t == null && a[nk] != null) + if (q.base == b) { + int nb = b + 1, nk = nb & (cap - 1); + if (eligible) { + if (U.compareAndSetReference( + a, slotOffset(k), t, null)) { + q.updateBase(nb); + t.doExec(); + locals = rescan = true; + break scan; + } + } + else if (a[k] == t) { + if (!rescan && t == null && + (a[nk] != null || q.top - b > 0)) rescan = true; // revisit break; } - if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { - q.base = nb; - U.storeFence(); - t.doExec(); - locals = rescan = true; - break scan; - } - b = q.base; } } } @@ -2427,17 +2444,18 @@ final int helpComplete(ForkJoinTask task, WorkQueue w, boolean internal) { * @return positive if quiescent, negative if interrupted, else 0 */ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { - int phase; // w.phase set negative when temporarily quiescent - if (w == null || (phase = w.phase) < 0) + int phase; // w.phase inactive bit set when temporarily quiescent + if (w == null || ((phase = w.phase) & IDLE) != 0) return 0; - int wsrc = w.source, r = w.config + 1, returnStatus = 1; - int activePhase = phase, inactivePhase = phase | INACTIVE, waits = 0; - long maxSleep = Math.min(nanos >>> 8, MAX_SLEEP); // approx 1% nanos + int wsrc = w.source; long startTime = System.nanoTime(); + long maxSleep = Math.min(nanos >>> 8, MAX_SLEEP); // approx 1% nanos + long prevSum = 0L; + int activePhase = phase, inactivePhase = phase + IDLE; + int r = phase + 1, waits = 0, returnStatus = 1; boolean locals = true; - for (int prevState = -1, rs = runState; ; prevState = rs) { - WorkQueue[] qs; WorkQueue q; - if ((rs & STOP) != 0) + for (int e = runState;;) { + if ((e & STOP) != 0) break; // terminating if (interruptible && Thread.interrupted()) { returnStatus = -1; @@ -2448,47 +2466,53 @@ private int helpQuiesce(WorkQueue w, long nanos, boolean interruptible) { for (ForkJoinTask u; (u = w.nextLocalTask()) != null;) u.doExec(); } + WorkQueue[] qs = queues; + int n = (qs == null) ? 0 : qs.length; + long phaseSum = 0L; boolean rescan = false, busy = false; - int n = ((qs = queues) == null) ? 0 : qs.length; - scan: for (int i = n, j; i > 0; --i, ++r) { + scan: for (int l = n; l > 0; --l, ++r) { + int j; WorkQueue q; if ((q = qs[j = r & SMASK & (n - 1)]) != null && q != w) { - for (int src = j + 1;;) { - ForkJoinTask[] a = q.array; - int b = q.base, nb = b + 1, cap, k; - if (a == null || (cap = a.length) <= 0) + for (;;) { + ForkJoinTask[] a; int b, cap, k; + if ((a = q.array) == null || (cap = a.length) <= 0) break; - ForkJoinTask t = a[k = (cap - 1) & b]; - U.loadFence(); - if (t != null && phase < 0) // reactivate before taking + ForkJoinTask t = a[k = (b = q.base) & (cap - 1)]; + if (t != null && phase == inactivePhase) // reactivate w.phase = phase = activePhase; - if (q.base != b || a[k] != t) - ; - else if (t == null) { - if (!rescan) { - if (q.access != 0 || q.top - b > 0) - rescan = true; - else if (q.phase > 0) - busy = true; + U.loadFence(); + if (q.base == b && a[k] == t) { + int nb = b + 1; + if (t == null) { + if (!rescan) { + int qp = q.phase, mq = qp & (IDLE | 1); + phaseSum += qp; + if (mq == 0 || q.top - b > 0) + rescan = true; + else if (mq == 1) + busy = true; + } + break; + } + if (U.compareAndSetReference( + a, slotOffset(k), t, null)) { + q.updateBase(nb); + w.source = j; + t.doExec(); + w.source = wsrc; + rescan = locals = true; + break scan; } - break; - } - else if (WorkQueue.cmpExSlotToNull(a, k, t) == t) { - q.base = nb; - w.source = src; - rescan = locals = true; - t.doExec(); - w.source = wsrc; - break scan; } } } } - rs = runState; - if (rescan || (rs & LOCKED) != 0 || rs != prevState) + if (e != (e = runState) || prevSum != (prevSum = phaseSum) || + rescan || (e & RS_LOCK) != 0) ; // inconsistent else if (!busy) break; - else if (phase >= 0) { + else if (phase == activePhase) { waits = 0; // recheck, then sleep w.phase = phase = inactivePhase; } @@ -2516,7 +2540,7 @@ else if (waits == 0) // same as spinLockRunState except * @return positive if quiescent, negative if interrupted, else 0 */ private int externalHelpQuiesce(long nanos, boolean interruptible) { - if (checkQuiescence() > 0) { + if (!isQuiescent(false)) { long startTime = System.nanoTime(); long maxSleep = Math.min(nanos >>> 8, MAX_SLEEP); for (int waits = 0;;) { @@ -2527,7 +2551,7 @@ else if ((t = pollScan(false)) != null) { waits = 0; t.doExec(); } - else if (checkQuiescence() <= 0) + else if (isQuiescent(false)) break; else if (System.nanoTime() - startTime > nanos) return 0; @@ -2591,24 +2615,26 @@ private WorkQueue submissionQueue(int r) { } for (;;) { int n, i, id; WorkQueue[] qs; WorkQueue q; - if ((qs = queues) == null || (n = qs.length) <= 0) + if ((qs = queues) == null) + break; + if ((n = qs.length) <= 0) break; - if ((q = qs[i = (id = r & SEMASK) & (n - 1)]) == null) { - WorkQueue v, w = new WorkQueue(null, id | REGISTERED); + if ((q = qs[i = (id = r & EXTERNAL_ID_MASK) & (n - 1)]) == null) { + WorkQueue w = new WorkQueue(null, id, 0, false); w.array = new ForkJoinTask[INITIAL_QUEUE_CAPACITY]; - acquireRunStateLock(); - if ((v = qs[i]) != null) - q = v; // lost race; discard w - else if (queues == qs) - q = qs[i] = w; // else resized - releaseRunStateLock(); + int stop = lockRunState() & STOP; + if (stop == 0 && queues == qs && qs[i] == null) + q = qs[i] = w; // else discard; retry + unlockRunState(); + if (q != null) + return q; + if (stop != 0) + break; } - if (q == null) - ; // retry if resized - else if (q.getAndSetAccess(1) != 0) // move index + else if (!q.tryLockPhase()) // move index r = ThreadLocalRandom.advanceProbe(r); else if ((runState & SHUTDOWN) != 0) { - q.access = 0; // check while q lock held + q.unlockPhase(); // check while q lock held break; } else @@ -2617,31 +2643,30 @@ else if ((runState & SHUTDOWN) != 0) { throw new RejectedExecutionException(); } - /** - * Pushes a submission to the pool, using internal queue if called - * from ForkJoinWorkerThread, else external queue. - */ private void poolSubmit(boolean signalIfEmpty, ForkJoinTask task) { - Thread t; ForkJoinWorkerThread wt; WorkQueue q; + Thread t; ForkJoinWorkerThread wt; WorkQueue q; boolean internal; if (((t = Thread.currentThread()) instanceof ForkJoinWorkerThread) && (wt = (ForkJoinWorkerThread)t).pool == this) { + internal = true; q = wt.workQueue; - U.storeStoreFence(); // ensure safely publishable } - else // find and lock queue + else { // find and lock queue + internal = false; q = submissionQueue(ThreadLocalRandom.getProbe()); - q.push(task, this, signalIfEmpty); + } + q.push(task, signalIfEmpty ? this : null, internal); } /** - * Pushes a forked task to an external queue. + * Returns queue for an external submission, bypassing call to + * submissionQueue if already established and unlocked. */ final WorkQueue externalSubmissionQueue() { WorkQueue[] qs; WorkQueue q; int n; int r = ThreadLocalRandom.getProbe(); return (((qs = queues) != null && (n = qs.length) > 0 && - (q = qs[r & SEMASK & (n - 1)]) != null&& r != 0 && - q.getAndSetAccess(1) == 0) ? q : submissionQueue(r)); + (q = qs[r & EXTERNAL_ID_MASK & (n - 1)]) != null&& r != 0 && + q.tryLockPhase()) ? q : submissionQueue(r)); } /** @@ -2654,7 +2679,7 @@ static WorkQueue externalQueue(ForkJoinPool p) { int r = ThreadLocalRandom.getProbe(); return (p != null && (qs = p.queues) != null && (n = qs.length) > 0 && r != 0) ? - qs[r & SEMASK & (n - 1)] : null; + qs[r & EXTERNAL_ID_MASK & (n - 1)] : null; } /** @@ -2751,34 +2776,35 @@ static int getSurplusQueuedTaskCount() { * @return runState (possibly only its status bits) on exit */ private int tryTerminate(boolean now, boolean enable) { - int rs, isShutdown; - if (((rs = runState) & STOP) == 0) { + int e, isShutdown; + if (((e = runState) & STOP) == 0) { if (now) - getAndBitwiseOrRunState(rs = STOP | SHUTDOWN); + getAndBitwiseOrRunState(e = STOP | SHUTDOWN); else { - if ((isShutdown = (rs & SHUTDOWN)) == 0 && enable) + if ((isShutdown = (e & SHUTDOWN)) == 0 && enable) getAndBitwiseOrRunState(isShutdown = SHUTDOWN); - if (isShutdown != 0 && checkQuiescence() < 0) - rs = STOP | SHUTDOWN; + if (isShutdown != 0 && isQuiescent(true)) + e = runState; } } - if ((rs & (STOP | TERMINATED)) == STOP) { - WorkQueue[] qs; WorkQueue q; // cancel tasks and interrupt workers - Thread current = Thread.currentThread(); - WorkQueue w = ((current instanceof ForkJoinWorkerThread) ? - ((ForkJoinWorkerThread)current).workQueue : null); - int r = (w != null) ? w.config : 0; // stagger traversals - int n = ((qs = queues) == null) ? 0 : qs.length; - for (int i = n, j; i > 0; --i, ++r) { - if ((q = qs[j = r & SMASK & (n - 1)]) != null && - (q.config & REGISTERED) != 0) { - if ((j & 1) != 0) - ForkJoinTask.interruptIgnoringExceptions(q.owner); + if ((e & (STOP | TERMINATED)) == STOP) { + int r = ThreadLocalRandom.nextSecondarySeed(); // stagger traversals + WorkQueue[] qs = queues; + int n = (qs == null) ? 0 : qs.length; + for (int l = n; l > 0; --l, ++r) { + int j; WorkQueue q; Thread o; // cancel tasks; interrupt workers + if ((q = qs[j = r & SMASK & (n - 1)]) != null) { + if ((o = q.owner) != null) { + try { + o.interrupt(); + } catch (Throwable ignore) { + } + } for (ForkJoinTask t; (t = q.poll(null)) != null; ) ForkJoinTask.cancelIgnoringExceptions(t); } } - if (((rs = runState) & TERMINATED) == 0 && ctl == 0L) { + if (((e = runState) & TERMINATED) == 0 && ctl == 0L) { if ((getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0) { CountDownLatch done; SharedThreadContainer ctr; if ((done = termination) != null) @@ -2786,10 +2812,10 @@ private int tryTerminate(boolean now, boolean enable) { if ((ctr = container) != null) ctr.close(); } - rs = runState; + e = runState; } } - return rs; + return e; } /** @@ -2975,18 +3001,17 @@ public ForkJoinPool(int parallelism, throw new IllegalArgumentException(); if (factory == null || unit == null) throw new NullPointerException(); + int size = 1 << (33 - Integer.numberOfLeadingZeros(p - 1)); this.parallelism = p; this.factory = factory; this.ueh = handler; this.saturate = saturate; - this.config = asyncMode ? FIFO : 0; this.keepAlive = Math.max(unit.toMillis(keepAliveTime), TIMEOUT_SLOP); - int corep = Math.clamp(corePoolSize, p, MAX_CAP); int maxSpares = Math.clamp(maximumPoolSize - p, 0, MAX_CAP); int minAvail = Math.clamp(minimumRunnable, 0, MAX_CAP); - this.bounds = (long)(minAvail & SMASK) | (long)(maxSpares << SWIDTH) | - ((long)corep << 32); - int size = 1 << (33 - Integer.numberOfLeadingZeros(p - 1)); + this.config = (((asyncMode ? FIFO : 0) & LMASK) | + (((long)maxSpares) << TC_SHIFT) | + (((long)minAvail) << RC_SHIFT)); this.queues = new WorkQueue[size]; String pid = Integer.toString(getAndAddPoolIds(1) + 1); String name = "ForkJoinPool-" + pid; @@ -3034,8 +3059,8 @@ private ForkJoinPool(byte forCommonPoolOnly) { int p = Math.min(pc, MAX_CAP); int size = (p == 0) ? 1 : 1 << (33 - Integer.numberOfLeadingZeros(p-1)); this.parallelism = p; - this.config = preset; - this.bounds = (long)(1 | (maxSpares << SWIDTH)); + this.config = ((preset & LMASK) | (((long)maxSpares) << TC_SHIFT) | + (1L << RC_SHIFT)); this.factory = fac; this.ueh = handler; this.keepAlive = DEFAULT_KEEPALIVE; @@ -3202,7 +3227,7 @@ public ForkJoinTask submit(Runnable task) { */ public ForkJoinTask externalSubmit(ForkJoinTask task) { Objects.requireNonNull(task); - externalSubmissionQueue().push(task, this, true); + externalSubmissionQueue().push(task, this, false); return task; } @@ -3336,7 +3361,7 @@ public List> invokeAll(Collection> tasks) public List> invokeAll(Collection> tasks, long timeout, TimeUnit unit) throws InterruptedException { - return invokeAll(tasks, ForkJoinTask.nanosTimeoutToDeadline(unit.toNanos(timeout))); + return invokeAll(tasks, (System.nanoTime() + unit.toNanos(timeout)) | 1L); } @Override @@ -3462,7 +3487,7 @@ public int getActiveThreadCount() { * @return {@code true} if all threads are currently idle */ public boolean isQuiescent() { - return checkQuiescence() <= 0; + return isQuiescent(false); } /** @@ -3593,7 +3618,7 @@ protected int drainTasksTo(Collection> c) { */ public String toString() { // Use a single pass through queues to collect counts - int rs = runState; + int e = runState; long st = stealCount; long qt = 0L, ss = 0L; int rc = 0; WorkQueue[] qs; WorkQueue q; @@ -3619,9 +3644,9 @@ public String toString() { int ac = (short)(c >>> RC_SHIFT); if (ac < 0) // ignore transient negative ac = 0; - String level = ((rs & TERMINATED) != 0 ? "Terminated" : - (rs & STOP) != 0 ? "Terminating" : - (rs & SHUTDOWN) != 0 ? "Shutting down" : + String level = ((e & TERMINATED) != 0 ? "Terminated" : + (e & STOP) != 0 ? "Terminating" : + (e & SHUTDOWN) != 0 ? "Shutting down" : "Running"); return super.toString() + "[" + level + @@ -4001,6 +4026,12 @@ protected RunnableFuture newTaskFor(Callable callable) { PARALLELISM = U.objectFieldOffset(klass, "parallelism"); THREADIDS = U.objectFieldOffset(klass, "threadIds"); TERMINATION = U.objectFieldOffset(klass, "termination"); + Class aklass = ForkJoinTask[].class; + ABASE = U.arrayBaseOffset(aklass); + int scale = U.arrayIndexScale(aklass); + ASHIFT = 31 - Integer.numberOfLeadingZeros(scale); + if ((scale & (scale - 1)) != 0) + throw new Error("array index scale not a power of two"); defaultForkJoinWorkerThreadFactory = new DefaultForkJoinWorkerThreadFactory(); diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index 238186bc84605..6614e95b398f2 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -400,8 +400,7 @@ private int awaitDone(ForkJoinPool pool, int compensation, } catch (OutOfMemoryError ex) { } boolean queued = false; - for (;;) { // try to install node - Aux a; + for (Aux a;;) { // try to install node if ((s = status) < 0) break; else if (node == null) @@ -495,21 +494,13 @@ private int awaitDone(boolean interruptible, long deadline) { awaitDone(internal ? p : null, s, interruptible, deadline); } - /** - * Convert timeout value to deadline, avoiding zero - */ - static long nanosTimeoutToDeadline(long nanos) { - long deadline = System.nanoTime() + nanos; - return (deadline == 0L) ? 1L : deadline; - } - /** * Runs a task body: Unless done, calls exec and records status if * completed, but doesn't wait for completion otherwise. */ final void doExec() { - boolean completed = false; if (status >= 0) { + boolean completed = false; try { completed = exec(); } catch (Throwable rex) { @@ -633,16 +624,6 @@ static final void cancelIgnoringExceptions(ForkJoinTask t) { } } - /** Similarly, for interrupts */ - static final void interruptIgnoringExceptions(Thread t) { - if (t != null) { - try { - t.interrupt(); - } catch (Throwable ignore) { - } - } - } - // public methods /** @@ -667,15 +648,15 @@ public ForkJoinTask() {} */ public final ForkJoinTask fork() { Thread t; ForkJoinWorkerThread wt; - ForkJoinPool p; ForkJoinPool.WorkQueue q; - if ((t = Thread.currentThread()) instanceof ForkJoinWorkerThread) { - p = (wt = (ForkJoinWorkerThread)t).pool; - q = wt.workQueue; - U.storeStoreFence(); // ensure safely publishable + ForkJoinPool p; ForkJoinPool.WorkQueue q; boolean internal; + if (internal = + (t = Thread.currentThread()) instanceof ForkJoinWorkerThread) { + q = (wt = (ForkJoinWorkerThread)t).workQueue; + p = wt.pool; } else q = (p = ForkJoinPool.common).externalSubmissionQueue(); - q.push(this, p, true); + q.push(this, p, internal); return this; } @@ -1023,13 +1004,10 @@ public final void quietlyComplete() { * member of a ForkJoinPool and was interrupted while waiting */ public final V get() throws InterruptedException, ExecutionException { - int s; - if ((s = status) >= 0) { - if (Thread.interrupted()) - s = ABNORMAL; - else - s = awaitDone(true, 0L); - } + int stat = status; + int s = ((stat < 0) ? stat : + (Thread.interrupted()) ? ABNORMAL : + awaitDone(true, 0L)); if (s == ABNORMAL) throw new InterruptedException(); else if ((s & ABNORMAL) != 0) @@ -1054,13 +1032,11 @@ else if ((s & ABNORMAL) != 0) public final V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { long nanos = unit.toNanos(timeout); - int s; - if ((s = status) >= 0) { - if (Thread.interrupted()) - s = ABNORMAL; - else if (nanos > 0L) - s = awaitDone(true, nanosTimeoutToDeadline(nanos)); - } + int stat = status; + int s = ((stat < 0) ? stat : + (Thread.interrupted()) ? ABNORMAL : + (nanos <= 0L) ? 0 : + awaitDone(true, (System.nanoTime() + nanos) | 1L)); if (s == ABNORMAL) throw new InterruptedException(); else if (s >= 0) @@ -1106,18 +1082,15 @@ public final void quietlyInvoke() { */ public final boolean quietlyJoin(long timeout, TimeUnit unit) throws InterruptedException { - int s; long nanos = unit.toNanos(timeout); - if ((s = status) >= 0) { - if (Thread.interrupted()) - s = ABNORMAL; - else if (nanos > 0L) - s = awaitDone(true, nanosTimeoutToDeadline(nanos)); - } + int stat = status; + int s = ((stat < 0) ? stat : + (Thread.interrupted()) ? ABNORMAL : + (nanos <= 0L) ? 0 : + awaitDone(true, (System.nanoTime() + nanos) | 1L)); if (s == ABNORMAL) throw new InterruptedException(); - else - return (s < 0); + return (s < 0); } /** @@ -1134,7 +1107,7 @@ public final boolean quietlyJoinUninterruptibly(long timeout, int s; long nanos = unit.toNanos(timeout); if ((s = status) >= 0 && nanos > 0L) - s = awaitDone(false, nanosTimeoutToDeadline(nanos)); + s = awaitDone(false, (System.nanoTime() + nanos) | 1L); return (s < 0); } @@ -1682,9 +1655,14 @@ public final boolean exec() { return true; } public boolean cancel(boolean mayInterruptIfRunning) { + Thread t; if (trySetCancelled() >= 0) { - if (mayInterruptIfRunning) - interruptIgnoringExceptions(runner); + if (mayInterruptIfRunning && (t = runner) != null) { + try { + t.interrupt(); + } catch (Throwable ignore) { + } + } return true; } return isCancelled(); diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinWorkerThread.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinWorkerThread.java index f7281998251d4..c4fc3de975084 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinWorkerThread.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinWorkerThread.java @@ -76,9 +76,8 @@ public class ForkJoinWorkerThread extends Thread { boolean clearThreadLocals) { super(group, null, pool.nextWorkerThreadName(), 0L, !clearThreadLocals); UncaughtExceptionHandler handler = (this.pool = pool).ueh; - this.workQueue = new ForkJoinPool.WorkQueue(this, 0); - if (clearThreadLocals) - workQueue.setClearThreadLocals(); + this.workQueue = new ForkJoinPool.WorkQueue(this, 0, (int)pool.config, + clearThreadLocals); super.setDaemon(true); if (handler != null) super.setUncaughtExceptionHandler(handler); From 030ed8f5442b6ef4103aa175086c63e6c03c8054 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Wed, 9 Aug 2023 14:17:57 -0400 Subject: [PATCH 20/61] ExecutorService tests --- .../java/util/concurrent/ForkJoinPool.java | 135 ++- .../java/util/concurrent/ForkJoinTask.java | 21 +- test/jdk/ProblemList.txt | 2 - .../concurrent/ExecutorService/CloseTest.java | 90 +- .../ExecutorService/InvokeTest.java | 787 ++++++++++++++++++ .../ExecutorService/SubmitTest.java | 370 ++++++++ .../concurrent/Future/DefaultMethods.java | 119 +-- .../concurrent/forkjoin/AsyncShutdownNow.java | 95 +-- 8 files changed, 1385 insertions(+), 234 deletions(-) create mode 100644 test/jdk/java/util/concurrent/ExecutorService/InvokeTest.java create mode 100644 test/jdk/java/util/concurrent/ExecutorService/SubmitTest.java diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index cc971b97b5eba..b7ad09b416f8a 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -370,7 +370,7 @@ public class ForkJoinPool extends AbstractExecutorService { * like workers except that they are restricted to executing local * tasks that they submitted (or when known, subtasks thereof). * Insertion of tasks in shared mode requires a lock. We use only - * a simple spinlock (as one role of filed "phase") because + * a simple spinlock (as one role of field "phase") because * submitters encountering a busy queue move to a different * position to use or create other queues. They (spin) block when * registering new queues, or indirectly elsewhere (by revisiting @@ -1256,15 +1256,18 @@ final boolean tryLockPhase() { // seqlock acquire */ WorkQueue(ForkJoinWorkerThread owner, int id, int cfg, boolean clearThreadLocals) { + if (clearThreadLocals) + cfg |= CLEAR_TLS; + this.config = cfg; top = base = 1; - this.config = (clearThreadLocals) ? cfg | CLEAR_TLS : cfg; this.phase = id; this.owner = owner; } /** * Returns an exportable index (used by ForkJoinWorkerThread). - */final int getPoolIndex() { + */ + final int getPoolIndex() { return (phase & 0xffff) >>> 1; // ignore odd/even tag bit } @@ -1348,13 +1351,12 @@ private void growAndPush(ForkJoinTask task, ForkJoinTask[] a, private ForkJoinTask nextLocalTask(int fifo) { ForkJoinTask t = null; ForkJoinTask[] a = array; - int b = base, p = top, s = p - 1, cap; - if (p - b > 0 && a != null && (cap = a.length) > 0) { - int m = cap - 1, nb; - do { + int b = base, p = top, cap; + if (a != null && (cap = a.length) > 0) { + for (int m = cap - 1, s, nb; p - b > 0; ) { if (fifo == 0 || (nb = b + 1) == p) { if ((t = (ForkJoinTask)U.getAndSetReference( - a, slotOffset(m & s), null)) != null) + a, slotOffset(m & (s = p - 1)), null)) != null) updateTop(s); // else lost race for only task break; } @@ -1367,7 +1369,7 @@ a, slotOffset(m & b), null)) != null) { U.loadFence(); Thread.onSpinWait(); // spin to reduce memory traffic } - } while (p - b > 0); + } } return t; } @@ -1386,20 +1388,20 @@ final ForkJoinTask nextLocalTask() { * @param internal if caller owns this queue */ final boolean tryUnpush(ForkJoinTask task, boolean internal) { - boolean taken; + boolean taken = false; ForkJoinTask[] a = array; int p = top, s = p - 1, cap, k; if (a != null && (cap = a.length) > 0 && a[k = (cap - 1) & s] == task && (internal || tryLockPhase())) { - if (taken = (top == p && U.compareAndSetReference( - a, slotOffset(k), task, null))) + if (top == p && + U.compareAndSetReference(a, slotOffset(k), task, null)) { + taken = true; updateTop(s); + } if (!internal) unlockPhase(); } - else - taken = false; return taken; } @@ -1444,7 +1446,7 @@ final ForkJoinTask poll(ForkJoinPool pool) { else if (t == (o = U.compareAndExchangeReference( a, slotOffset(k), t, null))) { updateBase(nb); - if (pool != null && a[nk] != null) + if (a[nk] != null && pool != null) pool.signalWork(); // propagate return t; } @@ -1490,8 +1492,8 @@ final void topLevelExec(ForkJoinTask task, WorkQueue src, int srcId) { ThreadLocalRandom.eraseThreadLocals(Thread.currentThread()); while (task != null) { task.doExec(); - if ((task = nextLocalTask(fifo)) == null && - src != null && (task = src.tryPoll()) != null) + if ((task = nextLocalTask(fifo)) == null && src != null && + (task = src.tryPoll()) != null) ++nstolen; } nsteals = nstolen; @@ -1503,18 +1505,20 @@ final void topLevelExec(ForkJoinTask task, WorkQueue src, int srcId) { * runs task if present. */ final void tryRemoveAndExec(ForkJoinTask task, boolean internal) { - boolean taken = false; ForkJoinTask[] a = array; int b = base, p = top, s = p - 1, d = p - b, cap; - if (task != null && a != null && (cap = a.length) > 0 && d > 0) { - for (int i = s; ; --i) { - ForkJoinTask t; int k; - if ((t = a[k = i & (cap - 1)]) == task) { + if (a != null && (cap = a.length) > 0) { + for (int m = cap - 1, i = s; d > 0; --i, --d) { + ForkJoinTask t; int k; boolean taken; + if ((t = a[k = i & m]) == null) + break; + if (t == task) { long pos = slotOffset(k); if (!internal && !tryLockPhase()) break; // fail if locked - if (top == p && - U.compareAndSetReference(a, pos, task, null)) { + if (taken = + (top == p && + U.compareAndSetReference(a, pos, task, null))) { if (i == s) // act as pop updateTop(s); else if (i == base) // act as poll @@ -1523,21 +1527,18 @@ else if (i == base) // act as poll U.putReferenceVolatile( a, pos, (ForkJoinTask) U.getAndSetReference( - a, slotOffset(s & (cap - 1)), null)); + a, slotOffset(s & m), null)); updateTop(s); } - taken = true; } if (!internal) unlockPhase(); + if (taken) + task.doExec(); break; } - else if (t == null || --d == 0) - break; } } - if (taken) - task.doExec(); } /** @@ -1546,7 +1547,7 @@ else if (t == null || --d == 0) * * @param task root of computation * @param limit max runs, or zero for no limit - * @return task status if known done; else 0 + * @return task status if known to be done */ final int helpComplete(ForkJoinTask task, boolean internal, int limit) { int status = 0; @@ -1596,27 +1597,25 @@ final int helpComplete(ForkJoinTask task, boolean internal, int limit) { * @param blocker the blocker */ final void helpAsyncBlocker(ManagedBlocker blocker) { - if (blocker != null) { - for (;;) { - ForkJoinTask[] a; int b, cap, k; - if ((a = array) == null || (cap = a.length) <= 0) - break; - ForkJoinTask t = a[k = (b = base) & (cap - 1)]; - U.loadFence(); - if (t == null) { - if (top - b <= 0) - break; - } - else if (!(t instanceof CompletableFuture - .AsynchronousCompletionTask)) - break; - if (blocker.isReleasable()) + for (;;) { + ForkJoinTask[] a; int b, cap, k; + if ((a = array) == null || (cap = a.length) <= 0) + break; + ForkJoinTask t = a[k = (b = base) & (cap - 1)]; + U.loadFence(); + if (t == null) { + if (top - b <= 0) break; - if (base == b && t != null && - U.compareAndSetReference(a, slotOffset(k), t, null)) { - updateBase(b + 1); - t.doExec(); - } + } + else if (!(t instanceof CompletableFuture + .AsynchronousCompletionTask)) + break; + if (blocker != null && blocker.isReleasable()) + break; + if (base == b && t != null && + U.compareAndSetReference(a, slotOffset(k), t, null)) { + updateBase(b + 1); + t.doExec(); } } } @@ -1748,12 +1747,10 @@ private int spinLockRunState() { // spin/sleep if (casRunState(s, u = s + RS_LOCK)) return u; waits = 0; - } - else if (waits < SPIN_WAITS) { + } else if (waits < SPIN_WAITS) { ++waits; Thread.onSpinWait(); - } - else { + } else { if (waits < MIN_SLEEP) waits = MIN_SLEEP; LockSupport.parkNanos(this, (long)waits); @@ -1919,19 +1916,16 @@ final void signalWork() { int sp = (int)c, i = sp & SMASK; if (qs == null || qs.length <= i) break; - WorkQueue v = null, w = qs[i]; - if ((short)(c >>> RC_SHIFT) >= pc) - break; + WorkQueue w = qs[i], v = null; if (sp == 0) { if ((short)(c >>> TC_SHIFT) >= pc) break; nc = ((c + TC_UNIT) & TC_MASK); } - else { - if ((v = w) == null) - break; + else if ((short)(c >>> RC_SHIFT) >= pc || (v = w) == null) + break; + else nc = (v.stackPred & LMASK) | (c & TC_MASK); - } if (c == (c = compareAndExchangeCtl(c, nc | ac))) { if (v == null) createWorker(); @@ -2002,7 +1996,7 @@ else if (possibleSubmission) { // cover signal race } else if (ctl != c) // re-snapshot ; - else if ((e == prevRunState && (e & RS_LOCK) == 0 && swept) && + else if (e == prevRunState && (e & RS_LOCK) == 0 && swept && (!transition || casRunState(e, ((e & SHUTDOWN) == 0 ? e + RS_EPOCH : // advance @@ -2155,7 +2149,7 @@ private int awaitWork(WorkQueue w, long queuedCtl, int idlePhase) { Thread.onSpinWait(); } while ((p = w.phase) == idlePhase && --spins >= 0); - if (spins < 0) { // emulate LockSupport.park + if (p == idlePhase) { // emulate LockSupport.park LockSupport.setCurrentBlocker(this); w.parker = Thread.currentThread(); for (;;) { @@ -2325,7 +2319,7 @@ else if (t != null) { // check steal chain if ((v & 1) == 0 || // external or none --d < 0 || // bound depth (p = qs[v & (n - 1)]) == null) - break; // stale + break; v = p.source; } } @@ -2551,7 +2545,7 @@ else if ((t = pollScan(false)) != null) { waits = 0; t.doExec(); } - else if (isQuiescent(false)) + else if (isQuiescent(true)) break; else if (System.nanoTime() - startTime > nanos) return 0; @@ -2665,7 +2659,7 @@ final WorkQueue externalSubmissionQueue() { WorkQueue[] qs; WorkQueue q; int n; int r = ThreadLocalRandom.getProbe(); return (((qs = queues) != null && (n = qs.length) > 0 && - (q = qs[r & EXTERNAL_ID_MASK & (n - 1)]) != null&& r != 0 && + (q = qs[r & EXTERNAL_ID_MASK & (n - 1)]) != null && r != 0 && q.tryLockPhase()) ? q : submissionQueue(r)); } @@ -2696,10 +2690,9 @@ static WorkQueue commonQueue() { */ static void helpAsyncBlocker(Executor e, ManagedBlocker blocker) { WorkQueue w = null; Thread t; ForkJoinWorkerThread wt; - if ((t = Thread.currentThread()) instanceof ForkJoinWorkerThread) { - if ((wt = (ForkJoinWorkerThread)t).pool == e) - w = wt.workQueue; - } + if (((t = Thread.currentThread()) instanceof ForkJoinWorkerThread) && + (wt = (ForkJoinWorkerThread)t).pool == e) + w = wt.workQueue; else if (e instanceof ForkJoinPool) w = externalQueue((ForkJoinPool)e); if (w != null) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index 6614e95b398f2..a6f603331f286 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -424,7 +424,10 @@ else if (Thread.interrupted()) { interrupts = interruptible ? -1 : 1; else { interrupts = 1; // re-assert if cleared - cancelIgnoringExceptions(this); + try { + cancel(true); + } catch (Throwable ignore) { + } } } else if (deadline != 0L) { @@ -608,22 +611,6 @@ final int getForkJoinTaskStatusMarkerBit() { return status & MARKER; } - /** - * Cancels with mayInterruptIfRunning, ignoring any exceptions - * thrown by cancel. Used only when cancelling tasks upon pool or - * worker thread termination. Cancel is spec'ed not to throw any - * exceptions, but if it does anyway, we have no recourse, so - * guard against this case. - */ - static final void cancelIgnoringExceptions(ForkJoinTask t) { - if (t != null) { - try { - t.cancel(true); - } catch (Throwable ignore) { - } - } - } - // public methods /** diff --git a/test/jdk/ProblemList.txt b/test/jdk/ProblemList.txt index e8aae25f5e064..5d86cfb9db0a9 100644 --- a/test/jdk/ProblemList.txt +++ b/test/jdk/ProblemList.txt @@ -713,8 +713,6 @@ com/sun/jdi/InvokeHangTest.java 8218463 linux-al java/util/Locale/LocaleProvidersRun.java 8268379 macosx-x64 sun/util/locale/provider/CalendarDataRegression.java 8268379 macosx-x64 -java/util/concurrent/forkjoin/AsyncShutdownNow.java 8286352 linux-all,windows-x64 -java/util/concurrent/ExecutorService/CloseTest.java 8288899 macosx-aarch64 ############################################################################ diff --git a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java index 193a467735448..0c5b6d2f6e970 100644 --- a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java +++ b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java @@ -23,9 +23,9 @@ /* * @test - * @summary Test ExecutorService.close, including default implementation + * @summary Test implementations of ExecutorService.close * @library ../lib - * @run testng CloseTest + * @run junit CloseTest */ import java.time.Duration; @@ -37,37 +37,32 @@ import java.util.concurrent.Future; import java.util.concurrent.Phaser; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; -import static org.testng.Assert.*; - -public class CloseTest { - - @DataProvider(name = "executors") - public Object[][] executors() { - var defaultThreadFactory = Executors.defaultThreadFactory(); - var virtualThreadFactory = Thread.ofVirtual().factory(); - return new Object[][] { - // ensures that default close method is tested - { new DelegatingExecutorService(Executors.newCachedThreadPool()), }, - - // implementations that may override close - { new ForkJoinPool(), }, - { Executors.newFixedThreadPool(1), }, - { Executors.newCachedThreadPool(), }, - { Executors.newThreadPerTaskExecutor(defaultThreadFactory), }, - { Executors.newThreadPerTaskExecutor(virtualThreadFactory), }, - }; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import static org.junit.jupiter.api.Assertions.*; + +class CloseTest { + + static Stream executors() { + return Stream.of( + // ensures that default close method is tested + new DelegatingExecutorService(Executors.newCachedThreadPool()), + + // implementations that may override close + Executors.newCachedThreadPool(), + Executors.newVirtualThreadPerTaskExecutor(), + new ForkJoinPool() + ); } /** * Test close with no tasks running. */ - @Test(dataProvider = "executors") - public void testCloseWithNoTasks(ExecutorService executor) throws Exception { + @ParameterizedTest + @MethodSource("executors") + void testCloseWithNoTasks(ExecutorService executor) throws Exception { executor.close(); assertTrue(executor.isShutdown()); assertTrue(executor.isTerminated()); @@ -77,8 +72,9 @@ public void testCloseWithNoTasks(ExecutorService executor) throws Exception { /** * Test close with tasks running. */ - @Test(dataProvider = "executors") - public void testCloseWithRunningTasks(ExecutorService executor) throws Exception { + @ParameterizedTest + @MethodSource("executors") + void testCloseWithRunningTasks(ExecutorService executor) throws Exception { Future future = executor.submit(() -> { Thread.sleep(Duration.ofMillis(100)); return "foo"; @@ -87,14 +83,15 @@ public void testCloseWithRunningTasks(ExecutorService executor) throws Exception assertTrue(executor.isShutdown()); assertTrue(executor.isTerminated()); assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - assertEquals(future.resultNow(), "foo"); + assertEquals("foo", future.resultNow()); } /** * Test close when executor is shutdown but not terminated. */ - @Test(dataProvider = "executors") - public void testShutdownBeforeClose(ExecutorService executor) throws Exception { + @ParameterizedTest + @MethodSource("executors") + void testShutdownBeforeClose(ExecutorService executor) throws Exception { Phaser phaser = new Phaser(2); Future future = executor.submit(() -> { phaser.arriveAndAwaitAdvance(); @@ -109,14 +106,15 @@ public void testShutdownBeforeClose(ExecutorService executor) throws Exception { assertTrue(executor.isShutdown()); assertTrue(executor.isTerminated()); assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - assertEquals(future.resultNow(), "foo"); + assertEquals("foo", future.resultNow()); } /** * Test close when terminated. */ - @Test(dataProvider = "executors") - public void testTerminateBeforeClose(ExecutorService executor) throws Exception { + @ParameterizedTest + @MethodSource("executors") + void testTerminateBeforeClose(ExecutorService executor) throws Exception { executor.shutdown(); assertTrue(executor.isTerminated()); @@ -129,8 +127,9 @@ public void testTerminateBeforeClose(ExecutorService executor) throws Exception /** * Test invoking close with interrupt status set. */ - @Test(dataProvider = "executors") - public void testInterruptBeforeClose(ExecutorService executor) throws Exception { + @ParameterizedTest + @MethodSource("executors") + void testInterruptBeforeClose(ExecutorService executor) throws Exception { Phaser phaser = new Phaser(2); Future future = executor.submit(() -> { phaser.arriveAndAwaitAdvance(); @@ -149,23 +148,30 @@ public void testInterruptBeforeClose(ExecutorService executor) throws Exception assertTrue(executor.isShutdown()); assertTrue(executor.isTerminated()); assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - expectThrows(ExecutionException.class, future::get); + assertThrows(ExecutionException.class, future::get); } /** * Test interrupting thread blocked in close. */ - @Test(dataProvider = "executors") - public void testInterruptDuringClose(ExecutorService executor) throws Exception { + @ParameterizedTest + @MethodSource("executors") + void testInterruptDuringClose(ExecutorService executor) throws Exception { + Phaser phaser = new Phaser(2); Future future = executor.submit(() -> { + phaser.arriveAndAwaitAdvance(); Thread.sleep(Duration.ofDays(1)); return null; }); + phaser.arriveAndAwaitAdvance(); // wait for task to start + + // schedule main thread to be interrupted Thread thread = Thread.currentThread(); new Thread(() -> { - try { Thread.sleep( Duration.ofMillis(500)); } catch (Exception ignore) { } + try { Thread.sleep( Duration.ofMillis(100)); } catch (Exception ignore) { } thread.interrupt(); }).start(); + try { executor.close(); assertTrue(Thread.currentThread().isInterrupted()); @@ -175,6 +181,6 @@ public void testInterruptDuringClose(ExecutorService executor) throws Exception assertTrue(executor.isShutdown()); assertTrue(executor.isTerminated()); assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - expectThrows(ExecutionException.class, future::get); + assertThrows(ExecutionException.class, future::get); } } diff --git a/test/jdk/java/util/concurrent/ExecutorService/InvokeTest.java b/test/jdk/java/util/concurrent/ExecutorService/InvokeTest.java new file mode 100644 index 0000000000000..a9a4eb0d8d78e --- /dev/null +++ b/test/jdk/java/util/concurrent/ExecutorService/InvokeTest.java @@ -0,0 +1,787 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @summary Test implementations of ExecutorService.invokeAll/invokeAny + * @run junit InvokeTest + */ + +import java.time.Duration; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; +import static java.lang.Thread.State.*; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import static org.junit.jupiter.api.Assertions.*; + +class InvokeTest { + + private static ScheduledExecutorService scheduler; + + @BeforeAll + static void setup() throws Exception { + scheduler = Executors.newSingleThreadScheduledExecutor(); + } + + @AfterAll + static void shutdown() { + scheduler.shutdown(); + } + + private static Stream executors() { + return Stream.of( + Executors.newCachedThreadPool(), + Executors.newVirtualThreadPerTaskExecutor(), + new ForkJoinPool() + ); + } + + /** + * Test invokeAny where all tasks complete normally. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAny1(ExecutorService executor) throws Exception { + try (executor) { + Callable task1 = () -> "foo"; + Callable task2 = () -> "bar"; + String result = executor.invokeAny(List.of(task1, task2)); + assertTrue(Set.of("foo", "bar").contains(result)); + } + } + + /** + * Test invokeAny where all tasks complete normally. The completion of the + * first task should cancel remaining tasks. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAny2(ExecutorService executor) throws Exception { + try (executor) { + Callable task1 = () -> "foo"; + + var task2Started = new AtomicBoolean(); + var task2Interrupted = new CountDownLatch(1); + Callable task2 = () -> { + task2Started.set(true); + try { + Thread.sleep(Duration.ofDays(1)); + } catch (InterruptedException e) { + task2Interrupted.countDown(); + } + return null; + }; + + String result = executor.invokeAny(List.of(task1, task2)); + assertEquals("foo", result); + + // if task2 started then it should have been interrupted + if (task2Started.get()) { + task2Interrupted.await(); + } + } + } + + /** + * Test invokeAny where all tasks complete with exception. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAny3(ExecutorService executor) throws Exception { + try (executor) { + class FooException extends Exception { } + Callable task1 = () -> { throw new FooException(); }; + Callable task2 = () -> { throw new FooException(); }; + try { + executor.invokeAny(List.of(task1, task2)); + fail("invokeAny did not throw"); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + assertTrue(cause instanceof FooException); + } + } + } + + /** + * Test invokeAny where all tasks complete with exception. The completion + * of the last task is delayed. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAny4(ExecutorService executor) throws Exception { + try (executor) { + class FooException extends Exception { } + Callable task1 = () -> { throw new FooException(); }; + Callable task2 = () -> { + Thread.sleep(Duration.ofMillis(50)); + throw new FooException(); + }; + try { + executor.invokeAny(List.of(task1, task2)); + fail("invokeAny did not throw"); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + assertTrue(cause instanceof FooException); + } + } + } + + /** + * Test invokeAny where some, not all, tasks complete normally. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAny5(ExecutorService executor) throws Exception { + try (executor) { + class FooException extends Exception { } + Callable task1 = () -> "foo"; + Callable task2 = () -> { throw new FooException(); }; + String result = executor.invokeAny(List.of(task1, task2)); + assertEquals("foo", result); + } + } + + /** + * Test invokeAny where some, not all, tasks complete normally. The first + * task to complete normally is delayed. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAny6(ExecutorService executor) throws Exception { + try (executor) { + class FooException extends Exception { } + Callable task1 = () -> { + Thread.sleep(Duration.ofMillis(50)); + return "foo"; + }; + Callable task2 = () -> { throw new FooException(); }; + String result = executor.invokeAny(List.of(task1, task2)); + assertEquals("foo", result); + } + } + + /** + * Test timed-invokeAny where all tasks complete normally before the timeout. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAnyWithTimeout1(ExecutorService executor) throws Exception { + try (executor) { + Callable task1 = () -> "foo"; + Callable task2 = () -> "bar"; + String result = executor.invokeAny(List.of(task1, task2), 1, TimeUnit.MINUTES); + assertTrue(Set.of("foo", "bar").contains(result)); + } + } + + /** + * Test timed-invokeAny where one task completes normally before the timeout. + * The remaining tests should be cancelled. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAnyWithTimeout2(ExecutorService executor) throws Exception { + try (executor) { + Callable task1 = () -> "foo"; + + var task2Started = new AtomicBoolean(); + var task2Interrupted = new CountDownLatch(1); + Callable task2 = () -> { + task2Started.set(true); + try { + Thread.sleep(Duration.ofDays(1)); + } catch (InterruptedException e) { + task2Interrupted.countDown(); + } + return null; + }; + + String result = executor.invokeAny(List.of(task1, task2), 1, TimeUnit.MINUTES); + assertEquals("foo", result); + + // if task2 started then it should have been interrupted + if (task2Started.get()) { + task2Interrupted.await(); + } + } + } + + /** + * Test timed-invokeAny where timeout expires before any task completes. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAnyWithTimeout3(ExecutorService executor) throws Exception { + try (executor) { + var task1Started = new AtomicBoolean(); + var task1Interrupted = new CountDownLatch(1); + Callable task1 = () -> { + task1Started.set(true); + try { + Thread.sleep(Duration.ofDays(1)); + } catch (InterruptedException e) { + task1Interrupted.countDown(); + } + return null; + }; + + var task2Started = new AtomicBoolean(); + var task2Interrupted = new CountDownLatch(1); + Callable task2 = () -> { + task2Started.set(true); + try { + Thread.sleep(Duration.ofDays(1)); + } catch (InterruptedException e) { + task2Interrupted.countDown(); + } + return null; + }; + + // invokeAny should throw TimeoutException + assertThrows(TimeoutException.class, + () -> executor.invokeAny(List.of(task1, task2), 100, TimeUnit.MILLISECONDS)); + + // tasks that started should be interrupted + if (task1Started.get()) { + task1Interrupted.await(); + } + if (task2Started.get()) { + task2Interrupted.await(); + } + } + } + + /** + * Test invokeAny with interrupt status set. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAnyWithInterruptSet(ExecutorService executor) throws Exception { + try (executor) { + Callable task1 = () -> { + Thread.sleep(Duration.ofMinutes(1)); + return "foo"; + }; + Callable task2 = () -> { + Thread.sleep(Duration.ofMinutes(1)); + return "bar"; + }; + Thread.currentThread().interrupt(); + try { + executor.invokeAny(List.of(task1, task2)); + fail("invokeAny did not throw"); + } catch (InterruptedException expected) { + assertFalse(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); // clear interrupt + } + } + } + + /** + * Test interrupting a thread blocked in invokeAny. + */ + @ParameterizedTest + @MethodSource("executors") + void testInterruptInvokeAny(ExecutorService executor) throws Exception { + try (executor) { + var task1Started = new AtomicBoolean(); + var task1Interrupted = new CountDownLatch(1); + Callable task1 = () -> { + task1Started.set(true); + try { + Thread.sleep(Duration.ofDays(1)); + } catch (InterruptedException e) { + task1Interrupted.countDown(); + } + return null; + }; + + var task2Started = new AtomicBoolean(); + var task2Interrupted = new CountDownLatch(1); + Callable task2 = () -> { + task2Started.set(true); + try { + Thread.sleep(Duration.ofDays(1)); + } catch (InterruptedException e) { + task2Interrupted.countDown(); + } + return null; + }; + + scheduleInterruptAt("invokeAny"); + try { + executor.invokeAny(List.of(task1, task2)); + fail("invokeAny did not throw"); + } catch (InterruptedException expected) { + assertFalse(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); // clear interrupt + } + + // tasks that started should be interrupted + if (task1Started.get()) { + task1Interrupted.await(); + } + if (task2Started.get()) { + task2Interrupted.await(); + } + } + } + + /** + * Test invokeAny after ExecutorService has been shutdown. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAnyAfterShutdown(ExecutorService executor) throws Exception { + executor.shutdown(); + Callable task1 = () -> "foo"; + Callable task2 = () -> "bar"; + assertThrows(RejectedExecutionException.class, + () -> executor.invokeAny(List.of(task1, task2))); + } + + /** + * Test invokeAny with empty collection. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAnyEmpty1(ExecutorService executor) throws Exception { + try (executor) { + assertThrows(IllegalArgumentException.class, () -> executor.invokeAny(List.of())); + } + } + + /** + * Test timed-invokeAny with empty collection. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAnyEmpty2(ExecutorService executor) throws Exception { + try (executor) { + assertThrows(IllegalArgumentException.class, + () -> executor.invokeAny(List.of(), 1, TimeUnit.MINUTES)); + } + } + + /** + * Test invokeAny with null. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAnyNull1(ExecutorService executor)throws Exception { + try (executor) { + assertThrows(NullPointerException.class, () -> executor.invokeAny(null)); + } + } + + /** + * Test invokeAny with null element + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAnyNull2(ExecutorService executor)throws Exception { + try (executor) { + List> list = new ArrayList<>(); + list.add(() -> "foo"); + list.add(null); + assertThrows(NullPointerException.class, () -> executor.invokeAny(null)); + } + } + + /** + * Test invokeAll where all tasks complete normally. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAll1(ExecutorService executor) throws Exception { + try (executor) { + Callable task1 = () -> "foo"; + Callable task2 = () -> { + Thread.sleep(Duration.ofMillis(50)); + return "bar"; + }; + + List> futures = executor.invokeAll(List.of(task1, task2)); + assertTrue(futures.size() == 2); + + // check results + List results = futures.stream().map(Future::resultNow).toList(); + assertEquals(results, List.of("foo", "bar")); + } + } + + /** + * Test invokeAll where all tasks complete with exception. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAll2(ExecutorService executor) throws Exception { + try (executor) { + class FooException extends Exception { } + class BarException extends Exception { } + Callable task1 = () -> { throw new FooException(); }; + Callable task2 = () -> { + Thread.sleep(Duration.ofMillis(50)); + throw new BarException(); + }; + + List> futures = executor.invokeAll(List.of(task1, task2)); + assertTrue(futures.size() == 2); + + // check results + Throwable e1 = assertThrows(ExecutionException.class, () -> futures.get(0).get()); + assertTrue(e1.getCause() instanceof FooException); + Throwable e2 = assertThrows(ExecutionException.class, () -> futures.get(1).get()); + assertTrue(e2.getCause() instanceof BarException); + } + } + + /** + * Test invokeAll where all tasks complete normally before the timeout expires. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAll3(ExecutorService executor) throws Exception { + try (executor) { + Callable task1 = () -> "foo"; + Callable task2 = () -> { + Thread.sleep(Duration.ofMillis(50)); + return "bar"; + }; + + List> futures = executor.invokeAll(List.of(task1, task2), 1, TimeUnit.MINUTES); + assertTrue(futures.size() == 2); + + // check results + List results = futures.stream().map(Future::resultNow).toList(); + assertEquals(results, List.of("foo", "bar")); + } + } + + /** + * Test invokeAll where some tasks do not complete before the timeout expires. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAll4(ExecutorService executor) throws Exception { + try (executor) { + Callable task1 = () -> "foo"; + + var task2Started = new AtomicBoolean(); + var task2Interrupted = new CountDownLatch(1); + Callable task2 = () -> { + task2Started.set(true); + try { + Thread.sleep(Duration.ofDays(1)); + } catch (InterruptedException e) { + task2Interrupted.countDown(); + } + return null; + }; + + List> futures = executor.invokeAll(List.of(task1, task2), 1, TimeUnit.SECONDS); + assertTrue(futures.size() == 2); + + // task1 should be done + assertTrue(futures.get(0).isDone()); + + // task2 should be cancelled and interrupted + assertTrue(futures.get(1).isCancelled()); + + // if task2 started then it should have been interrupted + if (task2Started.get()) { + task2Interrupted.await(); + } + } + } + + /** + * Test invokeAll with interrupt status set. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAllInterrupt1(ExecutorService executor) throws Exception { + try (executor) { + Callable task1 = () -> "foo"; + Callable task2 = () -> { + Thread.sleep(Duration.ofMinutes(1)); + return "bar"; + }; + + Thread.currentThread().interrupt(); + try { + executor.invokeAll(List.of(task1, task2)); + fail("invokeAll did not throw"); + } catch (InterruptedException expected) { + assertFalse(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); // clear interrupt + } + } + } + + /** + * Test timed-invokeAll with interrupt status set. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAllInterrupt3(ExecutorService executor) throws Exception { + try (executor) { + Callable task1 = () -> "foo"; + Callable task2 = () -> { + Thread.sleep(Duration.ofMinutes(1)); + return "bar"; + }; + + Thread.currentThread().interrupt(); + try { + executor.invokeAll(List.of(task1, task2), 1, TimeUnit.MINUTES); + fail("invokeAll did not throw"); + } catch (InterruptedException expected) { + assertFalse(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); // clear interrupt + } + } + } + + /** + * Test interrupt with thread blocked in invokeAll. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAllInterrupt4(ExecutorService executor) throws Exception { + try (executor) { + Callable task1 = () -> "foo"; + + var task2Started = new AtomicBoolean(); + var task2Interrupted = new CountDownLatch(1); + Callable task2 = () -> { + task2Started.set(true); + try { + Thread.sleep(Duration.ofDays(1)); + } catch (InterruptedException e) { + task2Interrupted.countDown(); + } + return null; + }; + + scheduleInterruptAt("invokeAll"); + try { + executor.invokeAll(List.of(task1, task2)); + fail("invokeAll did not throw"); + } catch (InterruptedException expected) { + assertFalse(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); // clear interrupt + } + + // if task2 started then it should have been interrupted + if (task2Started.get()) { + task2Interrupted.await(); + } + } + } + + /** + * Test interrupt with thread blocked in timed-invokeAll. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAllInterrupt6(ExecutorService executor) throws Exception { + try (executor) { + Callable task1 = () -> "foo"; + + var task2Started = new AtomicBoolean(); + var task2Interrupted = new CountDownLatch(1); + Callable task2 = () -> { + task2Started.set(true); + try { + Thread.sleep(Duration.ofDays(1)); + } catch (InterruptedException e) { + task2Interrupted.countDown(); + } + return null; + }; + + scheduleInterruptAt("invokeAll"); + try { + executor.invokeAll(List.of(task1, task2), 1, TimeUnit.MINUTES); + fail("invokeAll did not throw"); + } catch (InterruptedException expected) { + assertFalse(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); // clear interrupt + } + + // if task2 started then it should have been interrupted + if (task2Started.get()) { + task2Interrupted.await(); + } + } + } + + /** + * Test invokeAll after ExecutorService has been shutdown. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAllAfterShutdown1(ExecutorService executor) throws Exception { + executor.shutdown(); + + Callable task1 = () -> "foo"; + Callable task2 = () -> "bar"; + assertThrows(RejectedExecutionException.class, + () -> executor.invokeAll(List.of(task1, task2))); + } + + @ParameterizedTest + @MethodSource("executors") + void testInvokeAllAfterShutdown2(ExecutorService executor) throws Exception { + executor.shutdown(); + + Callable task1 = () -> "foo"; + Callable task2 = () -> "bar"; + assertThrows(RejectedExecutionException.class, + () -> executor.invokeAll(List.of(task1, task2), 1, TimeUnit.SECONDS)); + } + + /** + * Test invokeAll with empty collection. + */ + @ParameterizedTest + @MethodSource("executors") + void testInvokeAllEmpty1(ExecutorService executor) throws Exception { + try (executor) { + List> list = executor.invokeAll(List.of()); + assertTrue(list.size() == 0); + } + } + + @ParameterizedTest + @MethodSource("executors") + void testInvokeAllEmpty2(ExecutorService executor) throws Exception { + try (executor) { + List> list = executor.invokeAll(List.of(), 1, TimeUnit.SECONDS); + assertTrue(list.size() == 0); + } + } + + @ParameterizedTest + @MethodSource("executors") + void testInvokeAllNull1(ExecutorService executor)throws Exception { + try (executor) { + assertThrows(NullPointerException.class, () -> executor.invokeAll(null)); + } + } + + @ParameterizedTest + @MethodSource("executors") + void testInvokeAllNull2(ExecutorService executor)throws Exception { + try (executor) { + List> tasks = new ArrayList<>(); + tasks.add(() -> "foo"); + tasks.add(null); + assertThrows(NullPointerException.class, () -> executor.invokeAll(tasks)); + } + } + + @ParameterizedTest + @MethodSource("executors") + void testInvokeAllNull3(ExecutorService executor)throws Exception { + try (executor) { + assertThrows(NullPointerException.class, + () -> executor.invokeAll(null, 1, TimeUnit.SECONDS)); + } + } + + @ParameterizedTest + @MethodSource("executors") + void testInvokeAllNull4(ExecutorService executor)throws Exception { + try (executor) { + Callable task = () -> "foo"; + assertThrows(NullPointerException.class, + () -> executor.invokeAll(List.of(task), 1, null)); + } + } + + @ParameterizedTest + @MethodSource("executors") + void testInvokeAllNull5(ExecutorService executor)throws Exception { + try (executor) { + List> tasks = new ArrayList<>(); + tasks.add(() -> "foo"); + tasks.add(null); + assertThrows(NullPointerException.class, + () -> executor.invokeAll(tasks, 1, TimeUnit.SECONDS)); + } + } + + /** + * Schedules the current thread to be interrupted when it waits (timed or untimed) + * at the given method name. + */ + private void scheduleInterruptAt(String methodName) { + Thread target = Thread.currentThread(); + scheduler.submit(() -> { + try { + boolean found = false; + while (!found) { + Thread.State state = target.getState(); + assertTrue(state != TERMINATED); + if ((state == WAITING || state == TIMED_WAITING) + && contains(target.getStackTrace(), methodName)) { + found = true; + } else { + Thread.sleep(20); + } + } + target.interrupt(); + } catch (Exception e) { + e.printStackTrace(); + } + }); + } + + /** + * Returns true if the given stack trace contains an element for the given method name. + */ + private boolean contains(StackTraceElement[] stack, String methodName) { + return Arrays.stream(stack) + .anyMatch(e -> methodName.equals(e.getMethodName())); + } +} diff --git a/test/jdk/java/util/concurrent/ExecutorService/SubmitTest.java b/test/jdk/java/util/concurrent/ExecutorService/SubmitTest.java new file mode 100644 index 0000000000000..026d86a6c8501 --- /dev/null +++ b/test/jdk/java/util/concurrent/ExecutorService/SubmitTest.java @@ -0,0 +1,370 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @summary Test implementations of ExecutorService.submit/execute + * @run junit SubmitTest + */ + +import java.time.Duration; +import java.util.concurrent.*; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import static org.junit.jupiter.api.Assertions.*; + +class SubmitTest { + + private static Stream executors() { + return Stream.of( + Executors.newCachedThreadPool(), + Executors.newVirtualThreadPerTaskExecutor(), + new ForkJoinPool() + ); + } + + /** + * Test submit(Runnable) executes the task. + */ + @ParameterizedTest + @MethodSource("executors") + void testSubmitRunnable(ExecutorService executor) throws Exception { + try (executor) { + var latch = new CountDownLatch(1); + Future future = executor.submit(latch::countDown); + latch.await(); + assertNull(future.get()); + } + } + + /** + * Test submit(Runnable) throws if executor is shutdown. + */ + @ParameterizedTest + @MethodSource("executors") + void testSubmitRunnableAfterShutdown(ExecutorService executor) { + executor.shutdown(); + assertThrows(RejectedExecutionException.class, () -> executor.submit(() -> { })); + } + + /** + * Test task submitted with submit(Runnable) is not interrupted by cancel(false). + */ + @ParameterizedTest + @MethodSource("executors") + void testSubmitRunnableWithCancelFalse(ExecutorService executor) throws Exception { + try (executor) { + var started = new CountDownLatch(1); + var stop = new CountDownLatch(1); + var done = new CountDownLatch(1); + Future future = executor.submit(() -> { + started.countDown(); + try { + stop.await(); + } catch (InterruptedException e) { + // ignore + } finally { + done.countDown(); + } + }); + + // wait for task to start + started.await(); + + // cancel(false), task should not be interrupted + future.cancel(false); + assertFalse(done.await(500, TimeUnit.MILLISECONDS)); + + // let task finish + stop.countDown(); + } + } + + /** + * Test task submitted with submit(Runnable) is interrupted by cancel(true). + */ + @ParameterizedTest + @MethodSource("executors") + void testSubmitRunnableWithCancelTrue(ExecutorService executor) throws Exception { + try (executor) { + var started = new CountDownLatch(1); + var interrupted = new CountDownLatch(1); + Future future = executor.submit(() -> { + started.countDown(); + try { + Thread.sleep(Duration.ofDays(1)); + } catch (InterruptedException e) { + interrupted.countDown(); + } + }); + + // wait for task to start + started.await(); + + // cancel(true), task should be interrupted + future.cancel(true); + interrupted.await(); + } + } + + /** + * Test task submitted with submit(Runnable) is interrupted if executor is + * stopped with shutdownNow. + */ + @ParameterizedTest + @MethodSource("executors") + void testSubmitRunnableWithShutdownNow(ExecutorService executor) throws Exception { + try (executor) { + var started = new CountDownLatch(1); + var interrupted = new CountDownLatch(1); + Future future = executor.submit(() -> { + started.countDown(); + try { + Thread.sleep(Duration.ofDays(1)); + } catch (InterruptedException e) { + interrupted.countDown(); + } + }); + + // wait for task to start + started.await(); + + // shutdown forcefully, task should be interrupted + executor.shutdownNow(); + interrupted.await(); + } + } + + /** + * Test submit(Runnable) throws if task is null. + */ + @ParameterizedTest + @MethodSource("executors") + void testSubmitRunnableNull(ExecutorService executor) { + try (executor) { + Runnable nullTask = null; + assertThrows(NullPointerException.class, () -> executor.submit(nullTask)); + assertThrows(NullPointerException.class, () -> executor.submit(nullTask, Void.class)); + } + } + + // + + /** + * Test submit(Callable) executes the task. + */ + @ParameterizedTest + @MethodSource("executors") + void testSubmitCallable(ExecutorService executor) throws Exception { + try (executor) { + var latch = new CountDownLatch(1); + Future future = executor.submit(() -> { + latch.countDown(); + return "foo"; + }); + latch.await(); + assertEquals("foo", future.get()); + } + } + + /** + * Test submit(Callable) throws if executor is shutdown. + */ + @ParameterizedTest + @MethodSource("executors") + void testSubmitCallableAfterShutdown(ExecutorService executor) { + executor.shutdown(); + assertThrows(RejectedExecutionException.class, () -> executor.submit(() -> null)); + } + + /** + * Test task submitted with submit(Callable) is not interrupted by cancel(false). + */ + @ParameterizedTest + @MethodSource("executors") + void testSubmitCallableWithCancelFalse(ExecutorService executor) throws Exception { + try (executor) { + var started = new CountDownLatch(1); + var stop = new CountDownLatch(1); + var done = new CountDownLatch(1); + Future future = executor.submit(() -> { + started.countDown(); + try { + stop.await(); + } finally { + done.countDown(); + } + return null; + }); + + // wait for task to start + started.await(); + + // cancel(false), task should not be interrupted + future.cancel(false); + assertFalse(done.await(500, TimeUnit.MILLISECONDS)); + + // let task finish + stop.countDown(); + } + } + + /** + * Test task submitted with submit(Callable) is interrupted by cancel(true). + */ + @ParameterizedTest + @MethodSource("executors") + void testSubmitCallableWithCancelTrue(ExecutorService executor) throws Exception { + try (executor) { + var started = new CountDownLatch(1); + var interrupted = new CountDownLatch(1); + Future future = executor.submit(() -> { + started.countDown(); + try { + Thread.sleep(Duration.ofDays(1)); + } catch (InterruptedException e) { + interrupted.countDown(); + } + return null; + }); + + // wait for task to start + started.await(); + + // cancel(true), task should be interrupted + future.cancel(true); + interrupted.await(); + } + } + + /** + * Test task submitted with submit(Callable) is interrupted if executor is + * stopped with shutdownNow. + */ + @ParameterizedTest + @MethodSource("executors") + void testSubmitCallableWithShutdownNow(ExecutorService executor) throws Exception { + try (executor) { + var started = new CountDownLatch(1); + var interrupted = new CountDownLatch(1); + Future future = executor.submit(() -> { + started.countDown(); + try { + Thread.sleep(Duration.ofDays(1)); + } catch (InterruptedException e) { + interrupted.countDown(); + } + return null; + }); + + // wait for task to start + started.await(); + + // shutdown forcefully, task should be interrupted + executor.shutdownNow(); + interrupted.await(); + } + } + + /** + * Test submit(Callable) throws if task is null. + */ + @ParameterizedTest + @MethodSource("executors") + void testSubmitCallableNull(ExecutorService executor) { + try (executor) { + Callable nullTask = null; + assertThrows(NullPointerException.class, () -> executor.submit(nullTask)); + } + } + + // + + /** + * Test execute(Runnable) executes the task. + */ + @ParameterizedTest + @MethodSource("executors") + void testExecute(ExecutorService executor) throws Exception { + try (executor) { + var latch = new CountDownLatch(1); + executor.execute(latch::countDown); + latch.await(); + } + } + + /** + * Test execute(Runnable) throws if executor is shutdown. + */ + @ParameterizedTest + @MethodSource("executors") + void testExecuteAfterShutdown(ExecutorService executor) { + executor.shutdown(); + assertThrows(RejectedExecutionException.class, () -> executor.execute(() -> { })); + } + + /** + * Test task submitted with execute(Runnable) is interrupted if executor is + * stopped with shutdownNow. + */ + @ParameterizedTest + @MethodSource("executors") + void testExecuteWithShutdownNow(ExecutorService executor) throws Exception { + try (executor) { + var started = new CountDownLatch(1); + var interrupted = new CountDownLatch(1); + executor.execute(() -> { + started.countDown(); + try { + Thread.sleep(Duration.ofDays(1)); + } catch (InterruptedException e) { + interrupted.countDown(); + } + }); + + // wait for task to start + started.await(); + + // shutdown forcefully, task should be interrupted + executor.shutdownNow(); + interrupted.await(); + } + } + + /** + * Test execute(Runnable) throws if task is null. + */ + @ParameterizedTest + @MethodSource("executors") + void testExecuteNull(ExecutorService executor) { + try (executor) { + Runnable nullTask = null; + assertThrows(NullPointerException.class, () -> executor.execute(nullTask)); + } + } +} diff --git a/test/jdk/java/util/concurrent/Future/DefaultMethods.java b/test/jdk/java/util/concurrent/Future/DefaultMethods.java index 2571c56804322..dfeb5bde64e0d 100644 --- a/test/jdk/java/util/concurrent/Future/DefaultMethods.java +++ b/test/jdk/java/util/concurrent/Future/DefaultMethods.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,42 +25,50 @@ * @test * @summary Test Future's default methods * @library ../lib - * @run testng DefaultMethods + * @run junit DefaultMethods */ -import java.util.concurrent.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.Future; +import java.util.stream.Stream; import static java.util.concurrent.Future.State.*; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; -import static org.testng.Assert.*; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import static org.junit.jupiter.api.Assertions.*; -public class DefaultMethods { +class DefaultMethods { - @DataProvider(name = "executors") - public Object[][] executors() { - return new Object[][] { - // ensures that default implementation is tested - { new DelegatingExecutorService(Executors.newCachedThreadPool()), }, + static Stream executors() { + return Stream.of( + // ensures that default close method is tested + new DelegatingExecutorService(Executors.newCachedThreadPool()), - // executors that may return a Future that overrides the methods - { new ForkJoinPool(), }, - { Executors.newCachedThreadPool(), } - }; + // executors that may return a Future that overrides the methods + Executors.newCachedThreadPool(), + Executors.newVirtualThreadPerTaskExecutor(), + new ForkJoinPool() + ); } /** * Test methods when the task has not completed. */ - @Test(dataProvider = "executors") - public void testRunningTask(ExecutorService executor) { + @ParameterizedTest + @MethodSource("executors") + void testRunningTask(ExecutorService executor) { try (executor) { var latch = new CountDownLatch(1); Future future = executor.submit(() -> { latch.await(); return null; }); try { assertTrue(future.state() == RUNNING); - expectThrows(IllegalStateException.class, future::resultNow); - expectThrows(IllegalStateException.class, future::exceptionNow); + assertThrows(IllegalStateException.class, future::resultNow); + assertThrows(IllegalStateException.class, future::exceptionNow); } finally { latch.countDown(); } @@ -70,41 +78,44 @@ public void testRunningTask(ExecutorService executor) { /** * Test methods when the task has already completed with a result. */ - @Test(dataProvider = "executors") - public void testCompletedTask1(ExecutorService executor) { + @ParameterizedTest + @MethodSource("executors") + void testCompletedTask1(ExecutorService executor) { try (executor) { Future future = executor.submit(() -> "foo"); awaitDone(future); assertTrue(future.state() == SUCCESS); - assertEquals(future.resultNow(), "foo"); - expectThrows(IllegalStateException.class, future::exceptionNow); + assertEquals("foo", future.resultNow()); + assertThrows(IllegalStateException.class, future::exceptionNow); } } /** * Test methods when the task has already completed with null. */ - @Test(dataProvider = "executors") - public void testCompletedTask2(ExecutorService executor) { + @ParameterizedTest + @MethodSource("executors") + void testCompletedTask2(ExecutorService executor) { try (executor) { Future future = executor.submit(() -> null); awaitDone(future); assertTrue(future.state() == SUCCESS); - assertEquals(future.resultNow(), null); - expectThrows(IllegalStateException.class, future::exceptionNow); + assertNull(future.resultNow()); + assertThrows(IllegalStateException.class, future::exceptionNow); } } /** * Test methods when the task has completed with an exception. */ - @Test(dataProvider = "executors") - public void testFailedTask(ExecutorService executor) { + @ParameterizedTest + @MethodSource("executors") + void testFailedTask(ExecutorService executor) { try (executor) { Future future = executor.submit(() -> { throw new ArithmeticException(); }); awaitDone(future); assertTrue(future.state() == FAILED); - expectThrows(IllegalStateException.class, future::resultNow); + assertThrows(IllegalStateException.class, future::resultNow); Throwable ex = future.exceptionNow(); assertTrue(ex instanceof ArithmeticException); } @@ -113,16 +124,17 @@ public void testFailedTask(ExecutorService executor) { /** * Test methods when the task has been cancelled (mayInterruptIfRunning=false) */ - @Test(dataProvider = "executors") - public void testCancelledTask1(ExecutorService executor) { + @ParameterizedTest + @MethodSource("executors") + void testCancelledTask1(ExecutorService executor) { try (executor) { var latch = new CountDownLatch(1); Future future = executor.submit(() -> { latch.await(); return null; }); future.cancel(false); try { assertTrue(future.state() == CANCELLED); - expectThrows(IllegalStateException.class, future::resultNow); - expectThrows(IllegalStateException.class, future::exceptionNow); + assertThrows(IllegalStateException.class, future::resultNow); + assertThrows(IllegalStateException.class, future::exceptionNow); } finally { latch.countDown(); } @@ -132,16 +144,17 @@ public void testCancelledTask1(ExecutorService executor) { /** * Test methods when the task has been cancelled (mayInterruptIfRunning=true) */ - @Test(dataProvider = "executors") - public void testCancelledTask2(ExecutorService executor) { + @ParameterizedTest + @MethodSource("executors") + void testCancelledTask2(ExecutorService executor) { try (executor) { var latch = new CountDownLatch(1); Future future = executor.submit(() -> { latch.await(); return null; }); future.cancel(true); try { assertTrue(future.state() == CANCELLED); - expectThrows(IllegalStateException.class, future::resultNow); - expectThrows(IllegalStateException.class, future::exceptionNow); + assertThrows(IllegalStateException.class, future::resultNow); + assertThrows(IllegalStateException.class, future::exceptionNow); } finally { latch.countDown(); } @@ -152,46 +165,46 @@ public void testCancelledTask2(ExecutorService executor) { * Test CompletableFuture with the task has not completed. */ @Test - public void testCompletableFuture1() { + void testCompletableFuture1() { var future = new CompletableFuture(); assertTrue(future.state() == RUNNING); - expectThrows(IllegalStateException.class, future::resultNow); - expectThrows(IllegalStateException.class, future::exceptionNow); + assertThrows(IllegalStateException.class, future::resultNow); + assertThrows(IllegalStateException.class, future::exceptionNow); } /** * Test CompletableFuture with the task that completed with result. */ @Test - public void testCompletableFuture2() { + void testCompletableFuture2() { var future = new CompletableFuture(); future.complete("foo"); assertTrue(future.state() == SUCCESS); - assertEquals(future.resultNow(), "foo"); - expectThrows(IllegalStateException.class, future::exceptionNow); + assertEquals("foo", future.resultNow()); + assertThrows(IllegalStateException.class, future::exceptionNow); } /** * Test CompletableFuture with the task that completed with null. */ @Test - public void testCompletableFuture3() { + void testCompletableFuture3() { var future = new CompletableFuture(); future.complete(null); assertTrue(future.state() == SUCCESS); - assertEquals(future.resultNow(), null); - expectThrows(IllegalStateException.class, future::exceptionNow); + assertNull(future.resultNow()); + assertThrows(IllegalStateException.class, future::exceptionNow); } /** * Test CompletableFuture with the task that completed with exception. */ @Test - public void testCompletableFuture4() { + void testCompletableFuture4() { var future = new CompletableFuture(); future.completeExceptionally(new ArithmeticException()); assertTrue(future.state() == FAILED); - expectThrows(IllegalStateException.class, future::resultNow); + assertThrows(IllegalStateException.class, future::resultNow); Throwable ex = future.exceptionNow(); assertTrue(ex instanceof ArithmeticException); } @@ -200,12 +213,12 @@ public void testCompletableFuture4() { * Test CompletableFuture with the task that was cancelled. */ @Test - public void testCompletableFuture5() { + void testCompletableFuture5() { var future = new CompletableFuture(); future.cancel(false); assertTrue(future.state() == CANCELLED); - expectThrows(IllegalStateException.class, future::resultNow); - expectThrows(IllegalStateException.class, future::exceptionNow); + assertThrows(IllegalStateException.class, future::resultNow); + assertThrows(IllegalStateException.class, future::exceptionNow); } /** diff --git a/test/jdk/java/util/concurrent/forkjoin/AsyncShutdownNow.java b/test/jdk/java/util/concurrent/forkjoin/AsyncShutdownNow.java index d742bc0955fd3..5af31673167ba 100644 --- a/test/jdk/java/util/concurrent/forkjoin/AsyncShutdownNow.java +++ b/test/jdk/java/util/concurrent/forkjoin/AsyncShutdownNow.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,11 +23,11 @@ /* * @test - * @run testng AsyncShutdownNow - * @summary Test invoking shutdownNow with threads blocked in Future.get, - * invokeAll, and invokeAny + * @summary Test ForkJoinPool.shutdownNow with threads blocked in invokeXXX and Future.get + * @run junit AsyncShutdownNow */ +import java.time.Duration; import java.util.Arrays; import java.util.List; import java.util.concurrent.Callable; @@ -38,42 +38,39 @@ import java.util.concurrent.ForkJoinPool; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import static java.lang.Thread.State.*; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; -import static org.testng.Assert.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import static org.junit.jupiter.api.Assertions.*; -public class AsyncShutdownNow { +class AsyncShutdownNow { // long running interruptible task private static final Callable SLEEP_FOR_A_DAY = () -> { - Thread.sleep(86400_000); + Thread.sleep(Duration.ofDays(1)); return null; }; - /** - * The executors to test. - */ - @DataProvider(name = "executors") - public Object[][] executors() { - return new Object[][] { - { new ForkJoinPool() }, - { new ForkJoinPool(1) }, - }; + static Stream pools() { + return Stream.of( + new ForkJoinPool(), + new ForkJoinPool(1) + ); } /** - * Test shutdownNow with running task and thread blocked in Future::get. + * Test shutdownNow with a running task and main thread blocked in Future::get. */ - @Test(dataProvider = "executors") - public void testFutureGet(ExecutorService executor) throws Exception { - System.out.format("testFutureGet: %s%n", executor); - try (executor) { - Future future = executor.submit(SLEEP_FOR_A_DAY); + @ParameterizedTest + @MethodSource("pools") + void testFutureGet(ForkJoinPool pool) throws Exception { + try (pool) { + Future future = pool.submit(SLEEP_FOR_A_DAY); - // shutdownNow when main thread waits in ForkJoinTask.get - onWait("java.util.concurrent.ForkJoinTask.get", executor::shutdownNow); + // shutdownNow when main thread waits in ForkJoinTask.awaitDone + onWait("java.util.concurrent.ForkJoinTask.awaitDone", pool::shutdownNow); try { future.get(); fail(); @@ -84,16 +81,16 @@ public void testFutureGet(ExecutorService executor) throws Exception { } /** - * Test shutdownNow with running task and thread blocked in a timed Future::get. + * Test shutdownNow with a running task and main thread blocked in a timed Future::get. */ - @Test(dataProvider = "executors") - public void testTimedFutureGet(ExecutorService executor) throws Exception { - System.out.format("testTimedFutureGet: %s%n", executor); - try (executor) { - Future future = executor.submit(SLEEP_FOR_A_DAY); + @ParameterizedTest + @MethodSource("pools") + void testTimedFutureGet(ForkJoinPool pool) throws Exception { + try (pool) { + Future future = pool.submit(SLEEP_FOR_A_DAY); - // shutdownNow when main thread waits in ForkJoinTask.get - onWait("java.util.concurrent.ForkJoinTask.get", executor::shutdownNow); + // shutdownNow when main thread waits in ForkJoinTask.awaitDone + onWait("java.util.concurrent.ForkJoinTask.awaitDone", pool::shutdownNow); try { future.get(1, TimeUnit.HOURS); fail(); @@ -104,15 +101,15 @@ public void testTimedFutureGet(ExecutorService executor) throws Exception { } /** - * Test shutdownNow with thread blocked in invokeAll. + * Test shutdownNow with running tasks and main thread blocked in invokeAll. */ - @Test(dataProvider = "executors") - public void testInvokeAll(ExecutorService executor) throws Exception { - System.out.format("testInvokeAll: %s%n", executor); - try (executor) { - // shutdownNow when main thread waits in ForkJoinTask.quietlyJoin - onWait("java.util.concurrent.ForkJoinTask.quietlyJoin", executor::shutdownNow); - List> futures = executor.invokeAll(List.of(SLEEP_FOR_A_DAY, SLEEP_FOR_A_DAY)); + @ParameterizedTest + @MethodSource("pools") + void testInvokeAll(ForkJoinPool pool) throws Exception { + try (pool) { + // shutdownNow when main thread waits in ForkJoinTask.awaitDone + onWait("java.util.concurrent.ForkJoinTask.awaitDone", pool::shutdownNow); + List> futures = pool.invokeAll(List.of(SLEEP_FOR_A_DAY, SLEEP_FOR_A_DAY)); for (Future f : futures) { assertTrue(f.isDone()); try { @@ -126,16 +123,16 @@ public void testInvokeAll(ExecutorService executor) throws Exception { } /** - * Test shutdownNow with thread blocked in invokeAny. + * Test shutdownNow with running tasks and main thread blocked in invokeAny. */ - @Test(dataProvider = "executors") - public void testInvokeAny(ExecutorService executor) throws Exception { - System.out.format("testInvokeAny: %s%n", executor); - try (executor) { + @ParameterizedTest + @MethodSource("pools") + void testInvokeAny(ForkJoinPool pool) throws Exception { + try (pool) { // shutdownNow when main thread waits in ForkJoinTask.get - onWait("java.util.concurrent.ForkJoinTask.get", executor::shutdownNow); + onWait("java.util.concurrent.ForkJoinTask.get", pool::shutdownNow); try { - executor.invokeAny(List.of(SLEEP_FOR_A_DAY, SLEEP_FOR_A_DAY)); + pool.invokeAny(List.of(SLEEP_FOR_A_DAY, SLEEP_FOR_A_DAY)); fail(); } catch (ExecutionException e) { // expected From bec6e207bbb6c713f8e9283676dadf3c8c8dc332 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Wed, 9 Aug 2023 14:21:51 -0400 Subject: [PATCH 21/61] resync --- .../java/util/concurrent/ForkJoinPool.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index b7ad09b416f8a..d844cc817a035 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1874,8 +1874,12 @@ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { if ((phase & IDLE) != 0) reactivate(w); // pool stopped before released if (w.top - w.base > 0) { - for (ForkJoinTask t; (t = w.nextLocalTask()) != null; ) - ForkJoinTask.cancelIgnoringExceptions(t); + for (ForkJoinTask t; (t = w.nextLocalTask()) != null; ) { + try { + t.cancel(false); + } catch (Throwable ignore) { + } + } } } } @@ -2793,8 +2797,12 @@ private int tryTerminate(boolean now, boolean enable) { } catch (Throwable ignore) { } } - for (ForkJoinTask t; (t = q.poll(null)) != null; ) - ForkJoinTask.cancelIgnoringExceptions(t); + for (ForkJoinTask t; (t = q.poll(null)) != null; ) { + try { + t.cancel(false); + } catch (Throwable ignore) { + } + } } } if (((e = runState) & TERMINATED) == 0 && ctl == 0L) { From b276121744380edd77591935b71bbfeb0e1aae0f Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Mon, 14 Aug 2023 07:20:01 -0400 Subject: [PATCH 22/61] Update @since tags --- .../share/classes/java/util/concurrent/ForkJoinPool.java | 2 +- .../share/classes/java/util/concurrent/ForkJoinTask.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index d844cc817a035..66cef4ac94943 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -3308,7 +3308,7 @@ public int setParallelism(int size) { * @throws NullPointerException if tasks or any of its elements are {@code null} * @throws RejectedExecutionException if any task cannot be * scheduled for execution - * @since 21 + * @since 22 */ public List> invokeAllUninterruptibly(Collection> tasks) { ArrayList> futures = new ArrayList<>(tasks.size()); diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index a6f603331f286..daf639a27c441 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -1461,7 +1461,7 @@ public static ForkJoinTask adaptInterruptible(Callable calla * @param the type of the result * @return the task * - * @since 21 + * @since 22 */ public static ForkJoinTask adaptInterruptible(Runnable runnable, T result) { return new AdaptedInterruptibleRunnable(runnable, result); @@ -1479,7 +1479,7 @@ public static ForkJoinTask adaptInterruptible(Runnable runnable, T result * @param runnable the runnable action * @return the task * - * @since 21 + * @since 22 */ public static ForkJoinTask adaptInterruptible(Runnable runnable) { return new AdaptedInterruptibleRunnable(runnable, null); From 901f5bddde7a09d845a6a021634ad8012859c860 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Thu, 17 Aug 2023 09:31:27 -0400 Subject: [PATCH 23/61] Avoid false-alarm failures on shutdown --- .../java/util/concurrent/ForkJoinPool.java | 35 +++++++++++-------- .../concurrent/tck/ForkJoinPool19Test.java | 17 +++++---- .../concurrent/tck/ForkJoinPool8Test.java | 21 ++++++----- .../util/concurrent/tck/ForkJoinTaskTest.java | 8 +++-- .../concurrent/tck/RecursiveActionTest.java | 21 ++++++----- .../concurrent/tck/RecursiveTaskTest.java | 14 +++++--- 6 files changed, 72 insertions(+), 44 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 66cef4ac94943..d1a5de0de0adb 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -430,17 +430,24 @@ public class ForkJoinPool extends AbstractExecutorService { * letting them time out and terminate when idle. * * Array "queues" holds references to WorkQueues. It is updated - * (only during worker creation and termination) under the runState - * lock. It is otherwise concurrently readable but reads for use - * in scans (see below) are always prefaced by a volatile read of - * runState (or equivalent constructions), ensuring that its state is - * current at the point it is used (which is all we require). To - * simplify index-based operations, the array size is always a - * power of two, and all readers must tolerate null slots. Worker - * queues are at odd indices. Worker phase ids masked with SMASK - * match their index. Shared (submission) queues are at even - * indices. Grouping them together in this way simplifies and - * speeds up task scanning. + * (only during worker creation and termination) under the + * runState lock. It is otherwise concurrently readable but reads + * for use in scans (see below) are always prefaced by a volatile + * read of runState (or equivalent constructions), ensuring that + * its state is current at the point it is used (which is all we + * require). To simplify index-based operations, the array size is + * always a power of two, and all readers must tolerate null + * slots. Worker queues are at odd indices. Worker phase ids + * masked with SMASK match their index. Shared (submission) queues + * are at even indices. Grouping them together in this way aids in + * task scanning: At top-level, both kinds of queues should be + * sampled with approximately the same probability, which is + * simpler if they are all in the same array. But we also need to + * identify what kind they are without looking at them, leading to + * this odd/even scheme. One disadvantage is that there are + * usually many fewer submission queues, so there can be many + * wasted probes (null slots). But this is still cheaper than + * alternatives. * * All worker thread creation is on-demand, triggered by task * submissions, replacement of terminated workers, and/or @@ -453,7 +460,7 @@ public class ForkJoinPool extends AbstractExecutorService { * essence, the queues array serves as a weak reference * mechanism. In particular, the stack top subfield of ctl stores * indices, not references. Operations on queues obtained from - * these indices remain OK (with at most some unnecessary extra + * these indices remain valid (with at most some unnecessary extra * work) even if an underlying worker failed and was replaced by * another at the same index. During termination, worker queue * array updates are disabled. @@ -718,7 +725,7 @@ public class ForkJoinPool extends AbstractExecutorService { * because chains cannot include even-numbered external queues, * they are ignored, and 0 is an OK default.) The scan in method * helpJoin uses these markers to try to find a worker to help - * (i.e., steal back a task from and execute it) that could makje + * (i.e., steal back a task from and execute it) that could make * progress toward completion of the actively joined task. Thus, * the joiner executes a task that would be on its own local deque * if the to-be-joined task had not been stolen. This is a @@ -765,7 +772,7 @@ public class ForkJoinPool extends AbstractExecutorService { * without (unobtainably) perfect information about whether worker * creation is actually necessary. False alarms are common enough * to negatively impact performance, so compensation is by default - * attempted only when it appears possible the the pool could + * attempted only when it appears possible that the pool could * stall due to lack of any unblocked workers. However, we allow * users to override defaults using the long form of the * ForkJoinPool constructor. The compensation mechanism may also diff --git a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java index 3a402c46d4c22..1c351898c8064 100644 --- a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java +++ b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java @@ -249,14 +249,17 @@ static final class FailingFibAction extends RecursiveAction { FailingFibAction(int n) { number = n; } public void compute() { int n = number; - if (n <= 1) - throw new FJException(); - else { - FailingFibAction f1 = new FailingFibAction(n - 1); - FailingFibAction f2 = new FailingFibAction(n - 2); - invokeAll(f1, f2); - result = f1.result + f2.result; + if (n > 1) { + try { + FailingFibAction f1 = new FailingFibAction(n - 1); + FailingFibAction f2 = new FailingFibAction(n - 2); + invokeAll(f1, f2); + result = f1.result + f2.result; + return; + } catch (CancellationException fallthrough) { + } } + throw new FJException(); } } diff --git a/test/jdk/java/util/concurrent/tck/ForkJoinPool8Test.java b/test/jdk/java/util/concurrent/tck/ForkJoinPool8Test.java index 9be6ce3d4189c..2bc5021fcef08 100644 --- a/test/jdk/java/util/concurrent/tck/ForkJoinPool8Test.java +++ b/test/jdk/java/util/concurrent/tck/ForkJoinPool8Test.java @@ -232,14 +232,17 @@ static final class FailingFibAction extends RecursiveAction { FailingFibAction(int n) { number = n; } public void compute() { int n = number; - if (n <= 1) - throw new FJException(); - else { - FailingFibAction f1 = new FailingFibAction(n - 1); - FailingFibAction f2 = new FailingFibAction(n - 2); - invokeAll(f1, f2); - result = f1.result + f2.result; + if (n > 1) { + try { + FailingFibAction f1 = new FailingFibAction(n - 1); + FailingFibAction f2 = new FailingFibAction(n - 2); + invokeAll(f1, f2); + result = f1.result + f2.result; + return; + } catch (CancellationException fallthrough) { + } } + throw new FJException(); } } @@ -402,7 +405,9 @@ protected void realCompute() throws Exception { try { f.get(randomTimeout(), null); shouldThrow(); - } catch (NullPointerException success) {} + } catch (NullPointerException success) { + f.join(); + } }}; checkInvoke(a); } diff --git a/test/jdk/java/util/concurrent/tck/ForkJoinTaskTest.java b/test/jdk/java/util/concurrent/tck/ForkJoinTaskTest.java index 4eaa9d2a3db0b..24989cb152bf7 100644 --- a/test/jdk/java/util/concurrent/tck/ForkJoinTaskTest.java +++ b/test/jdk/java/util/concurrent/tck/ForkJoinTaskTest.java @@ -496,7 +496,9 @@ protected void realCompute() throws Exception { try { f.get(randomTimeout(), null); shouldThrow(); - } catch (NullPointerException success) {} + } catch (NullPointerException success) { + f.join(); + } }}; testInvokeOnPool(mainPool(), a); } @@ -1245,7 +1247,9 @@ protected void realCompute() throws Exception { try { f.get(randomTimeout(), null); shouldThrow(); - } catch (NullPointerException success) {} + } catch (NullPointerException success) { + f.join(); + } }}; testInvokeOnPool(singletonPool(), a); } diff --git a/test/jdk/java/util/concurrent/tck/RecursiveActionTest.java b/test/jdk/java/util/concurrent/tck/RecursiveActionTest.java index 64115eec1c9f0..d7d928d8e8f9f 100644 --- a/test/jdk/java/util/concurrent/tck/RecursiveActionTest.java +++ b/test/jdk/java/util/concurrent/tck/RecursiveActionTest.java @@ -222,14 +222,17 @@ static final class FailingFibAction extends RecursiveAction { FailingFibAction(int n) { number = n; } public void compute() { int n = number; - if (n <= 1) - throw new FJException(); - else { - FailingFibAction f1 = new FailingFibAction(n - 1); - FailingFibAction f2 = new FailingFibAction(n - 2); - invokeAll(f1, f2); - result = f1.result + f2.result; + if (n > 1) { + try { + FailingFibAction f1 = new FailingFibAction(n - 1); + FailingFibAction f2 = new FailingFibAction(n - 2); + invokeAll(f1, f2); + result = f1.result + f2.result; + return; + } catch (CancellationException fallthrough) { + } } + throw new FJException(); } } @@ -488,7 +491,9 @@ protected void realCompute() throws Exception { try { f.get(randomTimeout(), null); shouldThrow(); - } catch (NullPointerException success) {} + } catch (NullPointerException success) { + f.join(); + } }}; testInvokeOnPool(mainPool(), a); } diff --git a/test/jdk/java/util/concurrent/tck/RecursiveTaskTest.java b/test/jdk/java/util/concurrent/tck/RecursiveTaskTest.java index 66745f4a19957..206de42d4de51 100644 --- a/test/jdk/java/util/concurrent/tck/RecursiveTaskTest.java +++ b/test/jdk/java/util/concurrent/tck/RecursiveTaskTest.java @@ -240,11 +240,15 @@ final class FailingFibTask extends RecursiveTask { FailingFibTask(int n) { number = n; } public Integer compute() { int n = number; - if (n <= 1) - throw new FJException(); - FailingFibTask f1 = new FailingFibTask(n - 1); - f1.fork(); - return new FibTask(n - 2).compute() + f1.join(); + if (n > 1) { + try { + FailingFibTask f1 = new FailingFibTask(n - 1); + f1.fork(); + return new FibTask(n - 2).compute() + f1.join(); + } catch (CancellationException fallthrough) { + } + } + throw new FJException(); } } From 39832811d4caa53fc056cc31d1185a33efe70170 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Thu, 17 Aug 2023 15:00:00 -0400 Subject: [PATCH 24/61] Pass through cancellations in FJ tests --- test/jdk/java/util/concurrent/tck/JSR166TestCase.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/jdk/java/util/concurrent/tck/JSR166TestCase.java b/test/jdk/java/util/concurrent/tck/JSR166TestCase.java index a5d491f6ce0ea..130831dcd2dd5 100644 --- a/test/jdk/java/util/concurrent/tck/JSR166TestCase.java +++ b/test/jdk/java/util/concurrent/tck/JSR166TestCase.java @@ -106,6 +106,7 @@ import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; @@ -1917,6 +1918,8 @@ public abstract class CheckedRecursiveAction extends RecursiveAction { @Override protected final void compute() { try { realCompute(); + } catch (CancellationException ex) { + throw ex; // expected by some tests } catch (Throwable fail) { threadUnexpectedException(fail); } @@ -1932,6 +1935,8 @@ public abstract class CheckedRecursiveTask extends RecursiveTask { @Override protected final T compute() { try { return realCompute(); + } catch (CancellationException ex) { + throw ex; } catch (Throwable fail) { threadUnexpectedException(fail); } From 7ad962d4dae8419f2853568ec12456f0a805c848 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Thu, 17 Aug 2023 15:05:23 -0400 Subject: [PATCH 25/61] Allow cancel in checkCompletedAbnormally --- test/jdk/java/util/concurrent/tck/RecursiveActionTest.java | 4 ++-- test/jdk/java/util/concurrent/tck/RecursiveTaskTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/jdk/java/util/concurrent/tck/RecursiveActionTest.java b/test/jdk/java/util/concurrent/tck/RecursiveActionTest.java index d7d928d8e8f9f..fc84bb37a5fa7 100644 --- a/test/jdk/java/util/concurrent/tck/RecursiveActionTest.java +++ b/test/jdk/java/util/concurrent/tck/RecursiveActionTest.java @@ -162,10 +162,10 @@ void checkCancelled(RecursiveAction a) { void checkCompletedAbnormally(RecursiveAction a, Throwable t) { assertTrue(a.isDone()); - assertFalse(a.isCancelled()); assertFalse(a.isCompletedNormally()); assertTrue(a.isCompletedAbnormally()); - assertSame(t.getClass(), a.getException().getClass()); + if (!a.isCancelled() + assertSame(t.getClass(), a.getException().getClass()); assertNull(a.getRawResult()); assertFalse(a.cancel(false)); assertFalse(a.cancel(true)); diff --git a/test/jdk/java/util/concurrent/tck/RecursiveTaskTest.java b/test/jdk/java/util/concurrent/tck/RecursiveTaskTest.java index 206de42d4de51..348fef78a6f79 100644 --- a/test/jdk/java/util/concurrent/tck/RecursiveTaskTest.java +++ b/test/jdk/java/util/concurrent/tck/RecursiveTaskTest.java @@ -178,10 +178,10 @@ void checkCancelled(RecursiveTask a) { void checkCompletedAbnormally(RecursiveTask a, Throwable t) { assertTrue(a.isDone()); - assertFalse(a.isCancelled()); assertFalse(a.isCompletedNormally()); assertTrue(a.isCompletedAbnormally()); - assertSame(t.getClass(), a.getException().getClass()); + if (!a.isCancelled()) + assertSame(t.getClass(), a.getException().getClass()); assertNull(a.getRawResult()); assertFalse(a.cancel(false)); assertFalse(a.cancel(true)); From 27f7357b667c2fb1f1b2139c01f08c4bd79e9323 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Thu, 17 Aug 2023 18:49:06 -0400 Subject: [PATCH 26/61] Typo --- test/jdk/java/util/concurrent/tck/RecursiveActionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/jdk/java/util/concurrent/tck/RecursiveActionTest.java b/test/jdk/java/util/concurrent/tck/RecursiveActionTest.java index fc84bb37a5fa7..69bae253dd196 100644 --- a/test/jdk/java/util/concurrent/tck/RecursiveActionTest.java +++ b/test/jdk/java/util/concurrent/tck/RecursiveActionTest.java @@ -164,7 +164,7 @@ void checkCompletedAbnormally(RecursiveAction a, Throwable t) { assertTrue(a.isDone()); assertFalse(a.isCompletedNormally()); assertTrue(a.isCompletedAbnormally()); - if (!a.isCancelled() + if (!a.isCancelled()) assertSame(t.getClass(), a.getException().getClass()); assertNull(a.getRawResult()); assertFalse(a.cancel(false)); From 1a7466d3674340500c912d98993ec2419b76c25a Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sat, 19 Aug 2023 08:53:23 -0400 Subject: [PATCH 27/61] Use signalWork in isQuiescent --- .../java/util/concurrent/ForkJoinPool.java | 33 +++++++------------ 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index d1a5de0de0adb..01f3d1fd06f48 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -447,7 +447,11 @@ public class ForkJoinPool extends AbstractExecutorService { * this odd/even scheme. One disadvantage is that there are * usually many fewer submission queues, so there can be many * wasted probes (null slots). But this is still cheaper than - * alternatives. + * alternatives. Other loops over the queues array vary in origin + * and stride depending on whether they cover only submission + * (even) or worker (odd) queues or both, and whether they require + * randomness (in which case cyclically exhaustive strides may be + * used). * * All worker thread creation is on-demand, triggered by task * submissions, replacement of terminated workers, and/or @@ -1983,8 +1987,8 @@ private void reactivate(WorkQueue w) { * quiescent */ private boolean isQuiescent(boolean transition) { - long phaseSum = 0L; - boolean swept = false, possibleSubmission = false; + long phaseSum = -1L; + boolean swept = false; outer: for (int e = 0;;) { int prevRunState = e; long c = ctl; @@ -1992,22 +1996,9 @@ private boolean isQuiescent(boolean transition) { return true; // terminating else if ((c & RC_MASK) > 0L) break; // at least one active - else if (possibleSubmission) { // cover signal race - WorkQueue[] qs; int sp, j; WorkQueue v; Thread t; - possibleSubmission = swept = false; - if ((sp = (int)c) == 0 || (qs = queues) == null || - qs.length <= (j = sp & SMASK) || (v = qs[j]) == null) - break; // no inactive workers - if (compareAndSetCtl(c, (v.stackPred & LMASK) | - (UMASK & (c + RC_UNIT)) | (c & TC_MASK))) { - v.phase = sp; - if ((t = v.parker) != null) - U.unpark(t); // reactivate - } - } else if (ctl != c) // re-snapshot - ; - else if (e == prevRunState && (e & RS_LOCK) == 0 && swept && + swept = false; + else if (swept && e == prevRunState && (e & RS_LOCK) == 0 && (!transition || casRunState(e, ((e & SHUTDOWN) == 0 ? e + RS_EPOCH : // advance @@ -2021,8 +2012,8 @@ else if (e == prevRunState && (e & RS_LOCK) == 0 && swept && WorkQueue q; int p; if ((q = qs[i]) != null) { if (((p = q.phase) & IDLE) == 0 || q.top - q.base > 0) { - possibleSubmission = true; - continue outer; // recheck counts + signalWork(); // ensure live + break outer; } sum += p; } @@ -2556,7 +2547,7 @@ else if ((t = pollScan(false)) != null) { waits = 0; t.doExec(); } - else if (isQuiescent(true)) + else if (isQuiescent(false)) break; else if (System.nanoTime() - startTime > nanos) return 0; From 50fe9a4a9df200a27f9860001eea3111700b79a5 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sun, 20 Aug 2023 08:57:22 -0400 Subject: [PATCH 28/61] Ensure thread not interrupted when testing close --- test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java index 1c351898c8064..d9ef55f95b89d 100644 --- a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java +++ b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java @@ -508,6 +508,7 @@ public void testAdaptInterruptible_Callable_toString() { * Implicitly closing a new pool using try-with-resources terminates it */ public void testClose() { + Thread.interrupted(); ForkJoinTask f = new FibAction(8); ForkJoinPool pool = null; try (ForkJoinPool p = new ForkJoinPool()) { From 76abc1fd3024b24d56e6b2ca432a083c7662a2df Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Mon, 21 Aug 2023 11:32:37 -0400 Subject: [PATCH 29/61] Run close tests in seperate threads --- .../concurrent/tck/ForkJoinPool19Test.java | 53 ++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java index d9ef55f95b89d..84c61245a077c 100644 --- a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java +++ b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java @@ -508,37 +508,42 @@ public void testAdaptInterruptible_Callable_toString() { * Implicitly closing a new pool using try-with-resources terminates it */ public void testClose() { - Thread.interrupted(); - ForkJoinTask f = new FibAction(8); - ForkJoinPool pool = null; - try (ForkJoinPool p = new ForkJoinPool()) { - pool = p; - p.execute(f); - } - checkCompletedNormally(f); - assertTrue(pool != null && pool.isTerminated()); + Thread t = newStartedThread(new CheckedRunnable() { + public void realRun() throws InterruptedException { + ForkJoinTask f = new FibAction(8); + ForkJoinPool pool = null; + try (ForkJoinPool p = new ForkJoinPool()) { + pool = p; + p.execute(f); + } + checkCompletedNormally(f); + assertTrue(pool != null && pool.isTerminated()); + }}); + awaitTermination(t); } /** * Implicitly closing common pool using try-with-resources has no effect. */ public void testCloseCommonPool() { - ForkJoinTask f = new FibAction(8); - ForkJoinPool pool; - try (ForkJoinPool p = pool = ForkJoinPool.commonPool()) { - p.execute(f); - } - - assertFalse(pool.isShutdown()); - assertFalse(pool.isTerminating()); - assertFalse(pool.isTerminated()); - String prop = System.getProperty( "java.util.concurrent.ForkJoinPool.common.parallelism"); - if (! "0".equals(prop)) { - f.join(); - checkCompletedNormally(f); - } + boolean nothreads = "0".equals(prop); + Thread t = newStartedThread(new CheckedRunnable() { + public void realRun() throws InterruptedException { + ForkJoinTask f = new FibAction(8); + ForkJoinPool pool; + try (ForkJoinPool p = pool = ForkJoinPool.commonPool()) { + p.execute(f); + } + assertFalse(pool.isShutdown()); + assertFalse(pool.isTerminating()); + assertFalse(pool.isTerminated()); + if (!nothreads) { + f.join(); + checkCompletedNormally(f); + } + }}); + awaitTermination(t); } - } From c4b9749be515357475c3458c44ab1c8647a332fe Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Tue, 22 Aug 2023 12:38:04 -0400 Subject: [PATCH 30/61] Ensure each CloseTest test runs in a new thread --- .../concurrent/ExecutorService/CloseTest.java | 195 +++++++++++------- 1 file changed, 116 insertions(+), 79 deletions(-) diff --git a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java index 0c5b6d2f6e970..4af00d4fe61e9 100644 --- a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java +++ b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java @@ -45,6 +45,35 @@ class CloseTest { + // setup to ensure each test runs under a new thread, to preclude + // stale interrupts from changing expected close() behavior + + static abstract class TestAction { + abstract void run() throws Exception; + } + static final class CheckedAction implements Runnable { + final TestAction action; + volatile Exception error; + CheckedAction(TestAction a) { action = a; } + public void run() { + try { + a.run(); + } catch (Exception ex) { + error = ex; + } + } + } + static void testInNewThread(TestAction a) throws Throawble { + try { + Thread t = new Thread(new CheckedAction(a)); + t.start(); + t.join(); + } finally { + Exception e = a.error; + if (e != null) + throw e; + } + } static Stream executors() { return Stream.of( // ensures that default close method is tested @@ -63,10 +92,12 @@ static Stream executors() { @ParameterizedTest @MethodSource("executors") void testCloseWithNoTasks(ExecutorService executor) throws Exception { - executor.close(); - assertTrue(executor.isShutdown()); - assertTrue(executor.isTerminated()); - assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + testInNewThread(new TestAction() { void run() throws Exception { + executor.close(); + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + }}); } /** @@ -75,15 +106,17 @@ void testCloseWithNoTasks(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testCloseWithRunningTasks(ExecutorService executor) throws Exception { - Future future = executor.submit(() -> { - Thread.sleep(Duration.ofMillis(100)); - return "foo"; - }); - executor.close(); // waits for task to complete - assertTrue(executor.isShutdown()); - assertTrue(executor.isTerminated()); - assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - assertEquals("foo", future.resultNow()); + testInNewThread(new TestAction() { void run() throws Exception { + Future future = executor.submit(() -> { + Thread.sleep(Duration.ofMillis(100)); + return "foo"; + }); + executor.close(); // waits for task to complete + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + assertEquals("foo", future.resultNow()); + }}); } /** @@ -92,21 +125,21 @@ void testCloseWithRunningTasks(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testShutdownBeforeClose(ExecutorService executor) throws Exception { - Phaser phaser = new Phaser(2); - Future future = executor.submit(() -> { - phaser.arriveAndAwaitAdvance(); - Thread.sleep(Duration.ofMillis(100)); - return "foo"; - }); - phaser.arriveAndAwaitAdvance(); // wait for task to start - - executor.shutdown(); // shutdown, will not immediately terminate - - executor.close(); - assertTrue(executor.isShutdown()); - assertTrue(executor.isTerminated()); - assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - assertEquals("foo", future.resultNow()); + testInNewThread(new TestAction() { void run() throws Exception { + Phaser phaser = new Phaser(2); + Future future = executor.submit(() -> { + phaser.arriveAndAwaitAdvance(); + Thread.sleep(Duration.ofMillis(100)); + return "foo"; + }); + phaser.arriveAndAwaitAdvance(); // wait for task to start + executor.shutdown(); // shutdown, will not immediately terminate + executor.close(); + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + assertEquals("foo", future.resultNow()); + }}); } /** @@ -115,13 +148,14 @@ void testShutdownBeforeClose(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testTerminateBeforeClose(ExecutorService executor) throws Exception { - executor.shutdown(); - assertTrue(executor.isTerminated()); - - executor.close(); - assertTrue(executor.isShutdown()); - assertTrue(executor.isTerminated()); - assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + testInNewThread(new TestAction() { void run() throws Exception { + executor.shutdown(); + assertTrue(executor.isTerminated()); + executor.close(); + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + }}); } /** @@ -130,25 +164,26 @@ void testTerminateBeforeClose(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testInterruptBeforeClose(ExecutorService executor) throws Exception { - Phaser phaser = new Phaser(2); - Future future = executor.submit(() -> { - phaser.arriveAndAwaitAdvance(); - Thread.sleep(Duration.ofDays(1)); - return null; - }); - phaser.arriveAndAwaitAdvance(); // wait for task to start - - Thread.currentThread().interrupt(); - try { - executor.close(); - assertTrue(Thread.currentThread().isInterrupted()); - } finally { - Thread.interrupted(); // clear interrupt status - } - assertTrue(executor.isShutdown()); - assertTrue(executor.isTerminated()); - assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - assertThrows(ExecutionException.class, future::get); + testInNewThread(new TestAction() { void run() throws Exception { + Phaser phaser = new Phaser(2); + Future future = executor.submit(() -> { + phaser.arriveAndAwaitAdvance(); + Thread.sleep(Duration.ofDays(1)); + return null; + }); + phaser.arriveAndAwaitAdvance(); // wait for task to start + Thread.currentThread().interrupt(); + try { + executor.close(); + assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); // clear interrupt status + } + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + assertThrows(ExecutionException.class, future::get); + }}); } /** @@ -157,30 +192,32 @@ void testInterruptBeforeClose(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testInterruptDuringClose(ExecutorService executor) throws Exception { - Phaser phaser = new Phaser(2); - Future future = executor.submit(() -> { - phaser.arriveAndAwaitAdvance(); - Thread.sleep(Duration.ofDays(1)); - return null; - }); - phaser.arriveAndAwaitAdvance(); // wait for task to start - - // schedule main thread to be interrupted - Thread thread = Thread.currentThread(); - new Thread(() -> { - try { Thread.sleep( Duration.ofMillis(100)); } catch (Exception ignore) { } - thread.interrupt(); - }).start(); - - try { - executor.close(); - assertTrue(Thread.currentThread().isInterrupted()); - } finally { - Thread.interrupted(); // clear interrupt status - } - assertTrue(executor.isShutdown()); - assertTrue(executor.isTerminated()); - assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - assertThrows(ExecutionException.class, future::get); + testInNewThread(new TestAction() { void run() throws Exception { + Phaser phaser = new Phaser(2); + Future future = executor.submit(() -> { + phaser.arriveAndAwaitAdvance(); + Thread.sleep(Duration.ofDays(1)); + return null; + }); + phaser.arriveAndAwaitAdvance(); // wait for task to start + // schedule main thread to be interrupted + Thread thread = Thread.currentThread(); + new Thread(() -> { + try { + Thread.sleep( Duration.ofMillis(100)); + } catch (Exception ignore) { } + thread.interrupt(); + }).start(); + try { + executor.close(); + assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); // clear interrupt status + } + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + assertThrows(ExecutionException.class, future::get); + }}); } } From 61381dbd073448945fe68ed740c927ca517573f4 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Tue, 22 Aug 2023 15:07:21 -0400 Subject: [PATCH 31/61] Resync CloseTest --- .../util/concurrent/ExecutorService/CloseTest.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java index 4af00d4fe61e9..c5db0be1c7afe 100644 --- a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java +++ b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java @@ -57,23 +57,25 @@ static final class CheckedAction implements Runnable { CheckedAction(TestAction a) { action = a; } public void run() { try { - a.run(); + action.run(); } catch (Exception ex) { error = ex; } } } - static void testInNewThread(TestAction a) throws Throawble { + static void testInNewThread(TestAction a) throws Exception { + var wrapper = new CheckedAction(a); try { - Thread t = new Thread(new CheckedAction(a)); + Thread t = new Thread(wrapper); t.start(); t.join(); } finally { - Exception e = a.error; + Exception e = wrapper.error; if (e != null) throw e; } } + static Stream executors() { return Stream.of( // ensures that default close method is tested From 9d1b55a7da341fcd50962c65d00232b27f122008 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Thu, 24 Aug 2023 07:36:32 -0400 Subject: [PATCH 32/61] Ordering for termination triggering --- .../java/util/concurrent/ForkJoinPool.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 01f3d1fd06f48..ff166117730df 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1830,7 +1830,9 @@ final void registerWorker(WorkQueue w) { int stop = lockRunState() & STOP; try { WorkQueue[] qs; int n; - if ((qs = queues) != null && (n = qs.length) > 0) { + if (stop != 0 || (qs = queues) == null || (n = qs.length) <= 0) + w.owner = null; // terminating + else { for (int k = n, m = n - 1; ; id += 2) { if (qs[id &= m] == null) break; @@ -1839,21 +1841,21 @@ final void registerWorker(WorkQueue w) { break; } } - w.phase = id; // now publishable + w.phase = id; // now publishable if (id < n) qs[id] = w; - else if (stop == 0) { // expand unless stopping + else { // expand int an = n << 1, am = an - 1; WorkQueue[] as = new WorkQueue[an]; as[id & am] = w; for (int j = 1; j < n; j += 2) as[j] = qs[j]; for (int j = 0; j < n; j += 2) { - WorkQueue q; // shared queues may move + WorkQueue q; // shared queues may move if ((q = qs[j]) != null) as[q.phase & EXTERNAL_ID_MASK & am] = q; } - U.storeFence(); // fill before publish + U.storeFence(); // fill before publish queues = as; } } @@ -2786,10 +2788,12 @@ private int tryTerminate(boolean now, boolean enable) { int r = ThreadLocalRandom.nextSecondarySeed(); // stagger traversals WorkQueue[] qs = queues; int n = (qs == null) ? 0 : qs.length; + boolean active = false; for (int l = n; l > 0; --l, ++r) { - int j; WorkQueue q; Thread o; // cancel tasks; interrupt workers - if ((q = qs[j = r & SMASK & (n - 1)]) != null) { + WorkQueue q; Thread o; // cancel tasks; interrupt workers + if ((q = qs[r & SMASK & (n - 1)]) != null) { if ((o = q.owner) != null) { + active = true; try { o.interrupt(); } catch (Throwable ignore) { @@ -2803,7 +2807,7 @@ private int tryTerminate(boolean now, boolean enable) { } } } - if (((e = runState) & TERMINATED) == 0 && ctl == 0L) { + if (((e = runState) & TERMINATED) == 0 && !active && ctl == 0L) { if ((getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0) { CountDownLatch done; SharedThreadContainer ctr; if ((done = termination) != null) From 9a329b6cf05526bc15b5c3f9057bb64957eecdad Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Fri, 25 Aug 2023 07:47:59 -0400 Subject: [PATCH 33/61] Isolate unexpected interrupt status issues --- .../java/util/concurrent/ForkJoinPool.java | 18 +++-- .../concurrent/ExecutorService/CloseTest.java | 71 +++++++++++++++++++ .../concurrent/tck/ForkJoinPool19Test.java | 36 ++++++++++ 3 files changed, 118 insertions(+), 7 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index ff166117730df..79fb73483c014 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1881,7 +1881,10 @@ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { if (wt != null && (w = wt.workQueue) != null) { phase = w.phase; src = w.source; + Thread o = w.owner; w.owner = null; // disable signals + if (o == Thread.currentThread()) + Thread.interrupted(); // clear if (phase != 0) { // else failed to start replaceable = true; if ((phase & IDLE) != 0) @@ -2773,8 +2776,8 @@ static int getSurplusQueuedTaskCount() { * @return runState (possibly only its status bits) on exit */ private int tryTerminate(boolean now, boolean enable) { - int e, isShutdown; - if (((e = runState) & STOP) == 0) { + int e = runState, wasStopping = e & STOP, isShutdown; + if (wasStopping == 0) { if (now) getAndBitwiseOrRunState(e = STOP | SHUTDOWN); else { @@ -2792,11 +2795,13 @@ private int tryTerminate(boolean now, boolean enable) { for (int l = n; l > 0; --l, ++r) { WorkQueue q; Thread o; // cancel tasks; interrupt workers if ((q = qs[r & SMASK & (n - 1)]) != null) { - if ((o = q.owner) != null) { + if ((o = q.owner) != null && o != Thread.currentThread()) { active = true; - try { - o.interrupt(); - } catch (Throwable ignore) { + if (wasStopping == 0 || !o.isInterrupted()) { + try { + o.interrupt(); + } catch (Throwable ignore) { + } } } for (ForkJoinTask t; (t = q.poll(null)) != null; ) { @@ -3825,7 +3830,6 @@ public void close() { else { try { done.await(); - break; } catch (InterruptedException ex) { interrupted = true; } diff --git a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java index c5db0be1c7afe..0c8d2ce8bdd09 100644 --- a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java +++ b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java @@ -114,6 +114,7 @@ void testCloseWithRunningTasks(ExecutorService executor) throws Exception { return "foo"; }); executor.close(); // waits for task to complete + assertFalse(Thread.interrupted()); assertTrue(executor.isShutdown()); assertTrue(executor.isTerminated()); assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); @@ -121,6 +122,76 @@ void testCloseWithRunningTasks(ExecutorService executor) throws Exception { }}); } + /** + * Test shutdown with tasks running. + */ + @ParameterizedTest + @MethodSource("executors") + void testShutdownWithRunningTasks(ExecutorService executor) throws Exception { + testInNewThread(new TestAction() { void run() throws Exception { + Future future = executor.submit(() -> { + Thread.sleep(Duration.ofMillis(100)); + return "foo"; + }); + executor.shutdown(); + assertFalse(Thread.interrupted()); + assertTrue(executor.isShutdown()); + assertTrue(executor.awaitTermination(200, TimeUnit.MILLISECONDS)); + assertTrue(executor.isTerminated()); + assertEquals("foo", future.resultNow()); + }}); + } + + /** + * Test close with multiple tasks running + */ + @ParameterizedTest + @MethodSource("executors") + void testCloseWith2RunningTasks(ExecutorService executor) throws Exception { + testInNewThread(new TestAction() { void run() throws Exception { + Future f1 = executor.submit(() -> { + Thread.sleep(Duration.ofMillis(100)); + return "foo"; + }); + Future f2 = executor.submit(() -> { + Thread.sleep(Duration.ofMillis(100)); + return "bar"; + }); + executor.close(); // waits for task to complete + assertFalse(Thread.interrupted()); + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + assertEquals("foo", f1.resultNow()); + assertEquals("bar", f2.resultNow()); + }}); + } + + /** + * Test shutdown with multiple tasks running + */ + @ParameterizedTest + @MethodSource("executors") + void testShutdownWith2RunningTasks(ExecutorService executor) throws Exception { + testInNewThread(new TestAction() { void run() throws Exception { + Future f1 = executor.submit(() -> { + Thread.sleep(Duration.ofMillis(100)); + return "foo"; + }); + Future f2 = executor.submit(() -> { + Thread.sleep(Duration.ofMillis(100)); + return "bar"; + }); + executor.shutdown(); + assertFalse(Thread.interrupted()); + assertTrue(executor.isShutdown()); + assertTrue(executor.awaitTermination(200, TimeUnit.MILLISECONDS)); + assertTrue(executor.isTerminated()); + assertEquals("foo", f1.resultNow()); + assertEquals("bar", f2.resultNow()); + }}); + } + /** * Test close when executor is shutdown but not terminated. */ diff --git a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java index 84c61245a077c..ec963e222c83f 100644 --- a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java +++ b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java @@ -516,6 +516,42 @@ public void realRun() throws InterruptedException { pool = p; p.execute(f); } + assertFalse(Thread.interrupted()); + checkCompletedNormally(f); + assertTrue(pool != null && pool.isTerminated()); + }}); + awaitTermination(t); + } + + /** + * Explicitly closing a new pool terminates it + */ + public void testClose2() { + Thread t = newStartedThread(new CheckedRunnable() { + public void realRun() throws InterruptedException { + ForkJoinPool pool = new ForkJoinPool(); + ForkJoinTask f = new FibAction(8); + pool.execute(f); + pool.close(); + assertFalse(Thread.interrupted()); + checkCompletedNormally(f); + assertTrue(pool != null && pool.isTerminated()); + }}); + awaitTermination(t); + } + + /** + * Explicitly closing a shutdown pool awaits termination + */ + public void testClose3() { + Thread t = newStartedThread(new CheckedRunnable() { + public void realRun() throws InterruptedException { + ForkJoinPool pool = new ForkJoinPool(); + ForkJoinTask f = new FibAction(8); + pool.execute(f); + pool.shutdown(); + pool.close(); + assertFalse(Thread.interrupted()); checkCompletedNormally(f); assertTrue(pool != null && pool.isTerminated()); }}); From 5a4131f7cfa6ac1f732487be05056a2b55c01b43 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Fri, 25 Aug 2023 10:02:34 -0400 Subject: [PATCH 34/61] Conditionalize new tests --- .../concurrent/ExecutorService/CloseTest.java | 72 ++++++++++--------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java index 0c8d2ce8bdd09..568bd9ab38a31 100644 --- a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java +++ b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java @@ -148,23 +148,25 @@ void testShutdownWithRunningTasks(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testCloseWith2RunningTasks(ExecutorService executor) throws Exception { - testInNewThread(new TestAction() { void run() throws Exception { - Future f1 = executor.submit(() -> { - Thread.sleep(Duration.ofMillis(100)); - return "foo"; - }); - Future f2 = executor.submit(() -> { - Thread.sleep(Duration.ofMillis(100)); - return "bar"; - }); - executor.close(); // waits for task to complete - assertFalse(Thread.interrupted()); - assertTrue(executor.isShutdown()); - assertTrue(executor.isTerminated()); - assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - assertEquals("foo", f1.resultNow()); - assertEquals("bar", f2.resultNow()); - }}); + if (executor instanceof ForkJoinPool) { + testInNewThread(new TestAction() { void run() throws Exception { + Future f1 = executor.submit(() -> { + Thread.sleep(Duration.ofMillis(100)); + return "foo"; + }); + Future f2 = executor.submit(() -> { + Thread.sleep(Duration.ofMillis(100)); + return "bar"; + }); + executor.close(); // waits for task to complete + assertFalse(Thread.interrupted()); + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + assertEquals("foo", f1.resultNow()); + assertEquals("bar", f2.resultNow()); + }}); + } } /** @@ -173,23 +175,25 @@ void testCloseWith2RunningTasks(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testShutdownWith2RunningTasks(ExecutorService executor) throws Exception { - testInNewThread(new TestAction() { void run() throws Exception { - Future f1 = executor.submit(() -> { - Thread.sleep(Duration.ofMillis(100)); - return "foo"; - }); - Future f2 = executor.submit(() -> { - Thread.sleep(Duration.ofMillis(100)); - return "bar"; - }); - executor.shutdown(); - assertFalse(Thread.interrupted()); - assertTrue(executor.isShutdown()); - assertTrue(executor.awaitTermination(200, TimeUnit.MILLISECONDS)); - assertTrue(executor.isTerminated()); - assertEquals("foo", f1.resultNow()); - assertEquals("bar", f2.resultNow()); - }}); + if (executor instanceof ForkJoinPool) { + testInNewThread(new TestAction() { void run() throws Exception { + Future f1 = executor.submit(() -> { + Thread.sleep(Duration.ofMillis(100)); + return "foo"; + }); + Future f2 = executor.submit(() -> { + Thread.sleep(Duration.ofMillis(100)); + return "bar"; + }); + executor.shutdown(); + assertFalse(Thread.interrupted()); + assertTrue(executor.isShutdown()); + assertTrue(executor.awaitTermination(200, TimeUnit.MILLISECONDS)); + assertTrue(executor.isTerminated()); + assertEquals("foo", f1.resultNow()); + assertEquals("bar", f2.resultNow()); + }}); + } } /** From 2fd8bb8718e31a91872142cf682b33d0e7c5a2f9 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sat, 26 Aug 2023 08:57:53 -0400 Subject: [PATCH 35/61] Clear interrupts at top level --- .../classes/java/util/concurrent/ForkJoinPool.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 79fb73483c014..33e4375c469a7 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1495,13 +1495,22 @@ private ForkJoinTask tryPoll() { * remaining local tasks and/or others available from the * given queue, if any. */ - final void topLevelExec(ForkJoinTask task, WorkQueue src, int srcId) { + final void topLevelExec(ForkJoinTask task, ForkJoinPool pool, + WorkQueue src, int srcId) { int cfg = config, fifo = cfg & FIFO, nstolen = nsteals + 1; if ((srcId & 1) != 0) // don't record external sources source = srcId; if ((cfg & CLEAR_TLS) != 0) ThreadLocalRandom.eraseThreadLocals(Thread.currentThread()); while (task != null) { + Thread.interrupted(); + if (poolIsStopping(pool)) { + try { + task.cancel(false); + } catch (Throwable ignore) { + } + break; + } task.doExec(); if ((task = nextLocalTask(fifo)) == null && src != null && (task = src.tryPoll()) != null) @@ -2119,7 +2128,7 @@ a, slotOffset(k), t, null))) { signalWork(); // propagate at most twice/run next = (window & LMASK) | RESCAN; if (w != null) // always true - w.topLevelExec(t, q, j); + w.topLevelExec(t, this, q, j); break outer; } else if (o == null) { // contended; limit rescans From 4b12ddc9c65dd277c3cf1de8261698439e3bce9e Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sun, 27 Aug 2023 08:30:55 -0400 Subject: [PATCH 36/61] Avoid unwanted jtreg interrupts; undo unnecessary changes --- .../java/util/concurrent/ForkJoinPool.java | 49 +++++------- .../concurrent/ExecutorService/CloseTest.java | 77 +++++++++---------- .../concurrent/tck/ForkJoinPool19Test.java | 6 +- .../util/concurrent/tck/JSR166TestCase.java | 6 +- 4 files changed, 65 insertions(+), 73 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 33e4375c469a7..bfed64fdb0d97 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1495,22 +1495,13 @@ private ForkJoinTask tryPoll() { * remaining local tasks and/or others available from the * given queue, if any. */ - final void topLevelExec(ForkJoinTask task, ForkJoinPool pool, - WorkQueue src, int srcId) { + final void topLevelExec(ForkJoinTask task, WorkQueue src, int srcId) { int cfg = config, fifo = cfg & FIFO, nstolen = nsteals + 1; if ((srcId & 1) != 0) // don't record external sources source = srcId; if ((cfg & CLEAR_TLS) != 0) ThreadLocalRandom.eraseThreadLocals(Thread.currentThread()); while (task != null) { - Thread.interrupted(); - if (poolIsStopping(pool)) { - try { - task.cancel(false); - } catch (Throwable ignore) { - } - break; - } task.doExec(); if ((task = nextLocalTask(fifo)) == null && src != null && (task = src.tryPoll()) != null) @@ -1739,8 +1730,9 @@ private int getAndSetParallelism(int v) { private int getParallelismOpaque() { return U.getIntOpaque(this, PARALLELISM); } - private boolean casTerminationSignal(CountDownLatch x) { - return U.compareAndSetReference(this, TERMINATION, null, x); + private CountDownLatch cmpExTerminationSignal(CountDownLatch x) { + return (CountDownLatch) + U.compareAndExchangeReference(this, TERMINATION, null, x); } // runState operations @@ -1890,10 +1882,7 @@ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { if (wt != null && (w = wt.workQueue) != null) { phase = w.phase; src = w.source; - Thread o = w.owner; w.owner = null; // disable signals - if (o == Thread.currentThread()) - Thread.interrupted(); // clear if (phase != 0) { // else failed to start replaceable = true; if ((phase & IDLE) != 0) @@ -2128,7 +2117,7 @@ a, slotOffset(k), t, null))) { signalWork(); // propagate at most twice/run next = (window & LMASK) | RESCAN; if (w != null) // always true - w.topLevelExec(t, this, q, j); + w.topLevelExec(t, q, j); break outer; } else if (o == null) { // contended; limit rescans @@ -2785,8 +2774,8 @@ static int getSurplusQueuedTaskCount() { * @return runState (possibly only its status bits) on exit */ private int tryTerminate(boolean now, boolean enable) { - int e = runState, wasStopping = e & STOP, isShutdown; - if (wasStopping == 0) { + int e, isShutdown; + if (((e = runState) & STOP) == 0) { if (now) getAndBitwiseOrRunState(e = STOP | SHUTDOWN); else { @@ -2797,16 +2786,16 @@ private int tryTerminate(boolean now, boolean enable) { } } if ((e & (STOP | TERMINATED)) == STOP) { + boolean alive = false; int r = ThreadLocalRandom.nextSecondarySeed(); // stagger traversals WorkQueue[] qs = queues; int n = (qs == null) ? 0 : qs.length; - boolean active = false; for (int l = n; l > 0; --l, ++r) { WorkQueue q; Thread o; // cancel tasks; interrupt workers if ((q = qs[r & SMASK & (n - 1)]) != null) { - if ((o = q.owner) != null && o != Thread.currentThread()) { - active = true; - if (wasStopping == 0 || !o.isInterrupted()) { + if ((o = q.owner) != null) { + alive = true; + if (o != Thread.currentThread()) { try { o.interrupt(); } catch (Throwable ignore) { @@ -2821,7 +2810,7 @@ private int tryTerminate(boolean now, boolean enable) { } } } - if (((e = runState) & TERMINATED) == 0 && !active && ctl == 0L) { + if (((e = runState) & TERMINATED) == 0 && !alive && ctl == 0L) { if ((getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0) { CountDownLatch done; SharedThreadContainer ctr; if ((done = termination) != null) @@ -2836,14 +2825,13 @@ private int tryTerminate(boolean now, boolean enable) { } /** - * Lazily constructs termination signal + * Returns termination signal, constructing if necessary */ private CountDownLatch terminationSignal() { - CountDownLatch signal; - do { - signal = termination; - } while (signal == null && // OK to throw away if CAS failure - !casTerminationSignal(signal = new CountDownLatch(1))); + CountDownLatch signal, s, u; + if ((signal = termination) == null) + signal = ((u = cmpExTerminationSignal( + s = new CountDownLatch(1))) == null) ? s : u; return signal; } @@ -3832,13 +3820,14 @@ public void close() { if (workerNamePrefix != null && (runState & TERMINATED) == 0) { checkPermission(); CountDownLatch done = null; - boolean interrupted = Thread.interrupted(); + boolean interrupted = false; // Thread.interrupted(); while ((tryTerminate(interrupted, true) & TERMINATED) == 0) { if (done == null) done = terminationSignal(); else { try { done.await(); + break; } catch (InterruptedException ex) { interrupted = true; } diff --git a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java index 568bd9ab38a31..af07982d165d9 100644 --- a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java +++ b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java @@ -63,10 +63,13 @@ public void run() { } } } + // Avoids unwanted interrupts when run inder jtreg + static final ThreadGroup closeTestThreadGroup = + new ThreadGroup("closeTestThreadGroup"); static void testInNewThread(TestAction a) throws Exception { var wrapper = new CheckedAction(a); try { - Thread t = new Thread(wrapper); + Thread t = new Thread(closeTestThreadGroup, wrapper); t.start(); t.join(); } finally { @@ -148,25 +151,23 @@ void testShutdownWithRunningTasks(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testCloseWith2RunningTasks(ExecutorService executor) throws Exception { - if (executor instanceof ForkJoinPool) { - testInNewThread(new TestAction() { void run() throws Exception { - Future f1 = executor.submit(() -> { - Thread.sleep(Duration.ofMillis(100)); - return "foo"; - }); - Future f2 = executor.submit(() -> { - Thread.sleep(Duration.ofMillis(100)); - return "bar"; - }); - executor.close(); // waits for task to complete - assertFalse(Thread.interrupted()); - assertTrue(executor.isShutdown()); - assertTrue(executor.isTerminated()); - assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - assertEquals("foo", f1.resultNow()); - assertEquals("bar", f2.resultNow()); - }}); - } + testInNewThread(new TestAction() { void run() throws Exception { + Future f1 = executor.submit(() -> { + Thread.sleep(Duration.ofMillis(100)); + return "foo"; + }); + Future f2 = executor.submit(() -> { + Thread.sleep(Duration.ofMillis(100)); + return "bar"; + }); + executor.close(); // waits for task to complete + assertFalse(Thread.interrupted()); + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + assertEquals("foo", f1.resultNow()); + assertEquals("bar", f2.resultNow()); + }}); } /** @@ -175,25 +176,23 @@ void testCloseWith2RunningTasks(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testShutdownWith2RunningTasks(ExecutorService executor) throws Exception { - if (executor instanceof ForkJoinPool) { - testInNewThread(new TestAction() { void run() throws Exception { - Future f1 = executor.submit(() -> { - Thread.sleep(Duration.ofMillis(100)); - return "foo"; - }); - Future f2 = executor.submit(() -> { - Thread.sleep(Duration.ofMillis(100)); - return "bar"; - }); - executor.shutdown(); - assertFalse(Thread.interrupted()); - assertTrue(executor.isShutdown()); - assertTrue(executor.awaitTermination(200, TimeUnit.MILLISECONDS)); - assertTrue(executor.isTerminated()); - assertEquals("foo", f1.resultNow()); - assertEquals("bar", f2.resultNow()); - }}); - } + testInNewThread(new TestAction() { void run() throws Exception { + Future f1 = executor.submit(() -> { + Thread.sleep(Duration.ofMillis(100)); + return "foo"; + }); + Future f2 = executor.submit(() -> { + Thread.sleep(Duration.ofMillis(100)); + return "bar"; + }); + executor.shutdown(); + assertFalse(Thread.interrupted()); + assertTrue(executor.isShutdown()); + assertTrue(executor.awaitTermination(200, TimeUnit.MILLISECONDS)); + assertTrue(executor.isTerminated()); + assertEquals("foo", f1.resultNow()); + assertEquals("bar", f2.resultNow()); + }}); } /** diff --git a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java index ec963e222c83f..f369e06521ae7 100644 --- a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java +++ b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java @@ -516,9 +516,9 @@ public void realRun() throws InterruptedException { pool = p; p.execute(f); } + assertTrue(pool != null && pool.isTerminated()); assertFalse(Thread.interrupted()); checkCompletedNormally(f); - assertTrue(pool != null && pool.isTerminated()); }}); awaitTermination(t); } @@ -533,9 +533,9 @@ public void realRun() throws InterruptedException { ForkJoinTask f = new FibAction(8); pool.execute(f); pool.close(); + assertTrue(pool.isTerminated()); assertFalse(Thread.interrupted()); checkCompletedNormally(f); - assertTrue(pool != null && pool.isTerminated()); }}); awaitTermination(t); } @@ -551,9 +551,9 @@ public void realRun() throws InterruptedException { pool.execute(f); pool.shutdown(); pool.close(); + assertTrue(pool.isTerminated()); assertFalse(Thread.interrupted()); checkCompletedNormally(f); - assertTrue(pool != null && pool.isTerminated()); }}); awaitTermination(t); } diff --git a/test/jdk/java/util/concurrent/tck/JSR166TestCase.java b/test/jdk/java/util/concurrent/tck/JSR166TestCase.java index 130831dcd2dd5..e14db7ce9c076 100644 --- a/test/jdk/java/util/concurrent/tck/JSR166TestCase.java +++ b/test/jdk/java/util/concurrent/tck/JSR166TestCase.java @@ -1647,11 +1647,15 @@ void checkTimedGet(Future f, T expectedValue) { checkTimedGet(f, expectedValue, LONG_DELAY_MS); } + // Avoids unwanted interrupts when run inder jtreg + static final ThreadGroup jsr166TestThreadGroup = + new ThreadGroup("jsr1666TestThreadGroup"); + /** * Returns a new started daemon Thread running the given runnable. */ Thread newStartedThread(Runnable runnable) { - Thread t = new Thread(runnable); + Thread t = new Thread(jsr166TestThreadGroup, runnable); t.setDaemon(true); t.start(); return t; From 5359956630e57a1c0d874f5835289ccaf2473824 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Fri, 1 Sep 2023 07:32:34 -0400 Subject: [PATCH 37/61] Fix tests, undo workarounds --- .../java/util/concurrent/ForkJoinPool.java | 64 ++++++++++--------- .../concurrent/tck/ForkJoinPool19Test.java | 6 ++ 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index bfed64fdb0d97..f44ade365282f 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1916,11 +1916,13 @@ else if ((int)c == 0) // was dropped on timeout stealCount += ns; // accumulate steals } unlockRunState(); - if (stop == 0 && replaceable) + } + if ((runState & STOP) == 0) { + if (replaceable) signalWork(); // may replace unless trimmed or uninitialized + if (ex != null) + ForkJoinTask.rethrow(ex); } - if (ex != null) - ForkJoinTask.rethrow(ex); } /** @@ -2771,7 +2773,7 @@ static int getSurplusQueuedTaskCount() { * @param now if true, unconditionally terminate, else only * if no work and no active workers * @param enable if true, terminate when next possible - * @return runState (possibly only its status bits) on exit + * @return runState on exit */ private int tryTerminate(boolean now, boolean enable) { int e, isShutdown; @@ -2782,43 +2784,45 @@ private int tryTerminate(boolean now, boolean enable) { if ((isShutdown = (e & SHUTDOWN)) == 0 && enable) getAndBitwiseOrRunState(isShutdown = SHUTDOWN); if (isShutdown != 0 && isQuiescent(true)) - e = runState; + e = STOP | SHUTDOWN; } } - if ((e & (STOP | TERMINATED)) == STOP) { - boolean alive = false; - int r = ThreadLocalRandom.nextSecondarySeed(); // stagger traversals - WorkQueue[] qs = queues; - int n = (qs == null) ? 0 : qs.length; - for (int l = n; l > 0; --l, ++r) { - WorkQueue q; Thread o; // cancel tasks; interrupt workers - if ((q = qs[r & SMASK & (n - 1)]) != null) { - if ((o = q.owner) != null) { - alive = true; - if (o != Thread.currentThread()) { + if ((e & (STOP | TERMINATED)) == STOP) { // similar to isQuiescent + for (int r = ThreadLocalRandom.nextSecondarySeed();;) { // stagger + int prevRunState = e; + WorkQueue[] qs = queues; + int n = (qs == null) ? 0 : qs.length; + for (int l = n; l > 0; --l, ++r) { + WorkQueue q; Thread o; // interrupt workers, cancel tasks + if ((q = qs[r & SMASK & (n - 1)]) != null) { + if ((o = q.owner) != null && !o.isInterrupted()) { try { o.interrupt(); } catch (Throwable ignore) { } } - } - for (ForkJoinTask t; (t = q.poll(null)) != null; ) { - try { - t.cancel(false); - } catch (Throwable ignore) { + for (ForkJoinTask t; (t = q.poll(null)) != null; ) { + try { + t.cancel(false); + } catch (Throwable ignore) { + } } } } - } - if (((e = runState) & TERMINATED) == 0 && !alive && ctl == 0L) { - if ((getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0) { - CountDownLatch done; SharedThreadContainer ctr; - if ((done = termination) != null) - done.countDown(); - if ((ctr = container) != null) - ctr.close(); + if (((e = runState) & TERMINATED) != 0) + break; + if (e == prevRunState && (e & RS_LOCK) == 0) { + if (ctl != 0L) // another thread will finish + break; + if (casRunState(e, e = (e | TERMINATED) + RS_EPOCH)) { + CountDownLatch done; SharedThreadContainer ctr; + if ((done = termination) != null) + done.countDown(); + if ((ctr = container) != null) + ctr.close(); + break; + } } - e = runState; } } return e; diff --git a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java index f369e06521ae7..b31d301b90532 100644 --- a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java +++ b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java @@ -515,9 +515,11 @@ public void realRun() throws InterruptedException { try (ForkJoinPool p = new ForkJoinPool()) { pool = p; p.execute(f); + assertFalse(Thread.interrupted()); } assertTrue(pool != null && pool.isTerminated()); assertFalse(Thread.interrupted()); + f.quietlyJoin(); checkCompletedNormally(f); }}); awaitTermination(t); @@ -532,9 +534,11 @@ public void realRun() throws InterruptedException { ForkJoinPool pool = new ForkJoinPool(); ForkJoinTask f = new FibAction(8); pool.execute(f); + assertFalse(Thread.interrupted()); pool.close(); assertTrue(pool.isTerminated()); assertFalse(Thread.interrupted()); + f.quietlyJoin(); checkCompletedNormally(f); }}); awaitTermination(t); @@ -550,9 +554,11 @@ public void realRun() throws InterruptedException { ForkJoinTask f = new FibAction(8); pool.execute(f); pool.shutdown(); + assertFalse(Thread.interrupted()); pool.close(); assertTrue(pool.isTerminated()); assertFalse(Thread.interrupted()); + f.quietlyJoin(); checkCompletedNormally(f); }}); awaitTermination(t); From b35efb077012d0714728005bce355b4ee18317c7 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sat, 2 Sep 2023 07:11:13 -0400 Subject: [PATCH 38/61] Ignore more stray interrupts by test harness --- .../util/concurrent/ExecutorService/CloseTest.java | 12 +++++++++--- .../jdk/java/util/concurrent/tck/JSR166TestCase.java | 10 ++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java index af07982d165d9..4786682fcad47 100644 --- a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java +++ b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java @@ -71,7 +71,13 @@ static void testInNewThread(TestAction a) throws Exception { try { Thread t = new Thread(closeTestThreadGroup, wrapper); t.start(); - t.join(); + for (;;) { // ignore stray test harness exceptions + try { + t.join(); + break; + } catch (InterruptedException ignode) { + } + } } finally { Exception e = wrapper.error; if (e != null) @@ -139,7 +145,7 @@ void testShutdownWithRunningTasks(ExecutorService executor) throws Exception { executor.shutdown(); assertFalse(Thread.interrupted()); assertTrue(executor.isShutdown()); - assertTrue(executor.awaitTermination(200, TimeUnit.MILLISECONDS)); + assertTrue(executor.awaitTermination(1000, TimeUnit.MILLISECONDS)); assertTrue(executor.isTerminated()); assertEquals("foo", future.resultNow()); }}); @@ -188,7 +194,7 @@ void testShutdownWith2RunningTasks(ExecutorService executor) throws Exception { executor.shutdown(); assertFalse(Thread.interrupted()); assertTrue(executor.isShutdown()); - assertTrue(executor.awaitTermination(200, TimeUnit.MILLISECONDS)); + assertTrue(executor.awaitTermination(1000, TimeUnit.MILLISECONDS)); assertTrue(executor.isTerminated()); assertEquals("foo", f1.resultNow()); assertEquals("bar", f2.resultNow()); diff --git a/test/jdk/java/util/concurrent/tck/JSR166TestCase.java b/test/jdk/java/util/concurrent/tck/JSR166TestCase.java index e14db7ce9c076..3b3905a681ff2 100644 --- a/test/jdk/java/util/concurrent/tck/JSR166TestCase.java +++ b/test/jdk/java/util/concurrent/tck/JSR166TestCase.java @@ -1675,10 +1675,12 @@ Thread newStartedThread(Action action) { * the thread (in the hope that it may terminate later) and fails. */ void awaitTermination(Thread thread, long timeoutMillis) { - try { - thread.join(timeoutMillis); - } catch (InterruptedException fail) { - threadUnexpectedException(fail); + for (;;) { // ignore stray interrupts by test harness + try { + thread.join(timeoutMillis); + break; + } catch (InterruptedException ignore) { + } } if (thread.getState() != Thread.State.TERMINATED) { String detail = String.format( From 9d590699653a78e9f24da70cf047a8be86a3df78 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sun, 3 Sep 2023 07:01:30 -0400 Subject: [PATCH 39/61] Avoid jtreg test group --- .../concurrent/ExecutorService/CloseTest.java | 17 ++++++++++++++++- .../util/concurrent/tck/JSR166TestCase.java | 16 +++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java index 4786682fcad47..07d38f394ce17 100644 --- a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java +++ b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java @@ -63,9 +63,24 @@ public void run() { } } } + // Avoids unwanted interrupts when run inder jtreg + static ThreadGroup topThreadGroup() { + ThreadGroup g = Thread.currentThread().getThreadGroup(); + for (ThreadGroup p;;) { + try { + p = g.getParent(); + } catch (Exception ok) { // possible under SecurityManager + break; + } + if (p == null) + break; + g = p; + } + return g; + } static final ThreadGroup closeTestThreadGroup = - new ThreadGroup("closeTestThreadGroup"); + new ThreadGroup(topThreadGroup(), "closeTestThreadGroup"); static void testInNewThread(TestAction a) throws Exception { var wrapper = new CheckedAction(a); try { diff --git a/test/jdk/java/util/concurrent/tck/JSR166TestCase.java b/test/jdk/java/util/concurrent/tck/JSR166TestCase.java index 3b3905a681ff2..506e90948b611 100644 --- a/test/jdk/java/util/concurrent/tck/JSR166TestCase.java +++ b/test/jdk/java/util/concurrent/tck/JSR166TestCase.java @@ -1648,8 +1648,22 @@ void checkTimedGet(Future f, T expectedValue) { } // Avoids unwanted interrupts when run inder jtreg + static ThreadGroup topThreadGroup() { + ThreadGroup g = Thread.currentThread().getThreadGroup(); + for (ThreadGroup p;;) { + try { + p = g.getParent(); + } catch (Exception ok) { // possible under SecurityManager + break; + } + if (p == null) + break; + g = p; + } + return g; + } static final ThreadGroup jsr166TestThreadGroup = - new ThreadGroup("jsr1666TestThreadGroup"); + new ThreadGroup(topThreadGroup(), "jsr1666TestThreadGroup"); /** * Returns a new started daemon Thread running the given runnable. From 787c1cb1120b7c643df252f05ed29223fa00158e Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Tue, 5 Sep 2023 16:09:56 -0400 Subject: [PATCH 40/61] Avoid needing test threads --- .../concurrent/ExecutorService/CloseTest.java | 331 ++++++++---------- 1 file changed, 143 insertions(+), 188 deletions(-) diff --git a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java index 07d38f394ce17..cab28f45d4a35 100644 --- a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java +++ b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java @@ -45,61 +45,6 @@ class CloseTest { - // setup to ensure each test runs under a new thread, to preclude - // stale interrupts from changing expected close() behavior - - static abstract class TestAction { - abstract void run() throws Exception; - } - static final class CheckedAction implements Runnable { - final TestAction action; - volatile Exception error; - CheckedAction(TestAction a) { action = a; } - public void run() { - try { - action.run(); - } catch (Exception ex) { - error = ex; - } - } - } - - // Avoids unwanted interrupts when run inder jtreg - static ThreadGroup topThreadGroup() { - ThreadGroup g = Thread.currentThread().getThreadGroup(); - for (ThreadGroup p;;) { - try { - p = g.getParent(); - } catch (Exception ok) { // possible under SecurityManager - break; - } - if (p == null) - break; - g = p; - } - return g; - } - static final ThreadGroup closeTestThreadGroup = - new ThreadGroup(topThreadGroup(), "closeTestThreadGroup"); - static void testInNewThread(TestAction a) throws Exception { - var wrapper = new CheckedAction(a); - try { - Thread t = new Thread(closeTestThreadGroup, wrapper); - t.start(); - for (;;) { // ignore stray test harness exceptions - try { - t.join(); - break; - } catch (InterruptedException ignode) { - } - } - } finally { - Exception e = wrapper.error; - if (e != null) - throw e; - } - } - static Stream executors() { return Stream.of( // ensures that default close method is tested @@ -118,12 +63,10 @@ static Stream executors() { @ParameterizedTest @MethodSource("executors") void testCloseWithNoTasks(ExecutorService executor) throws Exception { - testInNewThread(new TestAction() { void run() throws Exception { - executor.close(); - assertTrue(executor.isShutdown()); - assertTrue(executor.isTerminated()); - assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - }}); + executor.close(); + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); } /** @@ -132,18 +75,20 @@ void testCloseWithNoTasks(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testCloseWithRunningTasks(ExecutorService executor) throws Exception { - testInNewThread(new TestAction() { void run() throws Exception { - Future future = executor.submit(() -> { - Thread.sleep(Duration.ofMillis(100)); - return "foo"; - }); - executor.close(); // waits for task to complete - assertFalse(Thread.interrupted()); - assertTrue(executor.isShutdown()); - assertTrue(executor.isTerminated()); - assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - assertEquals("foo", future.resultNow()); - }}); + Phaser phaser = new Phaser(2); + Future future = executor.submit(() -> { + phaser.arriveAndAwaitAdvance(); + Thread.sleep(Duration.ofMillis(100)); + return "foo"; + }); + phaser.arriveAndAwaitAdvance(); // wait for task to start + + executor.close(); // waits for task to complete + assertFalse(Thread.interrupted()); + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + assertEquals("foo", future.resultNow()); } /** @@ -152,18 +97,20 @@ void testCloseWithRunningTasks(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testShutdownWithRunningTasks(ExecutorService executor) throws Exception { - testInNewThread(new TestAction() { void run() throws Exception { - Future future = executor.submit(() -> { - Thread.sleep(Duration.ofMillis(100)); - return "foo"; - }); - executor.shutdown(); - assertFalse(Thread.interrupted()); - assertTrue(executor.isShutdown()); - assertTrue(executor.awaitTermination(1000, TimeUnit.MILLISECONDS)); - assertTrue(executor.isTerminated()); - assertEquals("foo", future.resultNow()); - }}); + Phaser phaser = new Phaser(2); + Future future = executor.submit(() -> { + phaser.arriveAndAwaitAdvance(); + Thread.sleep(Duration.ofMillis(100)); + return "foo"; + }); + phaser.arriveAndAwaitAdvance(); // wait for task to start + + executor.shutdown(); + assertFalse(Thread.interrupted()); + assertTrue(executor.isShutdown()); + assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES)); + assertTrue(executor.isTerminated()); + assertEquals("foo", future.resultNow()); } /** @@ -172,23 +119,26 @@ void testShutdownWithRunningTasks(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testCloseWith2RunningTasks(ExecutorService executor) throws Exception { - testInNewThread(new TestAction() { void run() throws Exception { - Future f1 = executor.submit(() -> { - Thread.sleep(Duration.ofMillis(100)); - return "foo"; - }); - Future f2 = executor.submit(() -> { - Thread.sleep(Duration.ofMillis(100)); - return "bar"; - }); - executor.close(); // waits for task to complete - assertFalse(Thread.interrupted()); - assertTrue(executor.isShutdown()); - assertTrue(executor.isTerminated()); - assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - assertEquals("foo", f1.resultNow()); - assertEquals("bar", f2.resultNow()); - }}); + Phaser phaser = new Phaser(3); + Future f1 = executor.submit(() -> { + phaser.arriveAndAwaitAdvance(); + Thread.sleep(Duration.ofMillis(100)); + return "foo"; + }); + Future f2 = executor.submit(() -> { + phaser.arriveAndAwaitAdvance(); + Thread.sleep(Duration.ofMillis(100)); + return "bar"; + }); + phaser.arriveAndAwaitAdvance(); // wait for tasks to start + + executor.close(); // waits for task to complete + assertFalse(Thread.interrupted()); + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + assertEquals("foo", f1.resultNow()); + assertEquals("bar", f2.resultNow()); } /** @@ -197,23 +147,26 @@ void testCloseWith2RunningTasks(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testShutdownWith2RunningTasks(ExecutorService executor) throws Exception { - testInNewThread(new TestAction() { void run() throws Exception { - Future f1 = executor.submit(() -> { - Thread.sleep(Duration.ofMillis(100)); - return "foo"; - }); - Future f2 = executor.submit(() -> { - Thread.sleep(Duration.ofMillis(100)); - return "bar"; - }); - executor.shutdown(); - assertFalse(Thread.interrupted()); - assertTrue(executor.isShutdown()); - assertTrue(executor.awaitTermination(1000, TimeUnit.MILLISECONDS)); - assertTrue(executor.isTerminated()); - assertEquals("foo", f1.resultNow()); - assertEquals("bar", f2.resultNow()); - }}); + Phaser phaser = new Phaser(3); + Future f1 = executor.submit(() -> { + phaser.arriveAndAwaitAdvance(); + Thread.sleep(Duration.ofMillis(100)); + return "foo"; + }); + Future f2 = executor.submit(() -> { + phaser.arriveAndAwaitAdvance(); + Thread.sleep(Duration.ofMillis(100)); + return "bar"; + }); + phaser.arriveAndAwaitAdvance(); // wait for tasks to start + + executor.shutdown(); + assertFalse(Thread.interrupted()); + assertTrue(executor.isShutdown()); + assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES)); + assertTrue(executor.isTerminated()); + assertEquals("foo", f1.resultNow()); + assertEquals("bar", f2.resultNow()); } /** @@ -222,21 +175,27 @@ void testShutdownWith2RunningTasks(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testShutdownBeforeClose(ExecutorService executor) throws Exception { - testInNewThread(new TestAction() { void run() throws Exception { - Phaser phaser = new Phaser(2); - Future future = executor.submit(() -> { - phaser.arriveAndAwaitAdvance(); - Thread.sleep(Duration.ofMillis(100)); - return "foo"; - }); - phaser.arriveAndAwaitAdvance(); // wait for task to start - executor.shutdown(); // shutdown, will not immediately terminate - executor.close(); - assertTrue(executor.isShutdown()); - assertTrue(executor.isTerminated()); - assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - assertEquals("foo", future.resultNow()); - }}); + Phaser phaser = new Phaser(2); + Future future = executor.submit(() -> { + phaser.arriveAndAwaitAdvance(); + Thread.sleep(Duration.ofMillis(100)); + return "foo"; + }); + phaser.arriveAndAwaitAdvance(); // wait for task to start + + executor.shutdown(); // shutdown, will not immediately terminate + executor.close(); + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + try { + Object s = future.resultNow(); + assertEquals("foo", s); + } catch (Exception e) { + System.err.println("future => " + future.state()); + e.printStackTrace(); + fail(); + } } /** @@ -245,14 +204,12 @@ void testShutdownBeforeClose(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testTerminateBeforeClose(ExecutorService executor) throws Exception { - testInNewThread(new TestAction() { void run() throws Exception { - executor.shutdown(); - assertTrue(executor.isTerminated()); - executor.close(); - assertTrue(executor.isShutdown()); - assertTrue(executor.isTerminated()); - assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - }}); + executor.shutdown(); + assertTrue(executor.isTerminated()); + executor.close(); + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); } /** @@ -261,26 +218,25 @@ void testTerminateBeforeClose(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testInterruptBeforeClose(ExecutorService executor) throws Exception { - testInNewThread(new TestAction() { void run() throws Exception { - Phaser phaser = new Phaser(2); - Future future = executor.submit(() -> { - phaser.arriveAndAwaitAdvance(); - Thread.sleep(Duration.ofDays(1)); - return null; - }); - phaser.arriveAndAwaitAdvance(); // wait for task to start - Thread.currentThread().interrupt(); - try { - executor.close(); - assertTrue(Thread.currentThread().isInterrupted()); - } finally { - Thread.interrupted(); // clear interrupt status - } - assertTrue(executor.isShutdown()); - assertTrue(executor.isTerminated()); - assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - assertThrows(ExecutionException.class, future::get); - }}); + Phaser phaser = new Phaser(2); + Future future = executor.submit(() -> { + phaser.arriveAndAwaitAdvance(); + Thread.sleep(Duration.ofDays(1)); + return null; + }); + phaser.arriveAndAwaitAdvance(); // wait for task to start + + Thread.currentThread().interrupt(); + try { + executor.close(); + assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); // clear interrupt status + } + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + assertThrows(ExecutionException.class, future::get); } /** @@ -289,32 +245,31 @@ void testInterruptBeforeClose(ExecutorService executor) throws Exception { @ParameterizedTest @MethodSource("executors") void testInterruptDuringClose(ExecutorService executor) throws Exception { - testInNewThread(new TestAction() { void run() throws Exception { - Phaser phaser = new Phaser(2); - Future future = executor.submit(() -> { - phaser.arriveAndAwaitAdvance(); - Thread.sleep(Duration.ofDays(1)); - return null; - }); - phaser.arriveAndAwaitAdvance(); // wait for task to start - // schedule main thread to be interrupted - Thread thread = Thread.currentThread(); - new Thread(() -> { - try { - Thread.sleep( Duration.ofMillis(100)); - } catch (Exception ignore) { } - thread.interrupt(); - }).start(); + Phaser phaser = new Phaser(2); + Future future = executor.submit(() -> { + phaser.arriveAndAwaitAdvance(); + Thread.sleep(Duration.ofDays(1)); + return null; + }); + phaser.arriveAndAwaitAdvance(); // wait for task to start + + // schedule main thread to be interrupted + Thread thread = Thread.currentThread(); + new Thread(() -> { try { - executor.close(); - assertTrue(Thread.currentThread().isInterrupted()); - } finally { - Thread.interrupted(); // clear interrupt status - } - assertTrue(executor.isShutdown()); - assertTrue(executor.isTerminated()); - assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - assertThrows(ExecutionException.class, future::get); - }}); + Thread.sleep( Duration.ofMillis(100)); + } catch (Exception ignore) { } + thread.interrupt(); + }).start(); + try { + executor.close(); + assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); // clear interrupt status + } + assertTrue(executor.isShutdown()); + assertTrue(executor.isTerminated()); + assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); + assertThrows(ExecutionException.class, future::get); } } From f2dd803e73faeb3349ba5601e30f3249beaa8368 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Wed, 6 Sep 2023 09:01:22 -0400 Subject: [PATCH 41/61] Allow ThreadGroup access in tck tests --- .../java/util/concurrent/tck/JSR166TestCase.java | 15 +++------------ test/jdk/java/util/concurrent/tck/tck.policy | 1 + 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/test/jdk/java/util/concurrent/tck/JSR166TestCase.java b/test/jdk/java/util/concurrent/tck/JSR166TestCase.java index 506e90948b611..96fce2026df54 100644 --- a/test/jdk/java/util/concurrent/tck/JSR166TestCase.java +++ b/test/jdk/java/util/concurrent/tck/JSR166TestCase.java @@ -1649,18 +1649,9 @@ void checkTimedGet(Future f, T expectedValue) { // Avoids unwanted interrupts when run inder jtreg static ThreadGroup topThreadGroup() { - ThreadGroup g = Thread.currentThread().getThreadGroup(); - for (ThreadGroup p;;) { - try { - p = g.getParent(); - } catch (Exception ok) { // possible under SecurityManager - break; - } - if (p == null) - break; - g = p; - } - return g; + for (ThreadGroup g = Thread.currentThread().getThreadGroup(), p; ; g = p) + if ((p = g.getParent()) == null) + return g; } static final ThreadGroup jsr166TestThreadGroup = new ThreadGroup(topThreadGroup(), "jsr1666TestThreadGroup"); diff --git a/test/jdk/java/util/concurrent/tck/tck.policy b/test/jdk/java/util/concurrent/tck/tck.policy index 66f79ab3e392d..62018f792a3f0 100644 --- a/test/jdk/java/util/concurrent/tck/tck.policy +++ b/test/jdk/java/util/concurrent/tck/tck.policy @@ -1,6 +1,7 @@ grant { // Permissions j.u.c. needs directly permission java.lang.RuntimePermission "modifyThread"; + permission java.lang.RuntimePermission "modifyThreadGroup"; permission java.lang.RuntimePermission "getClassLoader"; permission java.lang.RuntimePermission "setContextClassLoader"; permission java.util.PropertyPermission "*", "read"; From e4b6426a41e61e9a345a7f55c2a0365b8ffbf47b Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Fri, 8 Sep 2023 09:05:08 -0400 Subject: [PATCH 42/61] Use non-recursive tasks in close tests --- .../concurrent/tck/ForkJoinPool19Test.java | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java index b31d301b90532..5b1dcfc2b348d 100644 --- a/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java +++ b/test/jdk/java/util/concurrent/tck/ForkJoinPool19Test.java @@ -510,17 +510,15 @@ public void testAdaptInterruptible_Callable_toString() { public void testClose() { Thread t = newStartedThread(new CheckedRunnable() { public void realRun() throws InterruptedException { - ForkJoinTask f = new FibAction(8); + FibAction f = new FibAction(1); ForkJoinPool pool = null; try (ForkJoinPool p = new ForkJoinPool()) { pool = p; p.execute(f); - assertFalse(Thread.interrupted()); } assertTrue(pool != null && pool.isTerminated()); - assertFalse(Thread.interrupted()); - f.quietlyJoin(); - checkCompletedNormally(f); + f.join(); + assertEquals(1, f.result); }}); awaitTermination(t); } @@ -532,14 +530,12 @@ public void testClose2() { Thread t = newStartedThread(new CheckedRunnable() { public void realRun() throws InterruptedException { ForkJoinPool pool = new ForkJoinPool(); - ForkJoinTask f = new FibAction(8); + FibAction f = new FibAction(1); pool.execute(f); - assertFalse(Thread.interrupted()); pool.close(); assertTrue(pool.isTerminated()); - assertFalse(Thread.interrupted()); - f.quietlyJoin(); - checkCompletedNormally(f); + f.join(); + assertEquals(1, f.result); }}); awaitTermination(t); } @@ -551,15 +547,13 @@ public void testClose3() { Thread t = newStartedThread(new CheckedRunnable() { public void realRun() throws InterruptedException { ForkJoinPool pool = new ForkJoinPool(); - ForkJoinTask f = new FibAction(8); + FibAction f = new FibAction(1); pool.execute(f); pool.shutdown(); - assertFalse(Thread.interrupted()); pool.close(); assertTrue(pool.isTerminated()); - assertFalse(Thread.interrupted()); - f.quietlyJoin(); - checkCompletedNormally(f); + f.join(); + assertEquals(1, f.result); }}); awaitTermination(t); } From 3e103e09fc67de14e8753727a65ce81a316ba541 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sat, 9 Sep 2023 08:36:19 -0400 Subject: [PATCH 43/61] Always help terminate when stopping --- .../java/util/concurrent/ForkJoinPool.java | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index f44ade365282f..d0518b5b8eab2 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -2160,14 +2160,12 @@ private int awaitWork(WorkQueue w, long queuedCtl, int idlePhase) { LockSupport.setCurrentBlocker(this); w.parker = Thread.currentThread(); for (;;) { - if ((p = w.phase) != idlePhase) + if ((runState & STOP) != 0 || (p = w.phase) != idlePhase) break; U.park(quiescent, deadline); - if ((p = w.phase) != idlePhase) + if ((p = w.phase) != idlePhase || (runState & STOP) != 0) break; Thread.interrupted(); // clear status for next park - if ((runState & STOP) != 0) - break; if (quiescent && // trim on timeout deadline - System.currentTimeMillis() < TIMEOUT_SLOP) { int id = idlePhase & SMASK; @@ -2784,12 +2782,13 @@ private int tryTerminate(boolean now, boolean enable) { if ((isShutdown = (e & SHUTDOWN)) == 0 && enable) getAndBitwiseOrRunState(isShutdown = SHUTDOWN); if (isShutdown != 0 && isQuiescent(true)) - e = STOP | SHUTDOWN; + e = runState; } } - if ((e & (STOP | TERMINATED)) == STOP) { // similar to isQuiescent - for (int r = ThreadLocalRandom.nextSecondarySeed();;) { // stagger - int prevRunState = e; + if ((e & STOP) != 0) { // help terminate; similar to isQuiescent + int r = (int)Thread.currentThread().threadId(); // stagger traversals + long c = ctl; + for (int prevRunState = 0; ; prevRunState = e) { WorkQueue[] qs = queues; int n = (qs == null) ? 0 : qs.length; for (int l = n; l > 0; --l, ++r) { @@ -2809,10 +2808,11 @@ private int tryTerminate(boolean now, boolean enable) { } } } - if (((e = runState) & TERMINATED) != 0) - break; - if (e == prevRunState && (e & RS_LOCK) == 0) { - if (ctl != 0L) // another thread will finish + if (c == (c = ctl) && (e = runState) == prevRunState && + (e & RS_LOCK) == 0) { + if ((e & TERMINATED) != 0) + break; + if (c != 0L) // another thread will finish break; if (casRunState(e, e = (e | TERMINATED) + RS_EPOCH)) { CountDownLatch done; SharedThreadContainer ctr; @@ -2820,7 +2820,6 @@ private int tryTerminate(boolean now, boolean enable) { done.countDown(); if ((ctr = container) != null) ctr.close(); - break; } } } @@ -3821,17 +3820,16 @@ public boolean awaitQuiescence(long timeout, TimeUnit unit) { */ @Override public void close() { - if (workerNamePrefix != null && (runState & TERMINATED) == 0) { + if (workerNamePrefix != null) { checkPermission(); CountDownLatch done = null; - boolean interrupted = false; // Thread.interrupted(); + boolean interrupted = false; while ((tryTerminate(interrupted, true) & TERMINATED) == 0) { if (done == null) done = terminationSignal(); else { try { done.await(); - break; } catch (InterruptedException ex) { interrupted = true; } From 31a1f34e59517b09cbcb95e854cf499d0986c743 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sun, 10 Sep 2023 07:37:53 -0400 Subject: [PATCH 44/61] Simplify signalling --- .../java/util/concurrent/ForkJoinPool.java | 67 ++++++++----------- 1 file changed, 28 insertions(+), 39 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index d0518b5b8eab2..6221070442fa9 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -621,10 +621,7 @@ public class ForkJoinPool extends AbstractExecutorService { * this causes enough memory traffic and CAS contention to prefer * using quieter short spinwaits in awaitWork and elsewhere. * Those in awaitWork are set to small values that only cover - * near-miss scenarios for inactivate/activate races. Because idle - * workers are often not yet blocked (parked), we use the - * WorkQueue parker field to advertise that a waiter actually - * needs unparking upon signal. + * near-miss scenarios for inactivate/activate races. * * Quiescence. Workers scan looking for work, giving up when they * don't find any, without being sure that none are available. @@ -1044,7 +1041,7 @@ public class ForkJoinPool extends AbstractExecutorService { static final int PRESET_SIZE = 1 << 2; // size was set by property // others - static final int TRIMMED = 1 << 31; // timed out while idle + static final int DEREGISTERED = 1 << 31; // worker terminating static final int UNCOMPENSATE = 1 << 16; // tryCompensate return static final int IDLE = 1 << 16; // phase seqlock/version count static final long RESCAN = 1L << 63; // window retry indicator @@ -1214,8 +1211,7 @@ public ForkJoinWorkerThread run() { */ static final class WorkQueue { // fields declared in order of their likely layout on most VMs - volatile ForkJoinWorkerThread owner; // null if shared or terminated - volatile Thread parker; // set when parking in awaitWork + final ForkJoinWorkerThread owner; // null if shared or terminated ForkJoinTask[] array; // the queued tasks; power of 2 size int base; // index of next slot for poll final int config; // mode bits @@ -1228,7 +1224,7 @@ static final class WorkQueue { @jdk.internal.vm.annotation.Contended("w") int stackPred; // pool stack (ctl) predecessor link @jdk.internal.vm.annotation.Contended("w") - volatile int source; // source queue id (or TRIMMED if trimmed) + volatile int source; // source queue id (or DEREGISTERED) @jdk.internal.vm.annotation.Contended("w") int nsteals; // number of steals from other queues @@ -1831,9 +1827,7 @@ final void registerWorker(WorkQueue w) { int stop = lockRunState() & STOP; try { WorkQueue[] qs; int n; - if (stop != 0 || (qs = queues) == null || (n = qs.length) <= 0) - w.owner = null; // terminating - else { + if (stop == 0 && (qs = queues) != null && (n = qs.length) > 0) { for (int k = n, m = n - 1; ; id += 2) { if (qs[id &= m] == null) break; @@ -1881,24 +1875,23 @@ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { boolean replaceable = false; if (wt != null && (w = wt.workQueue) != null) { phase = w.phase; - src = w.source; - w.owner = null; // disable signals - if (phase != 0) { // else failed to start - replaceable = true; - if ((phase & IDLE) != 0) - reactivate(w); // pool stopped before released - if (w.top - w.base > 0) { + if ((src = w.source) != DEREGISTERED) { // else trimmed on timeout + w.source = DEREGISTERED; + if (phase != 0) { // else failed to start + replaceable = true; + if ((phase & IDLE) != 0) + reactivate(w); // pool stopped before released for (ForkJoinTask t; (t = w.nextLocalTask()) != null; ) { - try { - t.cancel(false); - } catch (Throwable ignore) { - } + try { + t.cancel(false); + } catch (Throwable ignore) { + } } } } } long c = ctl; - if (src != TRIMMED) // decrement counts + if (src != DEREGISTERED) // decrement counts do {} while (c != (c = compareAndExchangeCtl( c, ((RC_MASK & (c - RC_UNIT)) | (TC_MASK & (c - TC_UNIT)) | @@ -1950,10 +1943,8 @@ else if ((short)(c >>> RC_SHIFT) >= pc || (v = w) == null) if (v == null) createWorker(); else { - Thread t; v.phase = sp; - if ((t = v.parker) != null) - U.unpark(t); + U.unpark(v.owner); } break; } @@ -1975,8 +1966,7 @@ private void reactivate(WorkQueue w) { c, ((UMASK & (c + RC_UNIT)) | (c & TC_MASK) | (v.stackPred & LMASK))))) { v.phase = sp; - if ((t = v.parker) != null) - U.unpark(t); + U.unpark(v.owner); if (v == w) break; } @@ -2004,10 +1994,9 @@ else if ((c & RC_MASK) > 0L) else if (ctl != c) // re-snapshot swept = false; else if (swept && e == prevRunState && (e & RS_LOCK) == 0 && - (!transition || - casRunState(e, ((e & SHUTDOWN) == 0 ? - e + RS_EPOCH : // advance - e | STOP)))) // terminate + casRunState(e, !transition ? e : // confirm + (e & SHUTDOWN) == 0 ? e + RS_EPOCH : // advance + e | STOP)) // terminate return true; else { long sum = 0L; @@ -2158,7 +2147,6 @@ private int awaitWork(WorkQueue w, long queuedCtl, int idlePhase) { if (p == idlePhase) { // emulate LockSupport.park LockSupport.setCurrentBlocker(this); - w.parker = Thread.currentThread(); for (;;) { if ((runState & STOP) != 0 || (p = w.phase) != idlePhase) break; @@ -2172,14 +2160,13 @@ private int awaitWork(WorkQueue w, long queuedCtl, int idlePhase) { long sp = w.stackPred & LMASK; long c = ctl, nc = sp | (UMASK & (c - TC_UNIT)); if (((int)c & SMASK) == id && compareAndSetCtl(c, nc)) { - w.source = TRIMMED; // sentinel for deregisterWorker + w.source = DEREGISTERED; // deregisterWorker sentinel w.phase = idlePhase + IDLE; break; } deadline += keepAlive; // not at head; restart timer } } - w.parker = null; LockSupport.setCurrentBlocker(null); } } @@ -2238,8 +2225,7 @@ private int tryCompensate(long c) { (v = qs[i]) != null && compareAndSetCtl(c, (c & UMASK) | (v.stackPred & LMASK))) { v.phase = sp; - if ((t = v.parker) != null) - U.unpark(t); + U.unpark(v.owner); stat = UNCOMPENSATE; } } @@ -2785,7 +2771,7 @@ private int tryTerminate(boolean now, boolean enable) { e = runState; } } - if ((e & STOP) != 0) { // help terminate; similar to isQuiescent + if ((e & (STOP | TERMINATED)) == STOP) { // similar to isQuiescent int r = (int)Thread.currentThread().threadId(); // stagger traversals long c = ctl; for (int prevRunState = 0; ; prevRunState = e) { @@ -2794,7 +2780,8 @@ private int tryTerminate(boolean now, boolean enable) { for (int l = n; l > 0; --l, ++r) { WorkQueue q; Thread o; // interrupt workers, cancel tasks if ((q = qs[r & SMASK & (n - 1)]) != null) { - if ((o = q.owner) != null && !o.isInterrupted()) { + if ((o = q.owner) != null && q.source != DEREGISTERED && + !o.isInterrupted()) { try { o.interrupt(); } catch (Throwable ignore) { @@ -2820,6 +2807,7 @@ private int tryTerminate(boolean now, boolean enable) { done.countDown(); if ((ctr = container) != null) ctr.close(); + break; } } } @@ -3830,6 +3818,7 @@ public void close() { else { try { done.await(); + break; } catch (InterruptedException ex) { interrupted = true; } From 4eeb5e5ae4f1de7a281a53ea6135907968355652 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Mon, 11 Sep 2023 12:48:11 -0400 Subject: [PATCH 45/61] Force more orderings; improve diagnostics --- .../java/util/concurrent/ForkJoinPool.java | 68 ++++++++----------- .../java/util/concurrent/ForkJoinTask.java | 11 ++- 2 files changed, 37 insertions(+), 42 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 6221070442fa9..b92785567aee3 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1733,9 +1733,6 @@ private CountDownLatch cmpExTerminationSignal(CountDownLatch x) { // runState operations - private int getAndBitwiseOrRunState(int v) { // for status bits - return U.getAndBitwiseOrInt(this, RUNSTATE, v); - } private boolean casRunState(int c, int v) { return U.compareAndSetInt(this, RUNSTATE, c, v); } @@ -1823,6 +1820,7 @@ final void registerWorker(WorkQueue w) { w.array = new ForkJoinTask[INITIAL_QUEUE_CAPACITY]; ThreadLocalRandom.localInit(); int seed = w.stackPred = ThreadLocalRandom.getProbe(); + int phaseSeq = seed & ~((IDLE << 1) - 1); // initial phase tag int id = ((seed << 1) | 1) & SMASK; // base of linear-probe-like scan int stop = lockRunState() & STOP; try { @@ -1836,7 +1834,7 @@ final void registerWorker(WorkQueue w) { break; } } - w.phase = id; // now publishable + w.phase = id | phaseSeq; // now publishable if (id < n) qs[id] = w; else { // expand @@ -1982,10 +1980,10 @@ private void reactivate(WorkQueue w) { * quiescent */ private boolean isQuiescent(boolean transition) { + U.fullFence(); long phaseSum = -1L; boolean swept = false; - outer: for (int e = 0;;) { - int prevRunState = e; + outer: for (int e, prevRunState = 0; ; prevRunState = e) { long c = ctl; if (((e = runState) & STOP) != 0) return true; // terminating @@ -1993,11 +1991,13 @@ else if ((c & RC_MASK) > 0L) break; // at least one active else if (ctl != c) // re-snapshot swept = false; - else if (swept && e == prevRunState && (e & RS_LOCK) == 0 && - casRunState(e, !transition ? e : // confirm - (e & SHUTDOWN) == 0 ? e + RS_EPOCH : // advance - e | STOP)) // terminate - return true; + else if (swept && e == prevRunState && (e & RS_LOCK) == 0) { + if (casRunState(e, !transition ? e : // confirm + (e & SHUTDOWN) == 0 ? e + RS_EPOCH : // advance + e | STOP)) // terminate + return true; + swept = false; + } else { long sum = 0L; WorkQueue[] qs = queues; @@ -2026,44 +2026,31 @@ else if (swept && e == prevRunState && (e & RS_LOCK) == 0 && */ final void runWorker(WorkQueue w) { if (w != null) { - int phase = w.phase, r = w.stackPred; // seed from registerWorker - int e = runState; // reset or exit when changed + int r = w.stackPred; // seed from registerWorker long next = 0L; - for (boolean stateChange = true;;) { + for (int e = 0;;) { + int phase; int window; // encodes origin and previous non-empty queue - if (stateChange) { - stateChange = false; + r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift + if (e != (e = runState)) { if ((e & STOP) != 0) // terminating break; // else use random origin window = (INVALID_ID << 16) | (r >>> 16); } else window = (int)next; // continue from last scan - if ((next = scan(w, window, (r << 1) | 1)) < 0L) - ; // continue scanning - else if (e != (e = runState)) - stateChange = true; - else { // try to inactivate & enqueue - boolean enqueued = true; - int idlePhase = phase + IDLE; + if ((next = scan(w, window, (r << 1) | 1)) >= 0L && + ((phase = w.phase) & IDLE) == 0) { + int idlePhase = phase + IDLE; // try to inactivate & enqueue long np = (phase + (IDLE << 1)) & LMASK, pc = ctl; long qc = ((pc - RC_UNIT) & UMASK) | np; w.stackPred = (int)pc; // set ctl stack link w.phase = idlePhase; // try to enqueue - if (pc != (pc = compareAndExchangeCtl(pc, qc))) { - qc = ((pc - RC_UNIT) & UMASK) | np; - w.stackPred = (int)pc; // retry once - if (pc != compareAndExchangeCtl(pc, qc)) - enqueued = false; - } - if (!enqueued) - w.phase = phase; // back out on contention - else if ((phase = awaitWork(w, qc, idlePhase)) == idlePhase) + if (pc != (pc = compareAndExchangeCtl(pc, qc))) + w.phase = phase; // back out contention + else if ((awaitWork(w, qc, idlePhase) & IDLE) != 0) break; // worker exit - else if (e != (e = runState)) // recheck after blocking - stateChange = true; } - r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift } } } @@ -2760,19 +2747,20 @@ static int getSurplusQueuedTaskCount() { * @return runState on exit */ private int tryTerminate(boolean now, boolean enable) { - int e, isShutdown; + int e; if (((e = runState) & STOP) == 0) { if (now) - getAndBitwiseOrRunState(e = STOP | SHUTDOWN); + runState = e = (lockRunState() + RS_LOCK) | STOP | SHUTDOWN; else { - if ((isShutdown = (e & SHUTDOWN)) == 0 && enable) - getAndBitwiseOrRunState(isShutdown = SHUTDOWN); - if (isShutdown != 0 && isQuiescent(true)) + if ((e & SHUTDOWN) == 0 && enable) + runState = e = (lockRunState() + RS_LOCK) | SHUTDOWN; + if ((e & SHUTDOWN) != 0 && isQuiescent(true)) e = runState; } } if ((e & (STOP | TERMINATED)) == STOP) { // similar to isQuiescent int r = (int)Thread.currentThread().threadId(); // stagger traversals + U.fullFence(); long c = ctl; for (int prevRunState = 0; ; prevRunState = e) { WorkQueue[] qs = queues; diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index daf639a27c441..e8085e54ecc14 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -899,8 +899,15 @@ public State state() { @Override public V resultNow() { - if (!isCompletedNormally()) - throw new IllegalStateException(); + int s = status; + if ((s & DONE) == 0) + throw new IllegalStateException("Task has not completed"); + if ((s & ABNORMAL) != 0) { + if ((s & THROWN) != 0) + throw new IllegalStateException("Task completed with exception"); + else + throw new IllegalStateException("Task was cancelled"); + } return getRawResult(); } From e8d7b75fa780d3a6b28e3d2e517383d02d92646c Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Wed, 13 Sep 2023 08:46:26 -0400 Subject: [PATCH 46/61] Strengthen translation of getAndX atomics; revert or adapt FJP accordingly --- .../java/util/concurrent/ForkJoinPool.java | 173 +++++++++--------- .../classes/jdk/internal/misc/Unsafe.java | 46 ++--- 2 files changed, 111 insertions(+), 108 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index b92785567aee3..de5cde1c23993 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -621,7 +621,10 @@ public class ForkJoinPool extends AbstractExecutorService { * this causes enough memory traffic and CAS contention to prefer * using quieter short spinwaits in awaitWork and elsewhere. * Those in awaitWork are set to small values that only cover - * near-miss scenarios for inactivate/activate races. + * near-miss scenarios for inactivate/activate races. Because idle + * workers are often not yet blocked (parked), we use the + * WorkQueue parker field to advertise that a waiter actually + * needs unparking upon signal. * * Quiescence. Workers scan looking for work, giving up when they * don't find any, without being sure that none are available. @@ -1211,7 +1214,8 @@ public ForkJoinWorkerThread run() { */ static final class WorkQueue { // fields declared in order of their likely layout on most VMs - final ForkJoinWorkerThread owner; // null if shared or terminated + final ForkJoinWorkerThread owner; // null if shared + volatile Thread parker; // set when parking in awaitWork ForkJoinTask[] array; // the queued tasks; power of 2 size int base; // index of next slot for poll final int config; // mode bits @@ -1733,6 +1737,9 @@ private CountDownLatch cmpExTerminationSignal(CountDownLatch x) { // runState operations + private int getAndBitwiseOrRunState(int v) { // for status bits + return U.getAndBitwiseOrInt(this, RUNSTATE, v); + } private boolean casRunState(int c, int v) { return U.compareAndSetInt(this, RUNSTATE, c, v); } @@ -1941,8 +1948,10 @@ else if ((short)(c >>> RC_SHIFT) >= pc || (v = w) == null) if (v == null) createWorker(); else { + Thread t; v.phase = sp; - U.unpark(v.owner); + if ((t = v.parker) != null) + U.unpark(t); } break; } @@ -1964,7 +1973,8 @@ private void reactivate(WorkQueue w) { c, ((UMASK & (c + RC_UNIT)) | (c & TC_MASK) | (v.stackPred & LMASK))))) { v.phase = sp; - U.unpark(v.owner); + if ((t = v.parker) != null) + U.unpark(t); if (v == w) break; } @@ -1980,42 +1990,43 @@ private void reactivate(WorkQueue w) { * quiescent */ private boolean isQuiescent(boolean transition) { - U.fullFence(); - long phaseSum = -1L; - boolean swept = false; - outer: for (int e, prevRunState = 0; ; prevRunState = e) { - long c = ctl; - if (((e = runState) & STOP) != 0) - return true; // terminating - else if ((c & RC_MASK) > 0L) - break; // at least one active - else if (ctl != c) // re-snapshot - swept = false; - else if (swept && e == prevRunState && (e & RS_LOCK) == 0) { - if (casRunState(e, !transition ? e : // confirm - (e & SHUTDOWN) == 0 ? e + RS_EPOCH : // advance - e | STOP)) // terminate - return true; - swept = false; - } - else { - long sum = 0L; - WorkQueue[] qs = queues; - int n = (qs == null) ? 0 : qs.length; - for (int i = 0; i < n; i += 2) { // scan external queues - WorkQueue q; int p; - if ((q = qs[i]) != null) { - if (((p = q.phase) & IDLE) == 0 || q.top - q.base > 0) { - signalWork(); // ensure live - break outer; + for (;;) { + long phaseSum = -1L; + boolean swept = false; + for (int e, prevRunState = 0; ; prevRunState = e) { + long c = ctl; + if (((e = runState) & STOP) != 0) + return true; // terminating + else if ((c & RC_MASK) > 0L) + return false; // at least one active + else if (!swept || e != prevRunState || (e & RS_LOCK) != 0) { + long sum = c; + WorkQueue[] qs = queues; + int n = (qs == null) ? 0 : qs.length; + for (int i = 0; i < n; ++i) { // scan queues + WorkQueue q; int p; + if ((q = qs[i]) == null) + sum += (long)(i + 1) << 48; + else if (((p = q.phase) & IDLE) == 0 || + q.top - q.base > 0) { + if ((i & 1) == 0) + signalWork(); // ensure live + return false; } - sum += p; + else + sum += p; } + swept = (phaseSum == (phaseSum = sum)); } - swept = (phaseSum == (phaseSum = sum)); + else if (compareAndSetCtl(c, c) && // confirm + casRunState(e, !transition ? e : + (e & SHUTDOWN) != 0 ? e | STOP : + e + RS_EPOCH)) // advance + return true; + else + break; // restart } } - return false; } /** @@ -2026,10 +2037,9 @@ else if (swept && e == prevRunState && (e & RS_LOCK) == 0) { */ final void runWorker(WorkQueue w) { if (w != null) { - int r = w.stackPred; // seed from registerWorker + int phase = w.phase, r = w.stackPred; // seed from registerWorker long next = 0L; for (int e = 0;;) { - int phase; int window; // encodes origin and previous non-empty queue r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift if (e != (e = runState)) { @@ -2039,16 +2049,15 @@ final void runWorker(WorkQueue w) { } else window = (int)next; // continue from last scan - if ((next = scan(w, window, (r << 1) | 1)) >= 0L && - ((phase = w.phase) & IDLE) == 0) { - int idlePhase = phase + IDLE; // try to inactivate & enqueue - long np = (phase + (IDLE << 1)) & LMASK, pc = ctl; - long qc = ((pc - RC_UNIT) & UMASK) | np; + if ((next = scan(w, window, (r << 1) | 1)) >= 0L) { + int idlePhase = phase + IDLE; // try to inactivate + long np = (phase + (IDLE << 1)) & LMASK; + long pc = ctl, qc = ((pc - RC_UNIT) & UMASK) | np; w.stackPred = (int)pc; // set ctl stack link w.phase = idlePhase; // try to enqueue - if (pc != (pc = compareAndExchangeCtl(pc, qc))) - w.phase = phase; // back out contention - else if ((awaitWork(w, qc, idlePhase) & IDLE) != 0) + if (pc != compareAndExchangeCtl(pc, qc)) + w.phase = phase; // back out on contention + else if ((phase = awaitWork(w, qc, idlePhase)) == idlePhase) break; // worker exit } } @@ -2134,6 +2143,7 @@ private int awaitWork(WorkQueue w, long queuedCtl, int idlePhase) { if (p == idlePhase) { // emulate LockSupport.park LockSupport.setCurrentBlocker(this); + w.parker = Thread.currentThread(); for (;;) { if ((runState & STOP) != 0 || (p = w.phase) != idlePhase) break; @@ -2154,6 +2164,7 @@ private int awaitWork(WorkQueue w, long queuedCtl, int idlePhase) { deadline += keepAlive; // not at head; restart timer } } + w.parker = null; LockSupport.setCurrentBlocker(null); } } @@ -2212,7 +2223,8 @@ private int tryCompensate(long c) { (v = qs[i]) != null && compareAndSetCtl(c, (c & UMASK) | (v.stackPred & LMASK))) { v.phase = sp; - U.unpark(v.owner); + if ((t = v.parker) != null) + U.unpark(t); stat = UNCOMPENSATE; } } @@ -2747,57 +2759,48 @@ static int getSurplusQueuedTaskCount() { * @return runState on exit */ private int tryTerminate(boolean now, boolean enable) { - int e; + int e, isShutdown; if (((e = runState) & STOP) == 0) { if (now) - runState = e = (lockRunState() + RS_LOCK) | STOP | SHUTDOWN; + getAndBitwiseOrRunState(e = STOP | SHUTDOWN); else { - if ((e & SHUTDOWN) == 0 && enable) - runState = e = (lockRunState() + RS_LOCK) | SHUTDOWN; - if ((e & SHUTDOWN) != 0 && isQuiescent(true)) + if ((isShutdown = (e & SHUTDOWN)) == 0 && enable) + getAndBitwiseOrRunState(isShutdown = SHUTDOWN); + if (isShutdown != 0 && isQuiescent(true)) e = runState; } } - if ((e & (STOP | TERMINATED)) == STOP) { // similar to isQuiescent + if ((e & (STOP | TERMINATED)) == STOP) { int r = (int)Thread.currentThread().threadId(); // stagger traversals - U.fullFence(); - long c = ctl; - for (int prevRunState = 0; ; prevRunState = e) { - WorkQueue[] qs = queues; - int n = (qs == null) ? 0 : qs.length; - for (int l = n; l > 0; --l, ++r) { - WorkQueue q; Thread o; // interrupt workers, cancel tasks - if ((q = qs[r & SMASK & (n - 1)]) != null) { - if ((o = q.owner) != null && q.source != DEREGISTERED && - !o.isInterrupted()) { - try { - o.interrupt(); - } catch (Throwable ignore) { - } + WorkQueue[] qs = queues; + int n = (qs == null) ? 0 : qs.length; + for (int l = n; l > 0; --l, ++r) { + int j; WorkQueue q; Thread o; // cancel tasks; interrupt workers + if ((q = qs[r & SMASK & (n - 1)]) != null && + q.source != DEREGISTERED) { + if ((o = q.owner) != null && !o.isInterrupted()) { + try { + o.interrupt(); + } catch (Throwable ignore) { } - for (ForkJoinTask t; (t = q.poll(null)) != null; ) { - try { - t.cancel(false); - } catch (Throwable ignore) { - } + } + for (ForkJoinTask t; (t = q.poll(null)) != null; ) { + try { + t.cancel(false); + } catch (Throwable ignore) { } } } - if (c == (c = ctl) && (e = runState) == prevRunState && - (e & RS_LOCK) == 0) { - if ((e & TERMINATED) != 0) - break; - if (c != 0L) // another thread will finish - break; - if (casRunState(e, e = (e | TERMINATED) + RS_EPOCH)) { - CountDownLatch done; SharedThreadContainer ctr; - if ((done = termination) != null) - done.countDown(); - if ((ctr = container) != null) - ctr.close(); - break; - } + } + if (((e = runState) & TERMINATED) == 0 && ctl == 0L) { + if ((getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0) { + CountDownLatch done; SharedThreadContainer ctr; + if ((done = termination) != null) + done.countDown(); + if ((ctr = container) != null) + ctr.close(); } + e = runState; } } return e; diff --git a/src/java.base/share/classes/jdk/internal/misc/Unsafe.java b/src/java.base/share/classes/jdk/internal/misc/Unsafe.java index 47e8bf80c50ed..66bf39d4835c8 100644 --- a/src/java.base/share/classes/jdk/internal/misc/Unsafe.java +++ b/src/java.base/share/classes/jdk/internal/misc/Unsafe.java @@ -2446,7 +2446,7 @@ public final int getAndAddInt(Object o, long offset, int delta) { int v; do { v = getIntVolatile(o, offset); - } while (!weakCompareAndSetInt(o, offset, v, v + delta)); + } while (!compareAndSetInt(o, offset, v, v + delta)); return v; } @@ -2484,7 +2484,7 @@ public final long getAndAddLong(Object o, long offset, long delta) { long v; do { v = getLongVolatile(o, offset); - } while (!weakCompareAndSetLong(o, offset, v, v + delta)); + } while (!compareAndSetLong(o, offset, v, v + delta)); return v; } @@ -2511,7 +2511,7 @@ public final byte getAndAddByte(Object o, long offset, byte delta) { byte v; do { v = getByteVolatile(o, offset); - } while (!weakCompareAndSetByte(o, offset, v, (byte) (v + delta))); + } while (!compareAndSetByte(o, offset, v, (byte) (v + delta))); return v; } @@ -2538,7 +2538,7 @@ public final short getAndAddShort(Object o, long offset, short delta) { short v; do { v = getShortVolatile(o, offset); - } while (!weakCompareAndSetShort(o, offset, v, (short) (v + delta))); + } while (!compareAndSetShort(o, offset, v, (short) (v + delta))); return v; } @@ -2585,7 +2585,7 @@ public final float getAndAddFloat(Object o, long offset, float delta) { // may result in the loop not terminating. expectedBits = getIntVolatile(o, offset); v = Float.intBitsToFloat(expectedBits); - } while (!weakCompareAndSetInt(o, offset, + } while (!compareAndSetInt(o, offset, expectedBits, Float.floatToRawIntBits(v + delta))); return v; } @@ -2630,7 +2630,7 @@ public final double getAndAddDouble(Object o, long offset, double delta) { // may result in the loop not terminating. expectedBits = getLongVolatile(o, offset); v = Double.longBitsToDouble(expectedBits); - } while (!weakCompareAndSetLong(o, offset, + } while (!compareAndSetLong(o, offset, expectedBits, Double.doubleToRawLongBits(v + delta))); return v; } @@ -2681,7 +2681,7 @@ public final int getAndSetInt(Object o, long offset, int newValue) { int v; do { v = getIntVolatile(o, offset); - } while (!weakCompareAndSetInt(o, offset, v, newValue)); + } while (!compareAndSetInt(o, offset, v, newValue)); return v; } @@ -2719,7 +2719,7 @@ public final long getAndSetLong(Object o, long offset, long newValue) { long v; do { v = getLongVolatile(o, offset); - } while (!weakCompareAndSetLong(o, offset, v, newValue)); + } while (!compareAndSetLong(o, offset, v, newValue)); return v; } @@ -2757,7 +2757,7 @@ public final Object getAndSetReference(Object o, long offset, Object newValue) { Object v; do { v = getReferenceVolatile(o, offset); - } while (!weakCompareAndSetReference(o, offset, v, newValue)); + } while (!compareAndSetReference(o, offset, v, newValue)); return v; } @@ -2784,7 +2784,7 @@ public final byte getAndSetByte(Object o, long offset, byte newValue) { byte v; do { v = getByteVolatile(o, offset); - } while (!weakCompareAndSetByte(o, offset, v, newValue)); + } while (!compareAndSetByte(o, offset, v, newValue)); return v; } @@ -2826,7 +2826,7 @@ public final short getAndSetShort(Object o, long offset, short newValue) { short v; do { v = getShortVolatile(o, offset); - } while (!weakCompareAndSetShort(o, offset, v, newValue)); + } while (!compareAndSetShort(o, offset, v, newValue)); return v; } @@ -2954,7 +2954,7 @@ public final byte getAndBitwiseOrByte(Object o, long offset, byte mask) { byte current; do { current = getByteVolatile(o, offset); - } while (!weakCompareAndSetByte(o, offset, + } while (!compareAndSetByte(o, offset, current, (byte) (current | mask))); return current; } @@ -2985,7 +2985,7 @@ public final byte getAndBitwiseAndByte(Object o, long offset, byte mask) { byte current; do { current = getByteVolatile(o, offset); - } while (!weakCompareAndSetByte(o, offset, + } while (!compareAndSetByte(o, offset, current, (byte) (current & mask))); return current; } @@ -3016,7 +3016,7 @@ public final byte getAndBitwiseXorByte(Object o, long offset, byte mask) { byte current; do { current = getByteVolatile(o, offset); - } while (!weakCompareAndSetByte(o, offset, + } while (!compareAndSetByte(o, offset, current, (byte) (current ^ mask))); return current; } @@ -3094,7 +3094,7 @@ public final short getAndBitwiseOrShort(Object o, long offset, short mask) { short current; do { current = getShortVolatile(o, offset); - } while (!weakCompareAndSetShort(o, offset, + } while (!compareAndSetShort(o, offset, current, (short) (current | mask))); return current; } @@ -3125,7 +3125,7 @@ public final short getAndBitwiseAndShort(Object o, long offset, short mask) { short current; do { current = getShortVolatile(o, offset); - } while (!weakCompareAndSetShort(o, offset, + } while (!compareAndSetShort(o, offset, current, (short) (current & mask))); return current; } @@ -3156,7 +3156,7 @@ public final short getAndBitwiseXorShort(Object o, long offset, short mask) { short current; do { current = getShortVolatile(o, offset); - } while (!weakCompareAndSetShort(o, offset, + } while (!compareAndSetShort(o, offset, current, (short) (current ^ mask))); return current; } @@ -3188,7 +3188,7 @@ public final int getAndBitwiseOrInt(Object o, long offset, int mask) { int current; do { current = getIntVolatile(o, offset); - } while (!weakCompareAndSetInt(o, offset, + } while (!compareAndSetInt(o, offset, current, current | mask)); return current; } @@ -3230,7 +3230,7 @@ public final int getAndBitwiseAndInt(Object o, long offset, int mask) { int current; do { current = getIntVolatile(o, offset); - } while (!weakCompareAndSetInt(o, offset, + } while (!compareAndSetInt(o, offset, current, current & mask)); return current; } @@ -3261,7 +3261,7 @@ public final int getAndBitwiseXorInt(Object o, long offset, int mask) { int current; do { current = getIntVolatile(o, offset); - } while (!weakCompareAndSetInt(o, offset, + } while (!compareAndSetInt(o, offset, current, current ^ mask)); return current; } @@ -3293,7 +3293,7 @@ public final long getAndBitwiseOrLong(Object o, long offset, long mask) { long current; do { current = getLongVolatile(o, offset); - } while (!weakCompareAndSetLong(o, offset, + } while (!compareAndSetLong(o, offset, current, current | mask)); return current; } @@ -3324,7 +3324,7 @@ public final long getAndBitwiseAndLong(Object o, long offset, long mask) { long current; do { current = getLongVolatile(o, offset); - } while (!weakCompareAndSetLong(o, offset, + } while (!compareAndSetLong(o, offset, current, current & mask)); return current; } @@ -3355,7 +3355,7 @@ public final long getAndBitwiseXorLong(Object o, long offset, long mask) { long current; do { current = getLongVolatile(o, offset); - } while (!weakCompareAndSetLong(o, offset, + } while (!compareAndSetLong(o, offset, current, current ^ mask)); return current; } From 44eb2f77f32eefd9dcb7c57ec0c146dec5ddebe4 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Fri, 15 Sep 2023 08:34:28 -0400 Subject: [PATCH 47/61] Revert Unsafe; reduce signal stalls --- .../java/util/concurrent/ForkJoinPool.java | 31 ++++++------- .../classes/jdk/internal/misc/Unsafe.java | 46 +++++++++---------- 2 files changed, 38 insertions(+), 39 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index de5cde1c23993..fefa19fc67425 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1300,9 +1300,9 @@ final int queueSize() { */ final void push(ForkJoinTask task, ForkJoinPool pool, boolean internal) { - int s = top++, cap, m; ForkJoinTask[] a; + int s = top++, cap, m, d; ForkJoinTask[] a; if ((a = array) != null && (cap = a.length) > 0) { - if ((m = cap - 1) == s - base) + if ((d = (m = cap - 1) - (s - base)) == 0) growAndPush(task, a, s, internal); else { long pos = slotOffset(m & s); @@ -1312,9 +1312,9 @@ final void push(ForkJoinTask task, ForkJoinPool pool, U.putReference(a, pos, task); // inside lock unlockPhase(); } - if (a[m & (s - 1)] == null && pool != null) - pool.signalWork(); } + if ((d == 0 || a[m & (s - 1)] == null) && pool != null) + pool.signalWork(); } } @@ -1991,13 +1991,13 @@ private void reactivate(WorkQueue w) { */ private boolean isQuiescent(boolean transition) { for (;;) { - long phaseSum = -1L; + long phaseSum = 0L; boolean swept = false; for (int e, prevRunState = 0; ; prevRunState = e) { long c = ctl; if (((e = runState) & STOP) != 0) return true; // terminating - else if ((c & RC_MASK) > 0L) + else if ((c & RC_MASK) != 0L) return false; // at least one active else if (!swept || e != prevRunState || (e & RS_LOCK) != 0) { long sum = c; @@ -2005,16 +2005,15 @@ else if (!swept || e != prevRunState || (e & RS_LOCK) != 0) { int n = (qs == null) ? 0 : qs.length; for (int i = 0; i < n; ++i) { // scan queues WorkQueue q; int p; - if ((q = qs[i]) == null) - sum += (long)(i + 1) << 48; - else if (((p = q.phase) & IDLE) == 0 || - q.top - q.base > 0) { - if ((i & 1) == 0) - signalWork(); // ensure live - return false; + if ((q = qs[i]) != null) { + if (((p = q.phase) & IDLE) == 0 || + q.top - q.base > 0) { + if ((i & 1) == 0) + signalWork(); // ensure live + return false; + } + sum += p & 0xffffffffL; } - else - sum += p; } swept = (phaseSum == (phaseSum = sum)); } @@ -2132,7 +2131,7 @@ a, slotOffset(k), t, null))) { */ private int awaitWork(WorkQueue w, long queuedCtl, int idlePhase) { int p = idlePhase; - boolean quiescent = (queuedCtl & RC_MASK) <= 0L && isQuiescent(true); + boolean quiescent = (queuedCtl & RC_MASK) == 0L && isQuiescent(true); if (w != null && (p = w.phase) == idlePhase && (runState & STOP) == 0) { long deadline = (quiescent ? // timeout for trim keepAlive + System.currentTimeMillis() : 0L); diff --git a/src/java.base/share/classes/jdk/internal/misc/Unsafe.java b/src/java.base/share/classes/jdk/internal/misc/Unsafe.java index 66bf39d4835c8..47e8bf80c50ed 100644 --- a/src/java.base/share/classes/jdk/internal/misc/Unsafe.java +++ b/src/java.base/share/classes/jdk/internal/misc/Unsafe.java @@ -2446,7 +2446,7 @@ public final int getAndAddInt(Object o, long offset, int delta) { int v; do { v = getIntVolatile(o, offset); - } while (!compareAndSetInt(o, offset, v, v + delta)); + } while (!weakCompareAndSetInt(o, offset, v, v + delta)); return v; } @@ -2484,7 +2484,7 @@ public final long getAndAddLong(Object o, long offset, long delta) { long v; do { v = getLongVolatile(o, offset); - } while (!compareAndSetLong(o, offset, v, v + delta)); + } while (!weakCompareAndSetLong(o, offset, v, v + delta)); return v; } @@ -2511,7 +2511,7 @@ public final byte getAndAddByte(Object o, long offset, byte delta) { byte v; do { v = getByteVolatile(o, offset); - } while (!compareAndSetByte(o, offset, v, (byte) (v + delta))); + } while (!weakCompareAndSetByte(o, offset, v, (byte) (v + delta))); return v; } @@ -2538,7 +2538,7 @@ public final short getAndAddShort(Object o, long offset, short delta) { short v; do { v = getShortVolatile(o, offset); - } while (!compareAndSetShort(o, offset, v, (short) (v + delta))); + } while (!weakCompareAndSetShort(o, offset, v, (short) (v + delta))); return v; } @@ -2585,7 +2585,7 @@ public final float getAndAddFloat(Object o, long offset, float delta) { // may result in the loop not terminating. expectedBits = getIntVolatile(o, offset); v = Float.intBitsToFloat(expectedBits); - } while (!compareAndSetInt(o, offset, + } while (!weakCompareAndSetInt(o, offset, expectedBits, Float.floatToRawIntBits(v + delta))); return v; } @@ -2630,7 +2630,7 @@ public final double getAndAddDouble(Object o, long offset, double delta) { // may result in the loop not terminating. expectedBits = getLongVolatile(o, offset); v = Double.longBitsToDouble(expectedBits); - } while (!compareAndSetLong(o, offset, + } while (!weakCompareAndSetLong(o, offset, expectedBits, Double.doubleToRawLongBits(v + delta))); return v; } @@ -2681,7 +2681,7 @@ public final int getAndSetInt(Object o, long offset, int newValue) { int v; do { v = getIntVolatile(o, offset); - } while (!compareAndSetInt(o, offset, v, newValue)); + } while (!weakCompareAndSetInt(o, offset, v, newValue)); return v; } @@ -2719,7 +2719,7 @@ public final long getAndSetLong(Object o, long offset, long newValue) { long v; do { v = getLongVolatile(o, offset); - } while (!compareAndSetLong(o, offset, v, newValue)); + } while (!weakCompareAndSetLong(o, offset, v, newValue)); return v; } @@ -2757,7 +2757,7 @@ public final Object getAndSetReference(Object o, long offset, Object newValue) { Object v; do { v = getReferenceVolatile(o, offset); - } while (!compareAndSetReference(o, offset, v, newValue)); + } while (!weakCompareAndSetReference(o, offset, v, newValue)); return v; } @@ -2784,7 +2784,7 @@ public final byte getAndSetByte(Object o, long offset, byte newValue) { byte v; do { v = getByteVolatile(o, offset); - } while (!compareAndSetByte(o, offset, v, newValue)); + } while (!weakCompareAndSetByte(o, offset, v, newValue)); return v; } @@ -2826,7 +2826,7 @@ public final short getAndSetShort(Object o, long offset, short newValue) { short v; do { v = getShortVolatile(o, offset); - } while (!compareAndSetShort(o, offset, v, newValue)); + } while (!weakCompareAndSetShort(o, offset, v, newValue)); return v; } @@ -2954,7 +2954,7 @@ public final byte getAndBitwiseOrByte(Object o, long offset, byte mask) { byte current; do { current = getByteVolatile(o, offset); - } while (!compareAndSetByte(o, offset, + } while (!weakCompareAndSetByte(o, offset, current, (byte) (current | mask))); return current; } @@ -2985,7 +2985,7 @@ public final byte getAndBitwiseAndByte(Object o, long offset, byte mask) { byte current; do { current = getByteVolatile(o, offset); - } while (!compareAndSetByte(o, offset, + } while (!weakCompareAndSetByte(o, offset, current, (byte) (current & mask))); return current; } @@ -3016,7 +3016,7 @@ public final byte getAndBitwiseXorByte(Object o, long offset, byte mask) { byte current; do { current = getByteVolatile(o, offset); - } while (!compareAndSetByte(o, offset, + } while (!weakCompareAndSetByte(o, offset, current, (byte) (current ^ mask))); return current; } @@ -3094,7 +3094,7 @@ public final short getAndBitwiseOrShort(Object o, long offset, short mask) { short current; do { current = getShortVolatile(o, offset); - } while (!compareAndSetShort(o, offset, + } while (!weakCompareAndSetShort(o, offset, current, (short) (current | mask))); return current; } @@ -3125,7 +3125,7 @@ public final short getAndBitwiseAndShort(Object o, long offset, short mask) { short current; do { current = getShortVolatile(o, offset); - } while (!compareAndSetShort(o, offset, + } while (!weakCompareAndSetShort(o, offset, current, (short) (current & mask))); return current; } @@ -3156,7 +3156,7 @@ public final short getAndBitwiseXorShort(Object o, long offset, short mask) { short current; do { current = getShortVolatile(o, offset); - } while (!compareAndSetShort(o, offset, + } while (!weakCompareAndSetShort(o, offset, current, (short) (current ^ mask))); return current; } @@ -3188,7 +3188,7 @@ public final int getAndBitwiseOrInt(Object o, long offset, int mask) { int current; do { current = getIntVolatile(o, offset); - } while (!compareAndSetInt(o, offset, + } while (!weakCompareAndSetInt(o, offset, current, current | mask)); return current; } @@ -3230,7 +3230,7 @@ public final int getAndBitwiseAndInt(Object o, long offset, int mask) { int current; do { current = getIntVolatile(o, offset); - } while (!compareAndSetInt(o, offset, + } while (!weakCompareAndSetInt(o, offset, current, current & mask)); return current; } @@ -3261,7 +3261,7 @@ public final int getAndBitwiseXorInt(Object o, long offset, int mask) { int current; do { current = getIntVolatile(o, offset); - } while (!compareAndSetInt(o, offset, + } while (!weakCompareAndSetInt(o, offset, current, current ^ mask)); return current; } @@ -3293,7 +3293,7 @@ public final long getAndBitwiseOrLong(Object o, long offset, long mask) { long current; do { current = getLongVolatile(o, offset); - } while (!compareAndSetLong(o, offset, + } while (!weakCompareAndSetLong(o, offset, current, current | mask)); return current; } @@ -3324,7 +3324,7 @@ public final long getAndBitwiseAndLong(Object o, long offset, long mask) { long current; do { current = getLongVolatile(o, offset); - } while (!compareAndSetLong(o, offset, + } while (!weakCompareAndSetLong(o, offset, current, current & mask)); return current; } @@ -3355,7 +3355,7 @@ public final long getAndBitwiseXorLong(Object o, long offset, long mask) { long current; do { current = getLongVolatile(o, offset); - } while (!compareAndSetLong(o, offset, + } while (!weakCompareAndSetLong(o, offset, current, current ^ mask)); return current; } From 4d9c69a1f9dc1eeca263833443a0d222c057cba5 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sat, 16 Sep 2023 08:34:24 -0400 Subject: [PATCH 48/61] Fix header; don't shadow park state --- .../java/util/concurrent/ForkJoinPool.java | 47 +++++++------------ .../java/util/concurrent/ForkJoinTask.java | 3 +- 2 files changed, 19 insertions(+), 31 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index fefa19fc67425..a287aa1a05c8a 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -621,10 +621,7 @@ public class ForkJoinPool extends AbstractExecutorService { * this causes enough memory traffic and CAS contention to prefer * using quieter short spinwaits in awaitWork and elsewhere. * Those in awaitWork are set to small values that only cover - * near-miss scenarios for inactivate/activate races. Because idle - * workers are often not yet blocked (parked), we use the - * WorkQueue parker field to advertise that a waiter actually - * needs unparking upon signal. + * near-miss scenarios for inactivate/activate races. * * Quiescence. Workers scan looking for work, giving up when they * don't find any, without being sure that none are available. @@ -1215,7 +1212,6 @@ public ForkJoinWorkerThread run() { static final class WorkQueue { // fields declared in order of their likely layout on most VMs final ForkJoinWorkerThread owner; // null if shared - volatile Thread parker; // set when parking in awaitWork ForkJoinTask[] array; // the queued tasks; power of 2 size int base; // index of next slot for poll final int config; // mode bits @@ -1948,10 +1944,8 @@ else if ((short)(c >>> RC_SHIFT) >= pc || (v = w) == null) if (v == null) createWorker(); else { - Thread t; v.phase = sp; - if ((t = v.parker) != null) - U.unpark(t); + U.unpark(v.owner); } break; } @@ -1964,7 +1958,7 @@ else if ((short)(c >>> RC_SHIFT) >= pc || (v = w) == null) */ private void reactivate(WorkQueue w) { for (long c = ctl;;) { - WorkQueue[] qs; WorkQueue v; int sp, i; Thread t; + WorkQueue[] qs; WorkQueue v; int sp, i; if ((qs = queues) == null || (sp = (int)c) == 0 || qs.length <= (i = sp & SMASK) || (v = qs[i]) == null || (v != w && w != null && (w.phase & IDLE) == 0)) @@ -1973,8 +1967,7 @@ private void reactivate(WorkQueue w) { c, ((UMASK & (c + RC_UNIT)) | (c & TC_MASK) | (v.stackPred & LMASK))))) { v.phase = sp; - if ((t = v.parker) != null) - U.unpark(t); + U.unpark(v.owner); if (v == w) break; } @@ -1993,27 +1986,26 @@ private boolean isQuiescent(boolean transition) { for (;;) { long phaseSum = 0L; boolean swept = false; - for (int e, prevRunState = 0; ; prevRunState = e) { + recheck: for (int e, prevRunState = 0; ; prevRunState = e) { + U.fullFence(); long c = ctl; if (((e = runState) & STOP) != 0) return true; // terminating - else if ((c & RC_MASK) != 0L) + else if ((c & RC_MASK) > 0L) return false; // at least one active else if (!swept || e != prevRunState || (e & RS_LOCK) != 0) { long sum = c; WorkQueue[] qs = queues; int n = (qs == null) ? 0 : qs.length; for (int i = 0; i < n; ++i) { // scan queues - WorkQueue q; int p; - if ((q = qs[i]) != null) { - if (((p = q.phase) & IDLE) == 0 || - q.top - q.base > 0) { - if ((i & 1) == 0) - signalWork(); // ensure live - return false; - } - sum += p & 0xffffffffL; + int p = e; WorkQueue q; + if ((q = qs[i]) != null && + (((p = q.phase) & IDLE) == 0 || + q.top - q.base > 0)) { + signalWork(); // ensure live + break recheck; // restart } + sum += p & 0xffffffffL; } swept = (phaseSum == (phaseSum = sum)); } @@ -2131,7 +2123,7 @@ a, slotOffset(k), t, null))) { */ private int awaitWork(WorkQueue w, long queuedCtl, int idlePhase) { int p = idlePhase; - boolean quiescent = (queuedCtl & RC_MASK) == 0L && isQuiescent(true); + boolean quiescent = (queuedCtl & RC_MASK) <= 0L && isQuiescent(true); if (w != null && (p = w.phase) == idlePhase && (runState & STOP) == 0) { long deadline = (quiescent ? // timeout for trim keepAlive + System.currentTimeMillis() : 0L); @@ -2142,7 +2134,6 @@ private int awaitWork(WorkQueue w, long queuedCtl, int idlePhase) { if (p == idlePhase) { // emulate LockSupport.park LockSupport.setCurrentBlocker(this); - w.parker = Thread.currentThread(); for (;;) { if ((runState & STOP) != 0 || (p = w.phase) != idlePhase) break; @@ -2163,7 +2154,6 @@ private int awaitWork(WorkQueue w, long queuedCtl, int idlePhase) { deadline += keepAlive; // not at head; restart timer } } - w.parker = null; LockSupport.setCurrentBlocker(null); } } @@ -2217,13 +2207,12 @@ private int tryCompensate(long c) { sp = (int)c, stat = -1; // default retry return if (sp != 0 && active <= pc) { // activate idle worker - WorkQueue[] qs; WorkQueue v; int i; Thread t; + WorkQueue[] qs; WorkQueue v; int i; if ((qs = queues) != null && qs.length > (i = sp & SMASK) && (v = qs[i]) != null && compareAndSetCtl(c, (c & UMASK) | (v.stackPred & LMASK))) { v.phase = sp; - if ((t = v.parker) != null) - U.unpark(t); + U.unpark(v.owner); stat = UNCOMPENSATE; } } @@ -2777,7 +2766,7 @@ private int tryTerminate(boolean now, boolean enable) { int j; WorkQueue q; Thread o; // cancel tasks; interrupt workers if ((q = qs[r & SMASK & (n - 1)]) != null && q.source != DEREGISTERED) { - if ((o = q.owner) != null && !o.isInterrupted()) { + if ((o = q.owner) != null) { try { o.interrupt(); } catch (Throwable ignore) { diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index e8085e54ecc14..8b7e284ed7c31 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -15,8 +15,7 @@ * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor -, Boston, MA 02110-1301 USA. + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any From 907ad02b2b5722fd7b13ea2e725bf7e72b30bc95 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sat, 16 Sep 2023 09:32:14 -0400 Subject: [PATCH 49/61] Accommodate parallelism 0 --- .../share/classes/java/util/concurrent/ForkJoinPool.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index a287aa1a05c8a..0a046133e6743 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -2002,6 +2002,8 @@ else if (!swept || e != prevRunState || (e & RS_LOCK) != 0) { if ((q = qs[i]) != null && (((p = q.phase) & IDLE) == 0 || q.top - q.base > 0)) { + if (parallelism == 0) + return false; signalWork(); // ensure live break recheck; // restart } From a7ff60e90cadac450cef6ef52c947d4b5fae78c5 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Mon, 25 Sep 2023 08:05:16 -0400 Subject: [PATCH 50/61] Fix and improve windowing --- .../java/util/concurrent/ForkJoinPool.java | 325 +++++++++--------- 1 file changed, 168 insertions(+), 157 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 0a046133e6743..0da6984b7cde6 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -576,13 +576,13 @@ public class ForkJoinPool extends AbstractExecutorService { * scanning. These are maintained in the "source" field of * WorkQueues for use in method helpJoin and elsewhere (see * below). We also maintain them as arguments/results of - * top-level polls (argument "window" in method scan) as an - * encoded sliding window of the last two sources, and stop - * signalling when the last two were from the same source. And - * similarly not retry when two CAS failures were from the same - * source. These mechanisms may result in transiently too few - * workers, but once workers poll from a new source, they - * rapidly reactivate others. + * top-level polls (argument "srcs" in method scan, with setup + * in method runWorker) as an encoded sliding window of current + * and previous two sources, and stop signalling when all were + * from the same source. And similarly not retry under multiple + * CAS failures by newly activated workers. These mechanisms + * may result in transiently too few workers, but once workers + * poll from a new source, they rapidly reactivate others. * * * Despite these, signal contention and overhead effects still * occur during ramp-up and ramp-down of small computations. @@ -604,8 +604,7 @@ public class ForkJoinPool extends AbstractExecutorService { * (see method topLevelExec), which also reduces bookkeeping, * cache traffic, and scanning overhead. But it also reduces * fairness, which is partially counteracted by giving up on - * contention, as well as resetting origins to random values upon - * any runState change. + * contention. * * Deactivation. When method scan indicates that no tasks are * found by a worker, it tries to deactivate (in runWorker). Note @@ -621,7 +620,10 @@ public class ForkJoinPool extends AbstractExecutorService { * this causes enough memory traffic and CAS contention to prefer * using quieter short spinwaits in awaitWork and elsewhere. * Those in awaitWork are set to small values that only cover - * near-miss scenarios for inactivate/activate races. + * near-miss scenarios for inactivate/activate races. Because idle + * workers are often not yet blocked (parked), we use the + * WorkQueue parker field to advertise that a waiter actually + * needs unparking upon signal. * * Quiescence. Workers scan looking for work, giving up when they * don't find any, without being sure that none are available. @@ -633,29 +635,24 @@ public class ForkJoinPool extends AbstractExecutorService { * themselves do not guarantee that the pool is in a quiescent * state with respect to methods isQuiescent, shutdown (which * begins termination when quiescent), helpQuiesce, and indirectly - * others including tryCompensate. Method isQuiescent() is used in - * all of these contexts. It provides checks that all workers are - * idle and there are no submissions that they could poll if they - * were not idle, retrying on inconsistent reads of queues and - * using the runState seqLock to retry on queue array updates. (It - * also reports quiescence if the pool is terminating.) A true - * report means only that there was a moment at which quiescence - * held. False negatives are inevitable (for example when queues - * indices lag updates, as described above), which is accommodated - * when (tentatively) idle by scanning for work etc, and then - * re-invoking. This includes cases in which the final unparked - * thread (in awaitWork) uses isQuiescent() to check for tasks - * that could have been added during a race window that would not - * be accompanied by a signal, in which case re-activating itself - * (or any other worker) to rescan. Method helpQuiesce acts - * similarly but cannot rely on ctl counts to determine that all - * workers are inactive because the caller and any others - * executing helpQuiesce are not included in counts. - * - * The quiescence check in awaitWork also serves as a safeguard - * when all other workers gave up prematurely and inactivated (due - * to excessive contention) which will cause this thread to rescan - * and wake up others. + * others including tryCompensate. Method quiescent() is + * used in all of these contexts. It provides checks that all + * workers are idle and there are no submissions that they could + * poll if they were not idle, retrying on inconsistent reads of + * queues and using the runState seqLock to retry on queue array + * updates. (It also reports quiescence if the pool is + * terminating.) A true report means only that there was a moment + * at which quiescence held. False negatives are inevitable (for + * example when queues indices lag updates, as described above), + * which is accommodated when (tentatively) idle by scanning for + * work etc, and then re-invoking. This includes cases in which + * the final unparked thread (in awaitWork) uses quiescent() + * to check for tasks that could have been added during a race + * window that would not be accompanied by a signal, in which case + * re-activating itself (or any other worker) to rescan. Method + * helpQuiesce acts similarly but cannot rely on ctl counts to + * determine that all workers are inactive because the caller and + * any others executing helpQuiesce are not included in counts. * * Termination. A call to shutdownNow invokes tryTerminate to * atomically set a runState mode bit. However, the process of @@ -673,10 +670,6 @@ public class ForkJoinPool extends AbstractExecutorService { * Termination may fail to complete if running tasks repeatedly * ignore both task status and interrupts and/or produce more * tasks after others that could cancel them have exited. - * Parallelizing termination provides multiple attempts to cancel - * tasks and workers when some are only boundedly unresponsive, in - * addition to speeding termination when there are large numbers - * of tasks that need to be cancelled. * * Trimming workers. To release resources after periods of lack of * use, a worker starting to wait when the pool is quiescent will @@ -799,13 +792,13 @@ public class ForkJoinPool extends AbstractExecutorService { * initialization. Since it (or any other created pool) need * never be used, we minimize initial construction overhead and * footprint to the setup of about a dozen fields, although with - * some System property parsing and with security processing that - * takes far longer than the actual construction when - * SecurityManagers are used or properties are set. The common - * pool is distinguished by having a null workerNamePrefix (which - * is an odd convention, but avoids the need to decode status in - * factory classes). It also has PRESET_SIZE config set if - * parallelism was configured by system property. + * some System property parsing and security processing that takes + * far longer than the actual construction when SecurityManagers + * are used or properties are set. The common pool is + * distinguished by having a null workerNamePrefix (which is an + * odd convention, but avoids the need to decode status in factory + * classes). It also has PRESET_SIZE config set if parallelism + * was configured by system property. * * When external threads use the common pool, they can perform * subtask processing (see helpComplete and related methods) upon @@ -971,7 +964,7 @@ public class ForkJoinPool extends AbstractExecutorService { * * New abstract class ForkJoinTask.InterruptibleTask ensures * handling of tasks submitted under the ExecutorService * API are consistent with specs. - * * Method isQuiescent() replaces previous quiescence-related + * * Method quiescent() replaces previous quiescence-related * checks, relying on versioning and sequence locking instead * of ReentrantLock. * * Termination processing now ensures that internal data @@ -1028,7 +1021,6 @@ public class ForkJoinPool extends AbstractExecutorService { static final int SHUTDOWN = 1 << 1; // terminate when quiescent static final int TERMINATED = 1 << 2; // only set if STOP also set static final int RS_LOCK = 1 << 3; // lowest seqlock bit - static final int RS_EPOCH = 1 << 4; // version counter tag // spin/sleep limits for runState locking and elsewhere static final int SPIN_WAITS = 1 << 7; // max calls to onSpinWait @@ -1040,11 +1032,16 @@ public class ForkJoinPool extends AbstractExecutorService { static final int CLEAR_TLS = 1 << 1; // set for Innocuous workers static final int PRESET_SIZE = 1 << 2; // size was set by property + // source history window packing used in scan() and runWorker() + static final long WMASK = ((long)SMASK) << 48; // must be negative + static final long RESCAN = 1L << 63; + static final long RAN = 1L << 62; + static final long INVALID_HISTORY = ((((long)INVALID_ID) << 32) | // no 3rd + (((long)INVALID_ID) << 16)); // no 2nd // others static final int DEREGISTERED = 1 << 31; // worker terminating static final int UNCOMPENSATE = 1 << 16; // tryCompensate return static final int IDLE = 1 << 16; // phase seqlock/version count - static final long RESCAN = 1L << 63; // window retry indicator /* * Bits and masks for ctl and bounds are packed with 4 16 bit subfields: @@ -1212,9 +1209,11 @@ public ForkJoinWorkerThread run() { static final class WorkQueue { // fields declared in order of their likely layout on most VMs final ForkJoinWorkerThread owner; // null if shared + volatile Thread parker; // set when parking in awaitWork ForkJoinTask[] array; // the queued tasks; power of 2 size int base; // index of next slot for poll final int config; // mode bits + volatile int closed; // nonzero if pool terminating or deregistered // fields otherwise causing more unnecessary false-sharing cache misses @jdk.internal.vm.annotation.Contended("w") @@ -1234,6 +1233,7 @@ static final class WorkQueue { private static final long BASE; private static final long TOP; private static final long SOURCE; + private static final long CLOSED; private static final long ARRAY; final void updateBase(int v) { @@ -1296,9 +1296,9 @@ final int queueSize() { */ final void push(ForkJoinTask task, ForkJoinPool pool, boolean internal) { - int s = top++, cap, m, d; ForkJoinTask[] a; + int s = top++, cap, m; ForkJoinTask[] a; if ((a = array) != null && (cap = a.length) > 0) { - if ((d = (m = cap - 1) - (s - base)) == 0) + if ((m = cap - 1) == s - base) growAndPush(task, a, s, internal); else { long pos = slotOffset(m & s); @@ -1308,9 +1308,9 @@ final void push(ForkJoinTask task, ForkJoinPool pool, U.putReference(a, pos, task); // inside lock unlockPhase(); } + if (a[m & (s - 1)] == null && pool != null) + pool.signalWork(); } - if ((d == 0 || a[m & (s - 1)] == null) && pool != null) - pool.signalWork(); } } @@ -1629,6 +1629,28 @@ else if (!(t instanceof CompletableFuture // misc + /** + * Unless already closed, sets closed status, cancels tasks, + * and interrupts if a worker + */ + final void close() { + Thread o; + if (closed == 0 && U.getAndSetInt(this, CLOSED, 1) == 0) { + if ((o = owner) != null) { + try { + o.interrupt(); + } catch (Throwable ignore) { + } + } + for (ForkJoinTask t; (t = poll(null)) != null; ) { + try { + t.cancel(false); + } catch (Throwable ignore) { + } + } + } + } + /** * Returns true if internal and not known to be blocked. */ @@ -1647,6 +1669,7 @@ final boolean isApparentlyUnblocked() { BASE = U.objectFieldOffset(klass, "base"); TOP = U.objectFieldOffset(klass, "top"); SOURCE = U.objectFieldOffset(klass, "source"); + CLOSED = U.objectFieldOffset(klass, "closed"); ARRAY = U.objectFieldOffset(klass, "array"); } } @@ -1739,11 +1762,11 @@ private int getAndBitwiseOrRunState(int v) { // for status bits private boolean casRunState(int c, int v) { return U.compareAndSetInt(this, RUNSTATE, c, v); } - private void unlockRunState() { // increment lock bit + private void unlockRunState() { // increment lock bit U.getAndAddInt(this, RUNSTATE, RS_LOCK); } - private int lockRunState() { // lock and return current state - int s, u; // locked when RS_LOCK set + private int lockRunState() { // lock and return current state + int s, u; // locked when RS_LOCK set if (((s = runState) & RS_LOCK) == 0 && casRunState(s, u = s + RS_LOCK)) return u; else @@ -1875,13 +1898,13 @@ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { int src = 0, phase = 0; boolean replaceable = false; if (wt != null && (w = wt.workQueue) != null) { + w.closed = 1; phase = w.phase; if ((src = w.source) != DEREGISTERED) { // else trimmed on timeout - w.source = DEREGISTERED; - if (phase != 0) { // else failed to start + if (phase != 0) { // else failed to start replaceable = true; if ((phase & IDLE) != 0) - reactivate(w); // pool stopped before released + reactivate(w); // pool stopped before released for (ForkJoinTask t; (t = w.nextLocalTask()) != null; ) { try { t.cancel(false); @@ -1944,8 +1967,10 @@ else if ((short)(c >>> RC_SHIFT) >= pc || (v = w) == null) if (v == null) createWorker(); else { + Thread t; v.phase = sp; - U.unpark(v.owner); + if ((t = v.parker) != null) + U.unpark(t); } break; } @@ -1966,8 +1991,10 @@ private void reactivate(WorkQueue w) { if (c == (c = compareAndExchangeCtl( c, ((UMASK & (c + RC_UNIT)) | (c & TC_MASK) | (v.stackPred & LMASK))))) { + Thread t; v.phase = sp; - U.unpark(v.owner); + if ((t = v.parker) != null) + U.unpark(t); if (v == w) break; } @@ -1977,17 +2004,14 @@ private void reactivate(WorkQueue w) { /** * Internal version of isQuiescent and related functionality. * @return true if terminating or all workers are inactive and - * submission queues are empty and unlocked, else ensure at least - * one worker is active (if any exist) - * @param transition trigger termination or advance generation if - * quiescent + * submission queues are empty and unlocked; if so, setting STOP + * if shutdown is enabled */ - private boolean isQuiescent(boolean transition) { + private boolean quiescent() { for (;;) { long phaseSum = 0L; boolean swept = false; - recheck: for (int e, prevRunState = 0; ; prevRunState = e) { - U.fullFence(); + for (int e, prevRunState = 0; ; prevRunState = e) { long c = ctl; if (((e = runState) & STOP) != 0) return true; // terminating @@ -1998,23 +2022,21 @@ else if (!swept || e != prevRunState || (e & RS_LOCK) != 0) { WorkQueue[] qs = queues; int n = (qs == null) ? 0 : qs.length; for (int i = 0; i < n; ++i) { // scan queues - int p = e; WorkQueue q; - if ((q = qs[i]) != null && - (((p = q.phase) & IDLE) == 0 || - q.top - q.base > 0)) { - if (parallelism == 0) + int p; WorkQueue q; + if ((q = qs[i]) != null) { + if (((p = q.phase) & IDLE) == 0 || + q.top - q.base > 0) { + if ((i & 1) == 0) + signalWork(); // ensure live return false; - signalWork(); // ensure live - break recheck; // restart + } + sum += p & 0xffffffffL; } - sum += p & 0xffffffffL; } swept = (phaseSum == (phaseSum = sum)); } else if (compareAndSetCtl(c, c) && // confirm - casRunState(e, !transition ? e : - (e & SHUTDOWN) != 0 ? e | STOP : - e + RS_EPOCH)) // advance + casRunState(e, (e & SHUTDOWN) != 0 ? e | STOP : e)) return true; else break; // restart @@ -2031,27 +2053,24 @@ else if (compareAndSetCtl(c, c) && // confirm final void runWorker(WorkQueue w) { if (w != null) { int phase = w.phase, r = w.stackPred; // seed from registerWorker - long next = 0L; - for (int e = 0;;) { - int window; // encodes origin and previous non-empty queue - r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift - if (e != (e = runState)) { - if ((e & STOP) != 0) // terminating - break; // else use random origin - window = (INVALID_ID << 16) | (r >>> 16); - } - else - window = (int)next; // continue from last scan - if ((next = scan(w, window, (r << 1) | 1)) >= 0L) { - int idlePhase = phase + IDLE; // try to inactivate - long np = (phase + (IDLE << 1)) & LMASK; - long pc = ctl, qc = ((pc - RC_UNIT) & UMASK) | np; - w.stackPred = (int)pc; // set ctl stack link - w.phase = idlePhase; // try to enqueue - if (pc != compareAndExchangeCtl(pc, qc)) - w.phase = phase; // back out on contention + for (long window = INVALID_HISTORY | (r >>> 16);;) { + r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift + if ((runState & STOP) != 0) // terminating + break; + if ((window == (window = scan(w, window, r))) && + window >= 0L) { // empty scan + long c = ctl; // try to inactivate + int idlePhase = phase + IDLE; + long qc = (((phase + (IDLE << 1)) & LMASK) | + ((c - RC_UNIT) & UMASK)); + w.stackPred = (int)c; // set ctl stack link + w.phase = idlePhase; + if (!compareAndSetCtl(c, qc)) + w.phase = phase; // contended; back out else if ((phase = awaitWork(w, qc, idlePhase)) == idlePhase) - break; // worker exit + break; // worker exit + else // clear history + window = INVALID_HISTORY | (window & SMASK); } } } @@ -2060,55 +2079,58 @@ else if ((phase = awaitWork(w, qc, idlePhase)) == idlePhase) /** * Scans for and if found executes top-level tasks: Tries to poll * each queue starting at initial index with random stride, - * returning scan window and retry indicator. + * returning next scan window and retry indicator. * * @param w caller's WorkQueue - * @param window encodes up to two previously non-empty scanned queues - * @param step random array stride - * @return the next window value to use, with highest bit set for rescan + * @param window up to three queue indices and RAN indicator from last scan + * @param r random seed + * @return the next window value to use, nonnegative if empty */ - private long scan(WorkQueue w, int window, int step) { + private long scan(WorkQueue w, long window, int r) { WorkQueue[] qs = queues; - int n = (qs == null) ? 0 : qs.length; - long next = window & LMASK; // default for empty scan - outer: for (int i = window, l = n; l > 0; --l, i += step) { + int n = (qs == null) ? 0 : qs.length, step = (r << 1) | 1; + int i = (short)window; // origin + boolean running = (window & RAN) != 0L; // true if last scan succeeded + long next = window &= ~WMASK; // default for empty scan + outer: for (int l = n; l > 0; --l, i += step) { int j, cap; WorkQueue q; ForkJoinTask[] a; if ((q = qs[j = i & SMASK & (n - 1)]) != null && (a = q.array) != null && (cap = a.length) > 0) { - for (int b, k;;) { + for (boolean contended = false;;) { + int b, k; ForkJoinTask t = a[k = (b = q.base) & (cap - 1)]; U.loadFence(); // re-read b and t if (q.base == b) { // else inconsistent; retry Object o; // to check identities int nb = b + 1, nk = nb & (cap - 1); if (t == null) { + if (q.array != a) { // resized + next |= RESCAN; + break; + } + if (contended) // reduce incoming contention + break; if (a[k] == null) { // revisit if nonempty - if ((next & RESCAN) == 0L && + if (next >= 0L && (a[nk] != null || q.top - b > 0)) next |= RESCAN; break; } } else if (t == (o = U.compareAndExchangeReference( - a, slotOffset(k), t, null))) { + a, slotOffset(k), t, null))) { q.updateBase(nb); - if (window != (window = (window << 16) | j) && + long nw = ((window << 16) | j) & ~WMASK; + next = nw | (RESCAN | RAN); + if ((window != nw || (short)(nw >> 32) != j) && a[nk] != null) - signalWork(); // propagate at most twice/run - next = (window & LMASK) | RESCAN; + signalWork(); // limit propagation if (w != null) // always true w.topLevelExec(t, q, j); break outer; } - else if (o == null) { // contended; limit rescans - int qw = (window << 16) | j; - if ((next & RESCAN) == 0L && - (q.array != a || // resized - (qw != window && - (a[nk] != null || q.top - q.base > 0)))) - next = (qw & LMASK) | RESCAN; - break; - } + else if (!running) + contended = (o == null); } } } @@ -2125,24 +2147,25 @@ a, slotOffset(k), t, null))) { */ private int awaitWork(WorkQueue w, long queuedCtl, int idlePhase) { int p = idlePhase; - boolean quiescent = (queuedCtl & RC_MASK) <= 0L && isQuiescent(true); - if (w != null && (p = w.phase) == idlePhase && (runState & STOP) == 0) { - long deadline = (quiescent ? // timeout for trim - keepAlive + System.currentTimeMillis() : 0L); - int spins = ((((int)(queuedCtl >>> TC_SHIFT)) & SMASK) << 1) + 3; - do { // spin for approx #accesses to scan+signal - Thread.onSpinWait(); - } while ((p = w.phase) == idlePhase && --spins >= 0); - + int spins = ((((int)(queuedCtl >>> TC_SHIFT)) & SMASK) << 1) | 0x2f; + boolean quiescent = (queuedCtl & RC_MASK) <= 0L && quiescent(); + if (w != null && (runState & STOP) == 0) { + while ((p = w.phase) == idlePhase && --spins > 0) + Thread.onSpinWait(); // spin for approx #accesses to scan+signal if (p == idlePhase) { // emulate LockSupport.park + long deadline = (quiescent ? // timeout for trim + keepAlive + System.currentTimeMillis() : 0L); LockSupport.setCurrentBlocker(this); + w.parker = Thread.currentThread(); for (;;) { - if ((runState & STOP) != 0 || (p = w.phase) != idlePhase) + if ((p = w.phase) != idlePhase) break; U.park(quiescent, deadline); - if ((p = w.phase) != idlePhase || (runState & STOP) != 0) + if ((p = w.phase) != idlePhase) break; Thread.interrupted(); // clear status for next park + if ((runState & STOP) != 0) + break; if (quiescent && // trim on timeout deadline - System.currentTimeMillis() < TIMEOUT_SLOP) { int id = idlePhase & SMASK; @@ -2156,6 +2179,7 @@ private int awaitWork(WorkQueue w, long queuedCtl, int idlePhase) { deadline += keepAlive; // not at head; restart timer } } + w.parker = null; LockSupport.setCurrentBlocker(null); } } @@ -2209,12 +2233,13 @@ private int tryCompensate(long c) { sp = (int)c, stat = -1; // default retry return if (sp != 0 && active <= pc) { // activate idle worker - WorkQueue[] qs; WorkQueue v; int i; + WorkQueue[] qs; WorkQueue v; int i; Thread t; if ((qs = queues) != null && qs.length > (i = sp & SMASK) && (v = qs[i]) != null && compareAndSetCtl(c, (c & UMASK) | (v.stackPred & LMASK))) { v.phase = sp; - U.unpark(v.owner); + if ((t = v.parker) != null) + U.unpark(t); stat = UNCOMPENSATE; } } @@ -2514,7 +2539,7 @@ else if (waits == 0) // same as spinLockRunState except * @return positive if quiescent, negative if interrupted, else 0 */ private int externalHelpQuiesce(long nanos, boolean interruptible) { - if (!isQuiescent(false)) { + if (!quiescent()) { long startTime = System.nanoTime(); long maxSleep = Math.min(nanos >>> 8, MAX_SLEEP); for (int waits = 0;;) { @@ -2525,7 +2550,7 @@ else if ((t = pollScan(false)) != null) { waits = 0; t.doExec(); } - else if (isQuiescent(false)) + else if (quiescent()) break; else if (System.nanoTime() - startTime > nanos) return 0; @@ -2538,6 +2563,7 @@ else if (waits == 0) } } } + tryTerminate(false, false); return 1; } @@ -2756,31 +2782,16 @@ private int tryTerminate(boolean now, boolean enable) { else { if ((isShutdown = (e & SHUTDOWN)) == 0 && enable) getAndBitwiseOrRunState(isShutdown = SHUTDOWN); - if (isShutdown != 0 && isQuiescent(true)) + if (isShutdown != 0 && quiescent()) e = runState; } } - if ((e & (STOP | TERMINATED)) == STOP) { - int r = (int)Thread.currentThread().threadId(); // stagger traversals - WorkQueue[] qs = queues; + if ((e & (STOP | TERMINATED)) == STOP) { // help terminate + WorkQueue[] qs = queues; WorkQueue q; int n = (qs == null) ? 0 : qs.length; - for (int l = n; l > 0; --l, ++r) { - int j; WorkQueue q; Thread o; // cancel tasks; interrupt workers - if ((q = qs[r & SMASK & (n - 1)]) != null && - q.source != DEREGISTERED) { - if ((o = q.owner) != null) { - try { - o.interrupt(); - } catch (Throwable ignore) { - } - } - for (ForkJoinTask t; (t = q.poll(null)) != null; ) { - try { - t.cancel(false); - } catch (Throwable ignore) { - } - } - } + for (int i = 0; i < n; ++i) { + if ((q = qs[i]) != null) + q.close(); } if (((e = runState) & TERMINATED) == 0 && ctl == 0L) { if ((getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0) { @@ -3464,7 +3475,7 @@ public int getActiveThreadCount() { * @return {@code true} if all threads are currently idle */ public boolean isQuiescent() { - return isQuiescent(false); + return quiescent(); } /** From bc9cc049ee820ae6d701557f10b7bff7a94b21a6 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Mon, 25 Sep 2023 10:59:37 -0400 Subject: [PATCH 51/61] Ensure publishabliity on resize --- .../share/classes/java/util/concurrent/ForkJoinPool.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 0da6984b7cde6..1223d00825009 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1296,9 +1296,9 @@ final int queueSize() { */ final void push(ForkJoinTask task, ForkJoinPool pool, boolean internal) { - int s = top++, cap, m; ForkJoinTask[] a; + int s = top++, cap, m, room; ForkJoinTask[] a; if ((a = array) != null && (cap = a.length) > 0) { - if ((m = cap - 1) == s - base) + if ((room = (m = cap - 1) - (s - base)) == 0) growAndPush(task, a, s, internal); else { long pos = slotOffset(m & s); @@ -1308,9 +1308,9 @@ final void push(ForkJoinTask task, ForkJoinPool pool, U.putReference(a, pos, task); // inside lock unlockPhase(); } - if (a[m & (s - 1)] == null && pool != null) - pool.signalWork(); } + if ((room == 0 || a[m & (s - 1)] == null) && pool != null) + pool.signalWork(); } } @@ -1321,6 +1321,7 @@ final void push(ForkJoinTask task, ForkJoinPool pool, */ private void growAndPush(ForkJoinTask task, ForkJoinTask[] a, int s, boolean internal) { + U.storeFence(); // ensure task publishable int cap; // rapidly grow until large if (a != null && (cap = a.length) > 0) { int newCap = (cap < 1 << 24) ? cap << 2 : cap << 1; From 038d5874c3951a7b02b0b58145c10a0f9889488a Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Tue, 26 Sep 2023 18:50:06 -0400 Subject: [PATCH 52/61] more conservative resize checks --- .../java/util/concurrent/ForkJoinPool.java | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 1223d00825009..7ac4a5a17fb67 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1246,7 +1246,7 @@ final void forgetSource() { U.putIntOpaque(this, SOURCE, 0); } final void updateArray(ForkJoinTask[] a) { - U.putReferenceVolatile(this, ARRAY, a); + U.getAndSetReference(this, ARRAY, a); } final void unlockPhase() { U.getAndAddInt(this, PHASE, IDLE); @@ -2058,7 +2058,7 @@ final void runWorker(WorkQueue w) { r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift if ((runState & STOP) != 0) // terminating break; - if ((window == (window = scan(w, window, r))) && + if (window == (window = scan(w, window, r)) && window >= 0L) { // empty scan long c = ctl; // try to inactivate int idlePhase = phase + IDLE; @@ -2105,15 +2105,11 @@ private long scan(WorkQueue w, long window, int r) { Object o; // to check identities int nb = b + 1, nk = nb & (cap - 1); if (t == null) { - if (q.array != a) { // resized - next |= RESCAN; - break; - } - if (contended) // reduce incoming contention - break; if (a[k] == null) { // revisit if nonempty - if (next >= 0L && - (a[nk] != null || q.top - b > 0)) + if (next >= 0L && // and resized or uncontended + (q.array != a || + (!contended && + (a[nk] != null || q.top - b > 0)))) next |= RESCAN; break; } @@ -2159,14 +2155,12 @@ private int awaitWork(WorkQueue w, long queuedCtl, int idlePhase) { LockSupport.setCurrentBlocker(this); w.parker = Thread.currentThread(); for (;;) { - if ((p = w.phase) != idlePhase) + if ((runState & STOP) != 0 || (p = w.phase) != idlePhase) break; U.park(quiescent, deadline); - if ((p = w.phase) != idlePhase) + if ((p = w.phase) != idlePhase || (runState & STOP) != 0) break; Thread.interrupted(); // clear status for next park - if ((runState & STOP) != 0) - break; if (quiescent && // trim on timeout deadline - System.currentTimeMillis() < TIMEOUT_SLOP) { int id = idlePhase & SMASK; From ed3b749391c30116d8ce475fce42c14c7458f48e Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Wed, 27 Sep 2023 08:46:45 -0400 Subject: [PATCH 53/61] Possibly re-interrupt when stopping --- .../java/util/concurrent/ForkJoinPool.java | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 7ac4a5a17fb67..629188088a342 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1631,18 +1631,21 @@ else if (!(t instanceof CompletableFuture // misc /** - * Unless already closed, sets closed status, cancels tasks, - * and interrupts if a worker + * Sets closed status, interrupts if a worker, and unless + * already closed, cancels tasks, */ final void close() { - Thread o; - if (closed == 0 && U.getAndSetInt(this, CLOSED, 1) == 0) { - if ((o = owner) != null) { - try { - o.interrupt(); - } catch (Throwable ignore) { - } + Thread o = owner; + int wasClosed = closed; + if (wasClosed == 0) + wasClosed = U.getAndSetInt(this, CLOSED, 1); + if (o != null && (wasClosed == 0 || !o.isInterrupted())) { + try { + o.interrupt(); + } catch (Throwable ignore) { } + } + if (wasClosed == 0) { for (ForkJoinTask t; (t = poll(null)) != null; ) { try { t.cancel(false); @@ -1979,8 +1982,9 @@ else if ((short)(c >>> RC_SHIFT) >= pc || (v = w) == null) } /** - * Reactivates the given worker, and possibly others if not top of - * ctl stack. Needed during shutdown to ensure release on termination. + * Reactivates the given worker, and possibly interrupts others if + * not top of ctl stack. Called only during shutdown to ensure release + * on termination. */ private void reactivate(WorkQueue w) { for (long c = ctl;;) { @@ -1994,8 +1998,12 @@ private void reactivate(WorkQueue w) { (v.stackPred & LMASK))))) { Thread t; v.phase = sp; - if ((t = v.parker) != null) - U.unpark(t); + if ((t = v.parker) != null) { + try { + t.interrupt(); + } catch (Throwable ignore) { + } + } if (v == w) break; } @@ -2058,8 +2066,7 @@ final void runWorker(WorkQueue w) { r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift if ((runState & STOP) != 0) // terminating break; - if (window == (window = scan(w, window, r)) && - window >= 0L) { // empty scan + if ((window = scan(w, window, r)) >= 0L) { // empty scan long c = ctl; // try to inactivate int idlePhase = phase + IDLE; long qc = (((phase + (IDLE << 1)) & LMASK) | @@ -2773,7 +2780,7 @@ private int tryTerminate(boolean now, boolean enable) { int e, isShutdown; if (((e = runState) & STOP) == 0) { if (now) - getAndBitwiseOrRunState(e = STOP | SHUTDOWN); + runState = e = (lockRunState() + RS_LOCK) | STOP | SHUTDOWN; else { if ((isShutdown = (e & SHUTDOWN)) == 0 && enable) getAndBitwiseOrRunState(isShutdown = SHUTDOWN); @@ -2782,10 +2789,11 @@ private int tryTerminate(boolean now, boolean enable) { } } if ((e & (STOP | TERMINATED)) == STOP) { // help terminate + int r = (int)Thread.currentThread().threadId(); // stagger traversals WorkQueue[] qs = queues; WorkQueue q; int n = (qs == null) ? 0 : qs.length; - for (int i = 0; i < n; ++i) { - if ((q = qs[i]) != null) + for (int l = n; l > 0; --l, ++r) { + if ((q = qs[r & SMASK & (n - 1)]) != null) q.close(); } if (((e = runState) & TERMINATED) == 0 && ctl == 0L) { From 85c26d2ca3fa84a949324fdb99d6d6e70a11eba9 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Thu, 28 Sep 2023 15:42:37 -0400 Subject: [PATCH 54/61] Streamline push; add redundant interrupts --- .../java/util/concurrent/ForkJoinPool.java | 119 ++++++++---------- .../java/util/concurrent/ForkJoinTask.java | 27 ++-- 2 files changed, 64 insertions(+), 82 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 629188088a342..0e856847c345b 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1292,63 +1292,47 @@ final int queueSize() { * @param task the task. Caller must ensure non-null. * @param pool the pool to signal if was previously empty, else null * @param internal if caller owns this queue - * @throws RejectedExecutionException if array cannot be resized + * @throws RejectedExecutionException if array could not be resized */ final void push(ForkJoinTask task, ForkJoinPool pool, boolean internal) { int s = top++, cap, m, room; ForkJoinTask[] a; - if ((a = array) != null && (cap = a.length) > 0) { - if ((room = (m = cap - 1) - (s - base)) == 0) - growAndPush(task, a, s, internal); - else { - long pos = slotOffset(m & s); - if (internal) - U.getAndSetReference(a, pos, task); // fully fenced - else { - U.putReference(a, pos, task); // inside lock - unlockPhase(); - } - } - if ((room == 0 || a[m & (s - 1)] == null) && pool != null) - pool.signalWork(); + if ((a = array) == null || (cap = a.length) <= 0 || + (room = (m = cap - 1) - (s - base)) < 0) { + top = s; // revert on failure + if (!internal) + unlockPhase(); + throw new RejectedExecutionException("Queue capacity exceeded"); } - } - - /** - * Grows the task array if possible and adds the task. - * @param a the current task array - * @param s the old top value - */ - private void growAndPush(ForkJoinTask task, ForkJoinTask[] a, - int s, boolean internal) { - U.storeFence(); // ensure task publishable - int cap; // rapidly grow until large - if (a != null && (cap = a.length) > 0) { - int newCap = (cap < 1 << 24) ? cap << 2 : cap << 1; - int newMask = newCap - 1, k = s, b = k - cap, m = cap - 1; - if (newCap > 0) { - ForkJoinTask[] newArray = null; + long pos = slotOffset(m & s); + if (!internal) + U.putReference(a, pos, task); // inside lock + else + U.getAndSetReference(a, pos, task); // fully fenced + if (room == 0) { // resize for next time + int newCap; // rapidly grow until large + ForkJoinTask[] newArray = null; + if ((newCap = (cap < 1 << 24) ? cap << 2 : cap << 1) > 0) { try { newArray = new ForkJoinTask[newCap]; } catch (OutOfMemoryError ex) { } - if (newArray != null) { - do { // poll old, push to new; exit if lose to pollers - newArray[k & newMask] = task; - } while (--k != b && - (task = (ForkJoinTask)U.getAndSetReference( - a, slotOffset(k & m), null)) != null); - updateArray(newArray); - if (!internal) - unlockPhase(); - return; - } } + if (newArray != null) { // poll old, push to new + for (int nm = newCap - 1, k = s, j = cap; j > 0; --j, --k) { + if ((newArray[k & nm] = + (ForkJoinTask)U.getAndSetReference( + a, slotOffset(k & m), null)) == null) + break; // lost to pollers + } + updateArray(newArray); // fully fenced + } // else will throw on next push unless tasks taken } - top = s; // revert on failure if (!internal) unlockPhase(); - throw new RejectedExecutionException("Queue capacity exceeded"); + if ((room == 0 || room == m || a[m & (s - 1)] == null) && + pool != null) + pool.signalWork(); } /** @@ -1631,8 +1615,7 @@ else if (!(t instanceof CompletableFuture // misc /** - * Sets closed status, interrupts if a worker, and unless - * already closed, cancels tasks, + * Sets closed status, interrupts if a worker, and cancels tasks, */ final void close() { Thread o = owner; @@ -1645,12 +1628,10 @@ final void close() { } catch (Throwable ignore) { } } - if (wasClosed == 0) { - for (ForkJoinTask t; (t = poll(null)) != null; ) { - try { - t.cancel(false); - } catch (Throwable ignore) { - } + for (ForkJoinTask t; (t = poll(null)) != null; ) { + try { + t.cancel(false); + } catch (Throwable ignore) { } } } @@ -1938,12 +1919,10 @@ else if ((int)c == 0) // was dropped on timeout } unlockRunState(); } - if ((runState & STOP) == 0) { - if (replaceable) - signalWork(); // may replace unless trimmed or uninitialized - if (ex != null) - ForkJoinTask.rethrow(ex); - } + if ((runState & STOP) == 0 && replaceable) + signalWork(); // may replace unless trimmed or uninitialized + if (ex != null) + ForkJoinTask.rethrow(ex); } /** @@ -2017,7 +1996,7 @@ private void reactivate(WorkQueue w) { * if shutdown is enabled */ private boolean quiescent() { - for (;;) { + outer: for (;;) { long phaseSum = 0L; boolean swept = false; for (int e, prevRunState = 0; ; prevRunState = e) { @@ -2035,9 +2014,11 @@ else if (!swept || e != prevRunState || (e & RS_LOCK) != 0) { if ((q = qs[i]) != null) { if (((p = q.phase) & IDLE) == 0 || q.top - q.base > 0) { - if ((i & 1) == 0) + if ((i & 1) == 0 && compareAndSetCtl(c, c)) signalWork(); // ensure live - return false; + if (parallelism == 0) + return false; + continue outer; } sum += p & 0xffffffffL; } @@ -2112,12 +2093,10 @@ private long scan(WorkQueue w, long window, int r) { Object o; // to check identities int nb = b + 1, nk = nb & (cap - 1); if (t == null) { - if (a[k] == null) { // revisit if nonempty - if (next >= 0L && // and resized or uncontended - (q.array != a || - (!contended && - (a[nk] != null || q.top - b > 0)))) - next |= RESCAN; + if (a[k] == null) { + if (!contended && next >= 0L && + (a[nk] != null || q.top - b > 0)) + next |= RESCAN; // revisit break; } } @@ -2642,6 +2621,7 @@ else if ((runState & SHUTDOWN) != 0) { else return q; } + tryTerminate(false, false); throw new RejectedExecutionException(); } @@ -2784,8 +2764,9 @@ private int tryTerminate(boolean now, boolean enable) { else { if ((isShutdown = (e & SHUTDOWN)) == 0 && enable) getAndBitwiseOrRunState(isShutdown = SHUTDOWN); - if (isShutdown != 0 && quiescent()) - e = runState; + if (isShutdown != 0) + quiescent(); // may trigger STOP + e = runState; } } if ((e & (STOP | TERMINATED)) == STOP) { // help terminate diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index 8b7e284ed7c31..bd90dca37d458 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -1629,21 +1629,22 @@ static abstract class InterruptibleTask extends ForkJoinTask transient volatile Thread runner; abstract T compute() throws Exception; public final boolean exec() { - Thread t = Thread.currentThread(); - if ((t instanceof ForkJoinWorkerThread) && - ForkJoinPool.poolIsStopping(((ForkJoinWorkerThread)t).pool)) - cancel(false); - else { + Thread t = runner = Thread.currentThread(); + try { Thread.interrupted(); - runner = t; - try { - if (status >= 0) - setRawResult(compute()); - } catch (Exception ex) { - trySetException(ex); - } finally { - runner = null; + if ((t instanceof ForkJoinWorkerThread) && + ForkJoinPool.poolIsStopping(((ForkJoinWorkerThread)t).pool)) + cancel(true); + else { + try { + if (status >= 0) + setRawResult(compute()); + } catch (Exception ex) { + trySetException(ex); + } } + } finally { + runner = null; } return true; } From 9b714224ed2c153dc11f6db874e10402e9cfd64e Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Thu, 28 Sep 2023 18:39:08 -0400 Subject: [PATCH 55/61] Undo stray edit --- .../share/classes/java/util/concurrent/ForkJoinTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java index bd90dca37d458..8706359cda28f 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinTask.java @@ -1629,9 +1629,9 @@ static abstract class InterruptibleTask extends ForkJoinTask transient volatile Thread runner; abstract T compute() throws Exception; public final boolean exec() { + Thread.interrupted(); Thread t = runner = Thread.currentThread(); try { - Thread.interrupted(); if ((t instanceof ForkJoinWorkerThread) && ForkJoinPool.poolIsStopping(((ForkJoinWorkerThread)t).pool)) cancel(true); From ffc78395112bdb801eda7792a463ab05e832b602 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sun, 1 Oct 2023 09:29:14 -0400 Subject: [PATCH 56/61] refactor termination --- .../java/util/concurrent/ForkJoinPool.java | 139 +++++++++++------- 1 file changed, 84 insertions(+), 55 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 0e856847c345b..754afd95e23ef 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -663,13 +663,13 @@ public class ForkJoinPool extends AbstractExecutorService { * termination only when accessing pool state. This may take a * while but suffices for structured computational tasks. But not * necessarily for others. Class InterruptibleTask (see below) - * further arranges runState checks before executing task bodies, and - * ensures interrupts while terminating. Even so, there are no + * further arranges runState checks before executing task bodies, + * and ensures interrupts while terminating. Even so, there are no * guarantees after an abrupt shutdown that remaining tasks * complete normally or exceptionally or are cancelled. - * Termination may fail to complete if running tasks repeatedly - * ignore both task status and interrupts and/or produce more - * tasks after others that could cancel them have exited. + * Termination may fail to complete if running tasks ignore both + * task status and interrupts and/or produce more tasks after + * others that could cancel them have exited. * * Trimming workers. To release resources after periods of lack of * use, a worker starting to wait when the pool is quiescent will @@ -1213,7 +1213,6 @@ static final class WorkQueue { ForkJoinTask[] array; // the queued tasks; power of 2 size int base; // index of next slot for poll final int config; // mode bits - volatile int closed; // nonzero if pool terminating or deregistered // fields otherwise causing more unnecessary false-sharing cache misses @jdk.internal.vm.annotation.Contended("w") @@ -1233,7 +1232,6 @@ static final class WorkQueue { private static final long BASE; private static final long TOP; private static final long SOURCE; - private static final long CLOSED; private static final long ARRAY; final void updateBase(int v) { @@ -1296,14 +1294,14 @@ final int queueSize() { */ final void push(ForkJoinTask task, ForkJoinPool pool, boolean internal) { - int s = top++, cap, m, room; ForkJoinTask[] a; + int s = top, b = base, cap, m, room; ForkJoinTask[] a; if ((a = array) == null || (cap = a.length) <= 0 || - (room = (m = cap - 1) - (s - base)) < 0) { - top = s; // revert on failure + (room = (m = cap - 1) - (s - b)) < 0) { // could not resize if (!internal) unlockPhase(); throw new RejectedExecutionException("Queue capacity exceeded"); } + top = s + 1; long pos = slotOffset(m & s); if (!internal) U.putReference(a, pos, task); // inside lock @@ -1311,22 +1309,23 @@ final void push(ForkJoinTask task, ForkJoinPool pool, U.getAndSetReference(a, pos, task); // fully fenced if (room == 0) { // resize for next time int newCap; // rapidly grow until large - ForkJoinTask[] newArray = null; if ((newCap = (cap < 1 << 24) ? cap << 2 : cap << 1) > 0) { + ForkJoinTask[] newArray = null; try { newArray = new ForkJoinTask[newCap]; } catch (OutOfMemoryError ex) { } - } - if (newArray != null) { // poll old, push to new - for (int nm = newCap - 1, k = s, j = cap; j > 0; --j, --k) { - if ((newArray[k & nm] = - (ForkJoinTask)U.getAndSetReference( - a, slotOffset(k & m), null)) == null) - break; // lost to pollers + if (newArray != null) { // else throw on next push + int newMask = newCap - 1; // poll old, push to new + for (int k = s, j = cap; j > 0; --j, --k) { + if ((newArray[k & newMask] = + (ForkJoinTask)U.getAndSetReference( + a, slotOffset(k & m), null)) == null) + break; // lost to pollers + } + updateArray(newArray); // fully fenced } - updateArray(newArray); // fully fenced - } // else will throw on next push unless tasks taken + } } if (!internal) unlockPhase(); @@ -1615,23 +1614,22 @@ else if (!(t instanceof CompletableFuture // misc /** - * Sets closed status, interrupts if a worker, and cancels tasks, + * Unless already deregistered, interrupts if a worker, and cancels tasks */ - final void close() { + final void onStop() { Thread o = owner; - int wasClosed = closed; - if (wasClosed == 0) - wasClosed = U.getAndSetInt(this, CLOSED, 1); - if (o != null && (wasClosed == 0 || !o.isInterrupted())) { - try { - o.interrupt(); - } catch (Throwable ignore) { + if (source != DEREGISTERED) { + if (o != null) { + try { + o.interrupt(); + } catch (Throwable ignore) { + } } - } - for (ForkJoinTask t; (t = poll(null)) != null; ) { - try { - t.cancel(false); - } catch (Throwable ignore) { + for (ForkJoinTask t; (t = poll(null)) != null; ) { + try { + t.cancel(false); + } catch (Throwable ignore) { + } } } } @@ -1654,7 +1652,6 @@ final boolean isApparentlyUnblocked() { BASE = U.objectFieldOffset(klass, "base"); TOP = U.objectFieldOffset(klass, "top"); SOURCE = U.objectFieldOffset(klass, "source"); - CLOSED = U.objectFieldOffset(klass, "closed"); ARRAY = U.objectFieldOffset(klass, "array"); } } @@ -1883,9 +1880,9 @@ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { int src = 0, phase = 0; boolean replaceable = false; if (wt != null && (w = wt.workQueue) != null) { - w.closed = 1; phase = w.phase; if ((src = w.source) != DEREGISTERED) { // else trimmed on timeout + w.source = DEREGISTERED; if (phase != 0) { // else failed to start replaceable = true; if ((phase & IDLE) != 0) @@ -2007,27 +2004,29 @@ else if ((c & RC_MASK) > 0L) return false; // at least one active else if (!swept || e != prevRunState || (e & RS_LOCK) != 0) { long sum = c; - WorkQueue[] qs = queues; + WorkQueue[] qs = queues; WorkQueue q; int n = (qs == null) ? 0 : qs.length; for (int i = 0; i < n; ++i) { // scan queues - int p; WorkQueue q; if ((q = qs[i]) != null) { - if (((p = q.phase) & IDLE) == 0 || - q.top - q.base > 0) { - if ((i & 1) == 0 && compareAndSetCtl(c, c)) - signalWork(); // ensure live + int p = q.phase, s = q.top, b = q.base; + sum += (p & 0xffffffffL) | ((long)b << 32); + if ((p & IDLE) == 0 || s - b > 0) { if (parallelism == 0) return false; + if ((i & 1) == 0 && compareAndSetCtl(c, c)) + signalWork(); // ensure live continue outer; } - sum += p & 0xffffffffL; } } swept = (phaseSum == (phaseSum = sum)); } else if (compareAndSetCtl(c, c) && // confirm - casRunState(e, (e & SHUTDOWN) != 0 ? e | STOP : e)) + casRunState(e, (e & SHUTDOWN) != 0 ? e | STOP : e)) { + if ((e & SHUTDOWN) != 0) // enable termination + interruptAll(); return true; + } else break; // restart } @@ -2112,8 +2111,9 @@ a, slotOffset(k), t, null))) { w.topLevelExec(t, q, j); break outer; } - else if (!running) - contended = (o == null); + else // limit retries on contention + contended = (o == null && !running && + q.array == a && a[nk] != null); } } } @@ -2544,7 +2544,6 @@ else if (waits == 0) } } } - tryTerminate(false, false); return 1; } @@ -2757,25 +2756,35 @@ static int getSurplusQueuedTaskCount() { * @return runState on exit */ private int tryTerminate(boolean now, boolean enable) { - int e, isShutdown; + int e, s, isShutdown; if (((e = runState) & STOP) == 0) { - if (now) - runState = e = (lockRunState() + RS_LOCK) | STOP | SHUTDOWN; + if (now) { + runState = ((s = lockRunState()) + RS_LOCK) | STOP | SHUTDOWN; + if ((s & STOP) == 0) + interruptAll(); + } else { if ((isShutdown = (e & SHUTDOWN)) == 0 && enable) getAndBitwiseOrRunState(isShutdown = SHUTDOWN); if (isShutdown != 0) quiescent(); // may trigger STOP - e = runState; } + e = runState; } - if ((e & (STOP | TERMINATED)) == STOP) { // help terminate + if ((e & (STOP | TERMINATED)) == STOP) { // help cancel tasks int r = (int)Thread.currentThread().threadId(); // stagger traversals - WorkQueue[] qs = queues; WorkQueue q; + WorkQueue[] qs = queues; int n = (qs == null) ? 0 : qs.length; for (int l = n; l > 0; --l, ++r) { - if ((q = qs[r & SMASK & (n - 1)]) != null) - q.close(); + int j; WorkQueue q; ForkJoinTask t; + while ((q = qs[r & SMASK & (n - 1)]) != null && + q.source != DEREGISTERED && + (t = q.poll(null)) != null) { + try { + t.cancel(false); + } catch (Throwable ignore) { + } + } } if (((e = runState) & TERMINATED) == 0 && ctl == 0L) { if ((getAndBitwiseOrRunState(TERMINATED) & TERMINATED) == 0) { @@ -2791,6 +2800,26 @@ private int tryTerminate(boolean now, boolean enable) { return e; } + /** + * Interrupts all workers + */ + private void interruptAll() { + Thread current = Thread.currentThread(); + WorkQueue[] qs = queues; + int n = (qs == null) ? 0 : qs.length; + for (int i = 1; i < n; i += 2) { + WorkQueue q; Thread o; + if ((q = qs[i]) != null && (o = q.owner) != null && o != current && + q.source != DEREGISTERED) { + try { + o.interrupt(); + } catch (Throwable ignore) { + } + } + } + } + + /** * Returns termination signal, constructing if necessary */ From 078655edc43f6bc1fa1a3cd0764ac9eddc31596e Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Mon, 2 Oct 2023 10:37:49 -0400 Subject: [PATCH 57/61] cleanup --- .../java/util/concurrent/ForkJoinPool.java | 65 ++++++++----------- 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 754afd95e23ef..aaa5011a581b0 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1613,27 +1613,6 @@ else if (!(t instanceof CompletableFuture // misc - /** - * Unless already deregistered, interrupts if a worker, and cancels tasks - */ - final void onStop() { - Thread o = owner; - if (source != DEREGISTERED) { - if (o != null) { - try { - o.interrupt(); - } catch (Throwable ignore) { - } - } - for (ForkJoinTask t; (t = poll(null)) != null; ) { - try { - t.cancel(false); - } catch (Throwable ignore) { - } - } - } - } - /** * Returns true if internal and not known to be blocked. */ @@ -1887,12 +1866,6 @@ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { replaceable = true; if ((phase & IDLE) != 0) reactivate(w); // pool stopped before released - for (ForkJoinTask t; (t = w.nextLocalTask()) != null; ) { - try { - t.cancel(false); - } catch (Throwable ignore) { - } - } } } } @@ -1904,7 +1877,14 @@ final void deregisterWorker(ForkJoinWorkerThread wt, Throwable ex) { (LMASK & c))))); else if ((int)c == 0) // was dropped on timeout replaceable = false; - + if (w != null) { // cancel remaining tasks + for (ForkJoinTask t; (t = w.nextLocalTask()) != null; ) { + try { + t.cancel(false); + } catch (Throwable ignore) { + } + } + } if ((tryTerminate(false, false) & STOP) == 0 && w != null) { WorkQueue[] qs; int n, i; // remove index unless terminating long ns = w.nsteals & 0xffffffffL; @@ -2046,7 +2026,8 @@ final void runWorker(WorkQueue w) { r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift if ((runState & STOP) != 0) // terminating break; - if ((window = scan(w, window, r)) >= 0L) { // empty scan + if (window == (window = scan(w, window, r)) && + window >= 0L) { // empty scan long c = ctl; // try to inactivate int idlePhase = phase + IDLE; long qc = (((phase + (IDLE << 1)) & LMASK) | @@ -2756,29 +2737,30 @@ static int getSurplusQueuedTaskCount() { * @return runState on exit */ private int tryTerminate(boolean now, boolean enable) { - int e, s, isShutdown; - if (((e = runState) & STOP) == 0) { + int e = runState; + if ((e & STOP) == 0) { if (now) { - runState = ((s = lockRunState()) + RS_LOCK) | STOP | SHUTDOWN; + int s = lockRunState(); + runState = e = (s + RS_LOCK) | STOP | SHUTDOWN; if ((s & STOP) == 0) interruptAll(); } else { - if ((isShutdown = (e & SHUTDOWN)) == 0 && enable) + int isShutdown = (e & SHUTDOWN); + if (isShutdown == 0 && enable) getAndBitwiseOrRunState(isShutdown = SHUTDOWN); if (isShutdown != 0) quiescent(); // may trigger STOP + e = runState; } - e = runState; } if ((e & (STOP | TERMINATED)) == STOP) { // help cancel tasks int r = (int)Thread.currentThread().threadId(); // stagger traversals WorkQueue[] qs = queues; int n = (qs == null) ? 0 : qs.length; for (int l = n; l > 0; --l, ++r) { - int j; WorkQueue q; ForkJoinTask t; - while ((q = qs[r & SMASK & (n - 1)]) != null && - q.source != DEREGISTERED && + int j = r & SMASK & (n - 1); WorkQueue q; ForkJoinTask t; + while ((q = qs[j]) != null && q.source != DEREGISTERED && (t = q.poll(null)) != null) { try { t.cancel(false); @@ -3357,6 +3339,15 @@ public List> invokeAll(Collection> tasks) throws InterruptedException { return invokeAll(tasks, 0L); } + // for jdk version < 22, replace with + // /** + // * @throws NullPointerException {@inheritDoc} + // * @throws RejectedExecutionException {@inheritDoc} + // */ + // @Override + // public List> invokeAll(Collection> tasks) { + // return invokeAllUninterruptibly(tasks); + // } @Override public List> invokeAll(Collection> tasks, From eb0f8dc6940d384fce9f0668978f18f0dde31ae9 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Sat, 7 Oct 2023 09:39:22 -0400 Subject: [PATCH 58/61] better window encoding --- .../java/util/concurrent/ForkJoinPool.java | 62 +++++++++---------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index aaa5011a581b0..8503140e8805d 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -576,13 +576,15 @@ public class ForkJoinPool extends AbstractExecutorService { * scanning. These are maintained in the "source" field of * WorkQueues for use in method helpJoin and elsewhere (see * below). We also maintain them as arguments/results of - * top-level polls (argument "srcs" in method scan, with setup + * top-level polls (argument "window" in method scan, with setup * in method runWorker) as an encoded sliding window of current - * and previous two sources, and stop signalling when all were - * from the same source. And similarly not retry under multiple - * CAS failures by newly activated workers. These mechanisms - * may result in transiently too few workers, but once workers - * poll from a new source, they rapidly reactivate others. + * and previous two sources (or INVALID_ID if none), and stop + * signalling when all were from the same source. Also, retries + * are suppressed on CAS failures by newly activated workers, + * which serves as a form of admission control. These + * mechanisms may result in transiently too few workers, but + * once workers poll from a new source, they rapidly reactivate + * others. * * * Despite these, signal contention and overhead effects still * occur during ramp-up and ramp-down of small computations. @@ -1006,7 +1008,6 @@ public class ForkJoinPool extends AbstractExecutorService { static final int INITIAL_QUEUE_CAPACITY = 1 << 6; // conversions among short, int, long - static final int SWIDTH = 16; // width of short static final int SMASK = 0xffff; // (unsigned) short bits static final long LMASK = 0xffffffffL; // lower 32 bits of long static final long UMASK = ~LMASK; // upper 32 bits @@ -1033,11 +1034,11 @@ public class ForkJoinPool extends AbstractExecutorService { static final int PRESET_SIZE = 1 << 2; // size was set by property // source history window packing used in scan() and runWorker() - static final long WMASK = ((long)SMASK) << 48; // must be negative - static final long RESCAN = 1L << 63; - static final long RAN = 1L << 62; - static final long INVALID_HISTORY = ((((long)INVALID_ID) << 32) | // no 3rd + static final long RESCAN = 1L << 63; // must be negative + static final long WMASK = ~(((long)SMASK) << 48); // id bits only + static final long NO_HISTORY = ((((long)INVALID_ID) << 32) | // no 3rd (((long)INVALID_ID) << 16)); // no 2nd + // others static final int DEREGISTERED = 1 << 31; // worker terminating static final int UNCOMPENSATE = 1 << 16; // tryCompensate return @@ -1329,7 +1330,7 @@ a, slotOffset(k & m), null)) == null) } if (!internal) unlockPhase(); - if ((room == 0 || room == m || a[m & (s - 1)] == null) && + if ((room == 0 || room >= m || a[m & (s - 1)] == null) && pool != null) pool.signalWork(); } @@ -2022,11 +2023,11 @@ else if (compareAndSetCtl(c, c) && // confirm final void runWorker(WorkQueue w) { if (w != null) { int phase = w.phase, r = w.stackPred; // seed from registerWorker - for (long window = INVALID_HISTORY | (r >>> 16);;) { + for (long window = NO_HISTORY | (r >>> 16);;) { r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift if ((runState & STOP) != 0) // terminating break; - if (window == (window = scan(w, window, r)) && + if (window == (window = scan(w, window & WMASK, r)) && window >= 0L) { // empty scan long c = ctl; // try to inactivate int idlePhase = phase + IDLE; @@ -2039,7 +2040,7 @@ final void runWorker(WorkQueue w) { else if ((phase = awaitWork(w, qc, idlePhase)) == idlePhase) break; // worker exit else // clear history - window = INVALID_HISTORY | (window & SMASK); + window = NO_HISTORY | (window & SMASK); } } } @@ -2051,55 +2052,52 @@ else if ((phase = awaitWork(w, qc, idlePhase)) == idlePhase) * returning next scan window and retry indicator. * * @param w caller's WorkQueue - * @param window up to three queue indices and RAN indicator from last scan + * @param window up to three queue indices * @param r random seed - * @return the next window value to use, nonnegative if empty + * @return the next window to use, with RESCAN set for rescan */ private long scan(WorkQueue w, long window, int r) { WorkQueue[] qs = queues; int n = (qs == null) ? 0 : qs.length, step = (r << 1) | 1; - int i = (short)window; // origin - boolean running = (window & RAN) != 0L; // true if last scan succeeded - long next = window &= ~WMASK; // default for empty scan - outer: for (int l = n; l > 0; --l, i += step) { + outer: for (int i = (short)window, l = n; l > 0; --l, i += step) { int j, cap; WorkQueue q; ForkJoinTask[] a; if ((q = qs[j = i & SMASK & (n - 1)]) != null && (a = q.array) != null && (cap = a.length) > 0) { for (boolean contended = false;;) { - int b, k; + int b, k; Object o; ForkJoinTask t = a[k = (b = q.base) & (cap - 1)]; U.loadFence(); // re-read b and t if (q.base == b) { // else inconsistent; retry - Object o; // to check identities int nb = b + 1, nk = nb & (cap - 1); if (t == null) { if (a[k] == null) { - if (!contended && next >= 0L && + if (!contended && window >= 0L && (a[nk] != null || q.top - b > 0)) - next |= RESCAN; // revisit + window |= RESCAN; // revisit break; } } else if (t == (o = U.compareAndExchangeReference( a, slotOffset(k), t, null))) { q.updateBase(nb); - long nw = ((window << 16) | j) & ~WMASK; - next = nw | (RESCAN | RAN); - if ((window != nw || (short)(nw >> 32) != j) && + long pw = window, nw = ((pw << 16) | j) & WMASK; + window = nw | RESCAN; + if ((nw != pw || (short)(nw >>> 32) != j) && a[nk] != null) + if (nw != pw && a[nk] != null) signalWork(); // limit propagation if (w != null) // always true w.topLevelExec(t, q, j); break outer; } - else // limit retries on contention - contended = (o == null && !running && - q.array == a && a[nk] != null); + else if ((contended = (o == null)) && + (short)(window >>> 16) == INVALID_ID) + break; // contended and newly active } } } } - return next; + return window; } /** From 45585dc3819be60a8ab1f2274b776a1dc3fe7b27 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Wed, 11 Oct 2023 09:00:36 -0400 Subject: [PATCH 59/61] More cleanup --- .../classes/java/util/concurrent/ForkJoinPool.java | 4 ++-- .../util/concurrent/ExecutorService/CloseTest.java | 10 ++-------- test/jdk/java/util/concurrent/TEST.properties | 1 + 3 files changed, 5 insertions(+), 10 deletions(-) create mode 100644 test/jdk/java/util/concurrent/TEST.properties diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 8503140e8805d..165bc8d0ba957 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -373,7 +373,7 @@ public class ForkJoinPool extends AbstractExecutorService { * a simple spinlock (as one role of field "phase") because * submitters encountering a busy queue move to a different * position to use or create other queues. They (spin) block when - * registering new queues, or indirectly elsewhere (by revisiting + * registering new queues, or indirectly elsewhere, by revisiting * later. * * Management @@ -2394,7 +2394,7 @@ else if (a[k] == t) { } /** - * Runs tasks until all workerss are inactive and no tasks are + * Runs tasks until all workers are inactive and no tasks are * found. Rather than blocking when tasks cannot be found, rescans * until all others cannot find tasks either. * diff --git a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java index cab28f45d4a35..162d4f7d1b4b5 100644 --- a/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java +++ b/test/jdk/java/util/concurrent/ExecutorService/CloseTest.java @@ -188,14 +188,8 @@ void testShutdownBeforeClose(ExecutorService executor) throws Exception { assertTrue(executor.isShutdown()); assertTrue(executor.isTerminated()); assertTrue(executor.awaitTermination(10, TimeUnit.MILLISECONDS)); - try { - Object s = future.resultNow(); - assertEquals("foo", s); - } catch (Exception e) { - System.err.println("future => " + future.state()); - e.printStackTrace(); - fail(); - } + Object s = future.resultNow(); + assertEquals("foo", s); } /** diff --git a/test/jdk/java/util/concurrent/TEST.properties b/test/jdk/java/util/concurrent/TEST.properties new file mode 100644 index 0000000000000..74a024f2880ac --- /dev/null +++ b/test/jdk/java/util/concurrent/TEST.properties @@ -0,0 +1 @@ +maxOutputSize=2000000 From f46aa909400479ec94d0284728cd2c6988c5b0a0 Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Fri, 20 Oct 2023 14:05:43 -0400 Subject: [PATCH 60/61] Ensure liveness for InterruptibleTasks --- .../java/util/concurrent/ForkJoinPool.java | 143 ++++++++++-------- 1 file changed, 79 insertions(+), 64 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 165bc8d0ba957..cfeed20ed7418 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -613,18 +613,18 @@ public class ForkJoinPool extends AbstractExecutorService { * that not finding tasks doesn't mean that there won't soon be * some. Further, a scan may give up under contention, returning * even without knowing whether any tasks are still present, which - * is OK, given the above signalling rules that will eventually - * maintain progress. Blocking and unblocking via park/unpark can - * cause serious slowdowns when tasks are rapidly but irregularly - * generated (which is often due to garbage collectors and other - * activities). One way to ameliorate is for workers to rescan - * multiple times, even when there are unlikely to be tasks. But - * this causes enough memory traffic and CAS contention to prefer - * using quieter short spinwaits in awaitWork and elsewhere. - * Those in awaitWork are set to small values that only cover - * near-miss scenarios for inactivate/activate races. Because idle - * workers are often not yet blocked (parked), we use the - * WorkQueue parker field to advertise that a waiter actually + * is OK given, a secondary check (in awaitWork) needed to cover + * deactivation/signal races. Blocking and unblocking via + * park/unpark can cause serious slowdowns when tasks are rapidly + * but irregularly generated (which is often due to garbage + * collectors and other activities). One way to ameliorate is for + * workers to rescan multiple times, even when there are unlikely + * to be tasks. But this causes enough memory traffic and CAS + * contention to prefer using quieter short spinwaits in awaitWork + * and elsewhere. Those in awaitWork are set to small values that + * only cover near-miss scenarios for inactivate/activate races. + * Because idle workers are often not yet blocked (parked), we use + * the WorkQueue parker field to advertise that a waiter actually * needs unparking upon signal. * * Quiescence. Workers scan looking for work, giving up when they @@ -2024,23 +2024,13 @@ final void runWorker(WorkQueue w) { if (w != null) { int phase = w.phase, r = w.stackPred; // seed from registerWorker for (long window = NO_HISTORY | (r >>> 16);;) { - r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift - if ((runState & STOP) != 0) // terminating + r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift + if ((runState & STOP) != 0) // terminating break; - if (window == (window = scan(w, window & WMASK, r)) && - window >= 0L) { // empty scan - long c = ctl; // try to inactivate - int idlePhase = phase + IDLE; - long qc = (((phase + (IDLE << 1)) & LMASK) | - ((c - RC_UNIT) & UMASK)); - w.stackPred = (int)c; // set ctl stack link - w.phase = idlePhase; - if (!compareAndSetCtl(c, qc)) - w.phase = phase; // contended; back out - else if ((phase = awaitWork(w, qc, idlePhase)) == idlePhase) - break; // worker exit - else // clear history - window = NO_HISTORY | (window & SMASK); + if ((window = scan(w, window & WMASK, r)) >= 0L) { + if (((phase = awaitWork(w, phase)) & IDLE) != 0) + break; // worker exit + window = NO_HISTORY | (window & SMASK); // clear history } } } @@ -2084,7 +2074,6 @@ a, slotOffset(k), t, null))) { window = nw | RESCAN; if ((nw != pw || (short)(nw >>> 32) != j) && a[nk] != null) - if (nw != pw && a[nk] != null) signalWork(); // limit propagation if (w != null) // always true w.topLevelExec(t, q, j); @@ -2101,46 +2090,72 @@ else if ((contended = (o == null)) && } /** - * Awaits signal or termination. + * Tries to inactivate, and if successful, awaits signal or termination. * - * @param queuedCtl ctl at point of inactivation - * @param idlePhase w's phase while idle - * @return current phase, same as idlePhase for exit + * @param w the worker (may be null if already terminated) + * @param p current phase + * @return current phase, with IDLE set if worker should exit */ - private int awaitWork(WorkQueue w, long queuedCtl, int idlePhase) { - int p = idlePhase; - int spins = ((((int)(queuedCtl >>> TC_SHIFT)) & SMASK) << 1) | 0x2f; - boolean quiescent = (queuedCtl & RC_MASK) <= 0L && quiescent(); - if (w != null && (runState & STOP) == 0) { - while ((p = w.phase) == idlePhase && --spins > 0) - Thread.onSpinWait(); // spin for approx #accesses to scan+signal - if (p == idlePhase) { // emulate LockSupport.park - long deadline = (quiescent ? // timeout for trim - keepAlive + System.currentTimeMillis() : 0L); - LockSupport.setCurrentBlocker(this); - w.parker = Thread.currentThread(); - for (;;) { - if ((runState & STOP) != 0 || (p = w.phase) != idlePhase) - break; - U.park(quiescent, deadline); - if ((p = w.phase) != idlePhase || (runState & STOP) != 0) - break; - Thread.interrupted(); // clear status for next park - if (quiescent && // trim on timeout - deadline - System.currentTimeMillis() < TIMEOUT_SLOP) { - int id = idlePhase & SMASK; - long sp = w.stackPred & LMASK; - long c = ctl, nc = sp | (UMASK & (c - TC_UNIT)); - if (((int)c & SMASK) == id && compareAndSetCtl(c, nc)) { - w.source = DEREGISTERED; // deregisterWorker sentinel - w.phase = idlePhase + IDLE; - break; + private int awaitWork(WorkQueue w, int p) { + long pc = ctl; + int idlePhase = p + IDLE, nextPhase = p + (IDLE << 1); + long qc = (nextPhase & LMASK) | ((pc - RC_UNIT) & UMASK); + if (w != null) { // try to inactivate + w.stackPred = (int)pc; // set ctl stack link + w.phase = idlePhase; + if (!compareAndSetCtl(pc, qc)) // contended enque + w.phase = nextPhase; // don't wait + else { + boolean quiescent = (qc & RC_MASK) <= 0L && quiescent(); + if ((p = w.phase) != nextPhase && (runState & STOP) == 0) { + WorkQueue[] qs = queues; + int n = (qs == null) ? 0 : qs.length, k = n | 0x1f; + while ((p = w.phase) != nextPhase && --k > 0) + Thread.onSpinWait(); // spin approx #accesses to signal + if (p != nextPhase) { // recheck queues + for (int i = 0; i < n; ++i) { + WorkQueue q; ForkJoinTask[] a; int cap; + if ((q = qs[i]) != null && (a = q.array) != null && + (cap = a.length) > 0 && + a[q.base & (cap - 1)] != null) { + if (ctl == qc && compareAndSetCtl(qc, pc)) + w.phase = p = nextPhase; // release + break; + } + } + } + if (p != nextPhase && (p = w.phase) != nextPhase) { + long deadline = (!quiescent ? 0L : // timeout for trim + keepAlive + System.currentTimeMillis()); + LockSupport.setCurrentBlocker(this); + w.parker = Thread.currentThread(); + for (;;) { // emulate LockSupport.park + if ((runState & STOP) != 0 || + (p = w.phase) == nextPhase) + break; + U.park(quiescent, deadline); + if ((p = w.phase) == nextPhase || + (runState & STOP) != 0) + break; + Thread.interrupted(); // clear status for next park + if (quiescent && TIMEOUT_SLOP > + deadline - System.currentTimeMillis()) { + int id = idlePhase & SMASK; + long sp = w.stackPred & LMASK; + long c = ctl, nc = sp | (UMASK & (c - TC_UNIT)); + if (((int)c & SMASK) == id && + compareAndSetCtl(c, nc)) { + w.source = DEREGISTERED; // sentinel + w.phase = nextPhase; + break; // trim on timeout + } + deadline += keepAlive; // not head; restart timer + } } - deadline += keepAlive; // not at head; restart timer + w.parker = null; + LockSupport.setCurrentBlocker(null); } } - w.parker = null; - LockSupport.setCurrentBlocker(null); } } return p; From caeb37a4c3dad32228fe0cef08f4d9aba134d3fd Mon Sep 17 00:00:00 2001 From: Doug Lea Date: Thu, 26 Oct 2023 14:28:42 -0400 Subject: [PATCH 61/61] reduce memory contention --- .../java/util/concurrent/ForkJoinPool.java | 129 +++++++++--------- 1 file changed, 61 insertions(+), 68 deletions(-) diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index cfeed20ed7418..a14f6d884e0b5 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -1992,11 +1992,9 @@ else if (!swept || e != prevRunState || (e & RS_LOCK) != 0) { int p = q.phase, s = q.top, b = q.base; sum += (p & 0xffffffffL) | ((long)b << 32); if ((p & IDLE) == 0 || s - b > 0) { - if (parallelism == 0) - return false; if ((i & 1) == 0 && compareAndSetCtl(c, c)) signalWork(); // ensure live - continue outer; + return false; } } } @@ -2027,12 +2025,13 @@ final void runWorker(WorkQueue w) { r ^= r << 13; r ^= r >>> 17; r ^= r << 5; // xorshift if ((runState & STOP) != 0) // terminating break; - if ((window = scan(w, window & WMASK, r)) >= 0L) { - if (((phase = awaitWork(w, phase)) & IDLE) != 0) + if (window == (window = scan(w, window & WMASK, r)) && + window >= 0L && phase != (phase = awaitWork(w, phase))) { + if ((phase & IDLE) != 0) break; // worker exit window = NO_HISTORY | (window & SMASK); // clear history } - } + } } } @@ -2053,17 +2052,16 @@ private long scan(WorkQueue w, long window, int r) { int j, cap; WorkQueue q; ForkJoinTask[] a; if ((q = qs[j = i & SMASK & (n - 1)]) != null && (a = q.array) != null && (cap = a.length) > 0) { - for (boolean contended = false;;) { + for (;;) { int b, k; Object o; ForkJoinTask t = a[k = (b = q.base) & (cap - 1)]; U.loadFence(); // re-read b and t if (q.base == b) { // else inconsistent; retry int nb = b + 1, nk = nb & (cap - 1); if (t == null) { - if (a[k] == null) { - if (!contended && window >= 0L && - (a[nk] != null || q.top - b > 0)) - window |= RESCAN; // revisit + if (a[k] == null) { // revisit if another task + if (window >= 0L && a[nk] != null) + window |= RESCAN; break; } } @@ -2079,9 +2077,8 @@ a, slotOffset(k), t, null))) { w.topLevelExec(t, q, j); break outer; } - else if ((contended = (o == null)) && - (short)(window >>> 16) == INVALID_ID) - break; // contended and newly active + else if (o == null) // contended + break; // retried unless newly active } } } @@ -2097,64 +2094,60 @@ else if ((contended = (o == null)) && * @return current phase, with IDLE set if worker should exit */ private int awaitWork(WorkQueue w, int p) { - long pc = ctl; - int idlePhase = p + IDLE, nextPhase = p + (IDLE << 1); - long qc = (nextPhase & LMASK) | ((pc - RC_UNIT) & UMASK); - if (w != null) { // try to inactivate - w.stackPred = (int)pc; // set ctl stack link - w.phase = idlePhase; - if (!compareAndSetCtl(pc, qc)) // contended enque - w.phase = nextPhase; // don't wait - else { - boolean quiescent = (qc & RC_MASK) <= 0L && quiescent(); - if ((p = w.phase) != nextPhase && (runState & STOP) == 0) { - WorkQueue[] qs = queues; - int n = (qs == null) ? 0 : qs.length, k = n | 0x1f; - while ((p = w.phase) != nextPhase && --k > 0) - Thread.onSpinWait(); // spin approx #accesses to signal - if (p != nextPhase) { // recheck queues - for (int i = 0; i < n; ++i) { - WorkQueue q; ForkJoinTask[] a; int cap; - if ((q = qs[i]) != null && (a = q.array) != null && - (cap = a.length) > 0 && - a[q.base & (cap - 1)] != null) { - if (ctl == qc && compareAndSetCtl(qc, pc)) - w.phase = p = nextPhase; // release - break; - } - } + if (w != null) { + int idlePhase = p + IDLE, nextPhase = p + (IDLE << 1); + long pc = ctl, qc = (nextPhase & LMASK) | ((pc - RC_UNIT) & UMASK); + w.stackPred = (int)pc; // set ctl stack link + w.phase = idlePhase; // try to inactivate + if (!compareAndSetCtl(pc, qc)) // contended enque + return w.phase = p; // back out + int ac = (short)(qc >>> RC_SHIFT); + boolean quiescent = (ac <= 0 && quiescent()); + if ((runState & STOP) != 0) + return idlePhase; + int spins = ac + ((((int)(qc >>> TC_SHIFT)) & SMASK) << 1); + while ((p = w.phase) == idlePhase && --spins > 0) + Thread.onSpinWait(); // spin for approx #accesses to signal + if (p == idlePhase) { + long deadline = (!quiescent ? 0L : // timeout for trim + System.currentTimeMillis() + keepAlive); + WorkQueue[] qs = queues; + int n = (qs == null) ? 0 : qs.length; + for (int i = 0; i < n; ++i) { // recheck queues + WorkQueue q; ForkJoinTask[] a; int cap; + if ((q = qs[i]) != null && + (a = q.array) != null && (cap = a.length) > 0 && + a[q.base & (cap - 1)] != null && + ctl == qc && compareAndSetCtl(qc, pc)) { + w.phase = (int)qc; // release + break; } - if (p != nextPhase && (p = w.phase) != nextPhase) { - long deadline = (!quiescent ? 0L : // timeout for trim - keepAlive + System.currentTimeMillis()); - LockSupport.setCurrentBlocker(this); - w.parker = Thread.currentThread(); - for (;;) { // emulate LockSupport.park - if ((runState & STOP) != 0 || - (p = w.phase) == nextPhase) - break; - U.park(quiescent, deadline); - if ((p = w.phase) == nextPhase || - (runState & STOP) != 0) + } + if ((p = w.phase) == idlePhase) { // emulate LockSupport.park + LockSupport.setCurrentBlocker(this); + w.parker = Thread.currentThread(); + for (;;) { + if ((runState & STOP) != 0 || (p = w.phase) != idlePhase) + break; + U.park(quiescent, deadline); + if ((p = w.phase) != idlePhase || (runState & STOP) != 0) + break; + Thread.interrupted(); // clear for next park + if (quiescent && TIMEOUT_SLOP > + deadline - System.currentTimeMillis()) { + long sp = w.stackPred & LMASK; + long c = ctl, nc = sp | (UMASK & (c - TC_UNIT)); + if (((int)c & SMASK) == (idlePhase & SMASK) && + compareAndSetCtl(c, nc)) { + w.source = DEREGISTERED; + w.phase = (int)c; break; - Thread.interrupted(); // clear status for next park - if (quiescent && TIMEOUT_SLOP > - deadline - System.currentTimeMillis()) { - int id = idlePhase & SMASK; - long sp = w.stackPred & LMASK; - long c = ctl, nc = sp | (UMASK & (c - TC_UNIT)); - if (((int)c & SMASK) == id && - compareAndSetCtl(c, nc)) { - w.source = DEREGISTERED; // sentinel - w.phase = nextPhase; - break; // trim on timeout - } - deadline += keepAlive; // not head; restart timer } + deadline += keepAlive; // not head; reset timer } - w.parker = null; - LockSupport.setCurrentBlocker(null); } + w.parker = null; + LockSupport.setCurrentBlocker(null); } } } @@ -3282,7 +3275,7 @@ public int setParallelism(int size) { } /** - * Uninterrupible version of {@code InvokeAll}. Executes the given + * Uninterrupible version of {@code invokeAll}. Executes the given * tasks, returning a list of Futures holding their status and * results when all complete, ignoring interrupts. {@link * Future#isDone} is {@code true} for each element of the returned