[GR-38639] Lazily walk continuation stacks during GC (if still reachable).

PullRequest: graal/11836

Author: peter-hofer

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -902,7 +902,7 @@ private void walkStack(JavaStackWalk walk) {
                 }
 
                 CodeInfoAccess.lookupCodeInfo(codeInfo, CodeInfoAccess.relativeIP(codeInfo, ip), queryResult);
-                assert Deoptimizer.checkDeoptimized(sp) == null : "We are at a safepoint, so no deoptimization can have happened even though looking up the code info is not uninterruptible";
+                assert Deoptimizer.checkDeoptimized(sp) == null : "We are at a safepoint, so no deoptimization can have happened";
 
                 NonmovableArray<Byte> referenceMapEncoding = CodeInfoAccess.getStackReferenceMapEncoding(codeInfo);
                 long referenceMapIndex = queryResult.getReferenceMapIndex();

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/FrameAccess.java

@@ -51,6 +51,9 @@ public static FrameAccess singleton() {
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public abstract void writeReturnAddress(Pointer sourceSp, CodePointer newReturnAddress);
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public abstract Pointer getReturnAddressLocation(Pointer sourceSp);
+
     @Fold
     public static int returnAddressSize() {
         Architecture arch = ConfigurationValues.getTarget().arch;

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/aarch64/AArch64FrameAccess.java

@@ -50,7 +50,7 @@ public class AArch64FrameAccess extends FrameAccess {
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public CodePointer readReturnAddress(Pointer sourceSp) {
         /* Read the return address, which is stored immediately below the stack pointer. */
-        return (CodePointer) sourceSp.readWord(-returnAddressSize());
+        return sourceSp.readWord(-returnAddressSize());
     }
 
     @Override
@@ -59,6 +59,11 @@ public void writeReturnAddress(Pointer sourceSp, CodePointer newReturnAddress) {
         sourceSp.writeWord(-returnAddressSize(), newReturnAddress);
     }
 
+    @Override
+    public Pointer getReturnAddressLocation(Pointer sourceSp) {
+        return sourceSp.subtract(returnAddressSize());
+    }
+
     @Fold
     @Override
     public int savedBasePointerSize() {

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/amd64/AMD64FrameAccess.java

@@ -53,7 +53,7 @@ public final class AMD64FrameAccess extends FrameAccess {
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public CodePointer readReturnAddress(Pointer sourceSp) {
         /* Read the return address, which is stored just below the stack pointer. */
-        return (CodePointer) sourceSp.readWord(-returnAddressSize());
+        return sourceSp.readWord(-returnAddressSize());
     }
 
     @Override
@@ -62,6 +62,12 @@ public void writeReturnAddress(Pointer sourceSp, CodePointer newReturnAddress) {
         sourceSp.writeWord(-returnAddressSize(), newReturnAddress);
     }
 
+    @Override
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public Pointer getReturnAddressLocation(Pointer sourceSp) {
+        return sourceSp.subtract(returnAddressSize());
+    }
+
     @Fold
     @Override
     public int savedBasePointerSize() {

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java

@@ -44,7 +44,7 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, NonmovableArra
         assert referenceMapEncoding.isNonNull();
 
         if (Continuation.isSupported() && referenceMapIndex == ReferenceMapIndex.STORED_CONTINUATION) {
-            return StoredContinuationImpl.walkStoredContinuationFromPointer(baseAddress, null, visitor, holderObject);
+            return StoredContinuationAccess.walkReferences(baseAddress, visitor, holderObject);
         }
 
         Pointer position = NonmovableByteArrayReader.pointerTo(referenceMapEncoding, referenceMapIndex);

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java

@@ -26,14 +26,10 @@
 
 import com.oracle.svm.core.annotate.Hybrid;
 
-/**
- * This class is used for variably-sized objects that store continuation stack frames.
- *
- * For object layout and other implementation details, see {@link StoredContinuationImpl}.
- */
-@Hybrid(componentType = byte.class)
+/** Execution state of a continuation, use via {@link StoredContinuationAccess}. */
+@Hybrid(componentType = long.class)
 public final class StoredContinuation {
-    /** Must be allocated via {@link StoredContinuationImpl}. */
+    /** Must be allocated via {@link StoredContinuationAccess}. */
     private StoredContinuation() {
     }
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationAccess.java

@@ -0,0 +1,274 @@
+/*
+ * Copyright (c) 2020, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.core.heap;
+
+import org.graalvm.compiler.api.directives.GraalDirectives;
+import org.graalvm.compiler.graph.Node.NodeIntrinsic;
+import org.graalvm.compiler.nodes.java.ArrayLengthNode;
+import org.graalvm.compiler.word.Word;
+import org.graalvm.nativeimage.ImageSingletons;
+import org.graalvm.nativeimage.IsolateThread;
+import org.graalvm.nativeimage.StackValue;
+import org.graalvm.nativeimage.c.function.CodePointer;
+import org.graalvm.word.Pointer;
+import org.graalvm.word.UnsignedWord;
+import org.graalvm.word.WordFactory;
+
+import com.oracle.svm.core.UnmanagedMemoryUtil;
+import com.oracle.svm.core.annotate.AlwaysInline;
+import com.oracle.svm.core.annotate.Uninterruptible;
+import com.oracle.svm.core.c.NonmovableArray;
+import com.oracle.svm.core.code.CodeInfo;
+import com.oracle.svm.core.code.CodeInfoAccess;
+import com.oracle.svm.core.code.CodeInfoTable;
+import com.oracle.svm.core.code.FrameInfoQueryResult;
+import com.oracle.svm.core.code.SimpleCodeInfoQueryResult;
+import com.oracle.svm.core.code.UntetheredCodeInfo;
+import com.oracle.svm.core.deopt.DeoptimizedFrame;
+import com.oracle.svm.core.deopt.Deoptimizer;
+import com.oracle.svm.core.graal.nodes.SubstrateNewHybridInstanceNode;
+import com.oracle.svm.core.hub.LayoutEncoding;
+import com.oracle.svm.core.snippets.KnownIntrinsics;
+import com.oracle.svm.core.stack.JavaStackWalk;
+import com.oracle.svm.core.stack.JavaStackWalker;
+import com.oracle.svm.core.stack.StackFrameVisitor;
+import com.oracle.svm.core.thread.Continuation;
+import com.oracle.svm.core.thread.ContinuationSupport;
+import com.oracle.svm.core.thread.Safepoint;
+import com.oracle.svm.core.util.UnsignedUtils;
+import com.oracle.svm.core.util.VMError;
+
+/** Helper for allocating and accessing {@link StoredContinuation} instances. */
+public final class StoredContinuationAccess {
+    private static final int IP_OFFSET = 0; // instruction pointer of top frame
+    private static final int FRAMES_OFFSET = IP_OFFSET + Long.BYTES;
+
+    private StoredContinuationAccess() {
+    }
+
+    private static StoredContinuation allocate(int framesSize) {
+        // Using long[] to ensure that words are properly aligned.
+        int nlongs = Integer.divideUnsigned(FRAMES_OFFSET + framesSize, Long.BYTES);
+        StoredContinuation s = (StoredContinuation) SubstrateNewHybridInstanceNode.allocate(StoredContinuation.class, long.class, nlongs);
+        assert getFramesSizeInBytes(s) == framesSize;
+        return s;
+    }
+
+    @NodeIntrinsic(ArrayLengthNode.class)
+    private static native int arrayLength(StoredContinuation s);
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    private static int getSizeInBytes(StoredContinuation s) {
+        return arrayLength(s) * Long.BYTES;
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static int getFramesSizeInBytes(StoredContinuation s) {
+        return getSizeInBytes(s) - FRAMES_OFFSET;
+    }
+
+    @Uninterruptible(reason = "Prevent GC during accesses via object address.", callerMustBe = true)
+    private static Pointer arrayAddress(StoredContinuation s) {
+        int layout = KnownIntrinsics.readHub(s).getLayoutEncoding();
+        UnsignedWord baseOffset = LayoutEncoding.getArrayBaseOffset(layout);
+        return Word.objectToUntrackedPointer(s).add(baseOffset);
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static CodePointer getIP(StoredContinuation s) {
+        return arrayAddress(s).readWord(IP_OFFSET);
+    }
+
+    @Uninterruptible(reason = "Prevent GC during accesses via object address.", callerMustBe = true)
+    public static Pointer getFramesStart(StoredContinuation s) {
+        return arrayAddress(s).add(FRAMES_OFFSET);
+    }
+
+    public static int allocateToYield(Continuation c, Pointer baseSp, Pointer sp, CodePointer ip) {
+        assert sp.isNonNull() && ip.isNonNull();
+        return allocateFromStack(c, baseSp, sp, ip, WordFactory.nullPointer());
+    }
+
+    public static int allocateToPreempt(Continuation c, Pointer baseSp, IsolateThread targetThread) {
+        return allocateFromStack(c, baseSp, WordFactory.nullPointer(), WordFactory.nullPointer(), targetThread);
+    }
+
+    private static int allocateFromStack(Continuation cont, Pointer baseSp, Pointer sp, CodePointer ip, IsolateThread targetThread) {
+        boolean yield = sp.isNonNull();
+        assert yield == ip.isNonNull() && yield == targetThread.isNull();
+        assert baseSp.isNonNull();
+
+        Pointer startSp = sp;
+        CodePointer startIp = ip;
+        if (!yield) {
+            PreemptVisitor visitor = new PreemptVisitor(baseSp);
+            JavaStackWalker.walkThread(targetThread, visitor);
+            if (visitor.preemptStatus != Continuation.YIELD_SUCCESS) {
+                return visitor.preemptStatus;
+            }
+            startSp = visitor.leafSP;
+            startIp = visitor.leafIP;
+        }
+
+        VMError.guarantee(startSp.isNonNull());
+
+        int framesSize = UnsignedUtils.safeToInt(baseSp.subtract(startSp));
+        StoredContinuation instance = allocate(framesSize);
+        fillUninterruptibly(instance, startIp, startSp, framesSize);
+        cont.stored = instance;
+        return Continuation.YIELD_SUCCESS;
+    }
+
+    @Uninterruptible(reason = "Prevent modifications to the stack while initializing instance and copying frames.")
+    private static void fillUninterruptibly(StoredContinuation stored, CodePointer ip, Pointer sp, int size) {
+        arrayAddress(stored).writeWord(IP_OFFSET, ip);
+        UnmanagedMemoryUtil.copy(sp, getFramesStart(stored), WordFactory.unsigned(size));
+        afterFill(stored);
+    }
+
+    @Uninterruptible(reason = "Prevent modifications to the stack while initializing instance.")
+    private static void afterFill(StoredContinuation stored) {
+        /*
+         * Since its allocation, our StoredContinuation could have already been promoted to the old
+         * generation and some references we just copied might point to the young generation and
+         * need to be added to the remembered set.
+         *
+         * To support precise marking and pre-write barriers, we need to check first if the object
+         * needs barriers, then, on a slow path, individually copy references from stack frames.
+         */
+        // Drop type info to not trigger compiler assertions about StoredContinuation in barriers
+        Object opaque = GraalDirectives.opaque(stored);
+        Heap.getHeap().dirtyAllReferencesOf(opaque);
+    }
+
+    public static StoredContinuation clone(StoredContinuation cont) {
+        StoredContinuation clone = allocate(getFramesSizeInBytes(cont));
+        return fillCloneUninterruptibly(cont, clone);
+    }
+
+    @Uninterruptible(reason = "Prevent garbage collection while initializing instance and copying frames.")
+    private static StoredContinuation fillCloneUninterruptibly(StoredContinuation cont, StoredContinuation clone) {
+        CodePointer ip = ImageSingletons.lookup(ContinuationSupport.class).copyFrames(cont, getFramesStart(clone));
+        // copyFrames() above may do something interruptible before uninterruptibly copying frames,
+        // so set IP only afterwards so that the object is considered uninitialized until then.
+        arrayAddress(clone).writeWord(IP_OFFSET, ip);
+        afterFill(clone);
+        return clone;
+    }
+
+    /** Derived from {@link InstanceReferenceMapDecoder#walkOffsetsFromPointer}. */
+    @AlwaysInline("De-virtualize calls to ObjectReferenceVisitor")
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static boolean walkReferences(Pointer baseAddress, ObjectReferenceVisitor visitor, Object holderObject) {
+        assert !Heap.getHeap().isInImageHeap(baseAddress);
+
+        StoredContinuation s = (StoredContinuation) holderObject;
+        assert baseAddress.equal(Word.objectToUntrackedPointer(holderObject));
+
+        CodePointer startIp = getIP(s);
+        if (startIp.isNull()) {
+            return true; // uninitialized, ignore
+        }
+
+        Pointer startSp = getFramesStart(s);
+        Pointer endSp = arrayAddress(s).add(getSizeInBytes(s));
+
+        JavaStackWalk walk = StackValue.get(JavaStackWalk.class);
+        JavaStackWalker.initWalk(walk, startSp, endSp, startIp);
+
+        SimpleCodeInfoQueryResult queryResult = StackValue.get(SimpleCodeInfoQueryResult.class);
+        do {
+            Pointer sp = walk.getSP();
+            CodePointer ip = walk.getPossiblyStaleIP();
+
+            UntetheredCodeInfo untetheredCodeInfo = walk.getIPCodeInfo();
+            Object tether = CodeInfoAccess.acquireTether(untetheredCodeInfo);
+            try {
+                CodeInfo codeInfo = CodeInfoAccess.convert(untetheredCodeInfo);
+                VMError.guarantee(codeInfo.equal(CodeInfoTable.getImageCodeInfo()));
+                VMError.guarantee(Deoptimizer.checkDeoptimized(sp) == null);
+                if (codeInfo.isNull()) {
+                    throw JavaStackWalker.reportUnknownFrameEncountered(sp, ip, null);
+                }
+
+                CodeInfoAccess.lookupCodeInfo(codeInfo, CodeInfoAccess.relativeIP(codeInfo, ip), queryResult);
+
+                NonmovableArray<Byte> referenceMapEncoding = CodeInfoAccess.getStackReferenceMapEncoding(codeInfo);
+                long referenceMapIndex = queryResult.getReferenceMapIndex();
+                if (referenceMapIndex != ReferenceMapIndex.NO_REFERENCE_MAP) {
+                    CodeReferenceMapDecoder.walkOffsetsFromPointer(sp, referenceMapEncoding, referenceMapIndex, visitor, holderObject);
+                }
+            } finally {
+                CodeInfoAccess.releaseTether(untetheredCodeInfo, tether);
+            }
+        } while (JavaStackWalker.continueWalk(walk, queryResult, null));
+
+        return true;
+    }
+
+    private static final class PreemptVisitor extends StackFrameVisitor {
+        private final Pointer endSP;
+        private boolean startFromNextFrame = false;
+
+        Pointer leafSP;
+        CodePointer leafIP;
+        int preemptStatus = Continuation.YIELD_SUCCESS;
+
+        PreemptVisitor(Pointer endSP) {
+            this.endSP = endSP;
+        }
+
+        @Override
+        protected boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame) {
+            if (sp.aboveOrEqual(endSP)) {
+                return false;
+            }
+
+            FrameInfoQueryResult frameInfo = CodeInfoTable.lookupCodeInfoQueryResult(codeInfo, ip).getFrameInfo();
+            if (frameInfo.getSourceClass().equals(StoredContinuationAccess.class) && frameInfo.getSourceMethodName().equals("allocateToYield")) {
+                // Continuation is already in the process of yielding, cancel preemption.
+                preemptStatus = Continuation.YIELDING;
+                return false;
+            }
+
+            if (leafSP.isNull()) {
+                // Should start from the method calling `enterSlowPathSafepointCheck`.
+                if (startFromNextFrame) {
+                    leafSP = sp;
+                    leafIP = ip;
+                } else {
+                    if (frameInfo.getSourceClass().equals(Safepoint.class) && frameInfo.getSourceMethodName().equals("enterSlowPathSafepointCheck")) {
+                        startFromNextFrame = true;
+                    }
+                    return true;
+                }
+            }
+
+            VMError.guarantee(codeInfo.equal(CodeInfoTable.getImageCodeInfo()));
+
+            return true;
+        }
+    }
+}