[GR-64981] Reliably detect pending deoptimization and correctly handle speculations and invalidation.

substratevm/src/com.oracle.svm.core.graal.aarch64/src/com/oracle/svm/core/graal/aarch64/AArch64FarReturnOp.java

@@ -27,6 +27,7 @@
 import static com.oracle.svm.core.graal.aarch64.SubstrateAArch64RegisterConfig.fp;
 import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.ILLEGAL;
 import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.REG;
+import static jdk.vm.ci.aarch64.AArch64.lr;
 import static jdk.vm.ci.code.ValueUtil.asRegister;
 
 import com.oracle.svm.core.CalleeSavedRegisters;
@@ -67,6 +68,16 @@ public AArch64FarReturnOp(AllocatableValue result, AllocatableValue sp, Allocata
     public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
         assert sp.getPlatformKind().getSizeInBytes() == FrameAccess.wordSize() && FrameAccess.wordSize() == Long.BYTES : Assertions.errorMessage(sp.getPlatformKind().getSizeInBytes(),
                         FrameAccess.wordSize());
+        /*
+         * Set lr to the new instruction pointer like a regular return does.
+         *
+         * For dispatching an exception into a frame pending deoptimization, we farReturn into a
+         * deopt stub which will do a regular enter and thus push lr. This keeps the stack walkable
+         * and the frame can be recognized as pending deoptimization due to the return address from
+         * lr matching the deopt stub entry point.
+         */
+        masm.mov(64, lr, asRegister(ip));
+
         if (!SubstrateOptions.PreserveFramePointer.getValue() && !fromMethodWithCalleeSavedRegisters) {
             /* No need to restore anything in the frame of the new stack pointer. */
             masm.mov(64, AArch64.sp, asRegister(sp));

substratevm/src/com.oracle.svm.core.graal.aarch64/src/com/oracle/svm/core/graal/aarch64/SubstrateAArch64Backend.java

@@ -1026,8 +1026,9 @@ public void returned(CompilationResultBuilder crb) {
     }
 
     /**
-     * Generates the prolog of a
-     * {@link com.oracle.svm.core.deopt.Deoptimizer.StubType#EagerEntryStub} method.
+     * Generates the prologue of a
+     * {@link com.oracle.svm.core.deopt.Deoptimizer.StubType#EagerEntryStub} or
+     * {@link com.oracle.svm.core.deopt.Deoptimizer.StubType#LazyEntryStub} method.
      */
     protected static class DeoptEntryStubContext extends SubstrateAArch64FrameContext {
         protected final CallingConvention callingConvention;
@@ -1052,8 +1053,8 @@ public void enter(CompilationResultBuilder crb) {
             masm.fmov(64, thirdParameter, fpReturnReg);
 
             /*
-             * Load DeoptimizedFrame. Perform this operation last, because the first argument
-             * register may overlap with the object return register.
+             * Pass the address of the frame to deoptimize as first argument. Do this last, because
+             * the first argument register may overlap with the object return register.
              */
             Register firstParameter = ValueUtil.asRegister(callingConvention.getArgument(0));
             masm.mov(64, firstParameter, registerConfig.getFrameRegister());
@@ -1063,7 +1064,7 @@ public void enter(CompilationResultBuilder crb) {
     }
 
     /**
-     * Generates the epilog of a {@link com.oracle.svm.core.deopt.Deoptimizer.StubType#ExitStub}
+     * Generates the epilogue of a {@link com.oracle.svm.core.deopt.Deoptimizer.StubType#ExitStub}
      * method.
      */
     protected static class DeoptExitStubContext extends SubstrateAArch64FrameContext {

substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/AMD64FarReturnOp.java

@@ -72,6 +72,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
         if (!SubstrateOptions.PreserveFramePointer.getValue() && !fromMethodWithCalleeSavedRegisters) {
             /* No need to restore anything in the frame of the new stack pointer. */
             masm.movq(AMD64.rsp, asRegister(sp));
+            masm.movq(masm.makeAddress(AMD64.rsp, -FrameAccess.returnAddressSize()), asRegister(ip));
             masm.jmp(asRegister(ip));
             return;
         }

substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64Backend.java

@@ -1335,8 +1335,9 @@ public void returned(CompilationResultBuilder crb) {
     }
 
     /**
-     * Generates the prolog of a
-     * {@link com.oracle.svm.core.deopt.Deoptimizer.StubType#EagerEntryStub} method.
+     * Generates the prologue of a
+     * {@link com.oracle.svm.core.deopt.Deoptimizer.StubType#EagerEntryStub} or
+     * {@link com.oracle.svm.core.deopt.Deoptimizer.StubType#LazyEntryStub} method.
      */
     protected static class DeoptEntryStubContext extends SubstrateAMD64FrameContext {
         protected DeoptEntryStubContext(SharedMethod method, CallingConvention callingConvention) {
@@ -1350,28 +1351,35 @@ public void enter(CompilationResultBuilder tasm) {
             Register gpReturnReg = registerConfig.getReturnRegister(JavaKind.Long);
             Register fpReturnReg = registerConfig.getReturnRegister(JavaKind.Double);
 
-            /* Move the DeoptimizedFrame into the first calling convention register. */
-            Register deoptimizedFrame = ValueUtil.asRegister(callingConvention.getArgument(0));
-            assert !deoptimizedFrame.equals(gpReturnReg) : "overwriting return reg";
+            Register firstArgument = ValueUtil.asRegister(callingConvention.getArgument(0));
+            assert !firstArgument.equals(gpReturnReg) : "overwriting return register";
             /*
              * Since this is the target for all deoptimizations we must mark the start of this
              * routine as an indirect target.
              */
             asm.maybeEmitIndirectTargetMarker();
-            asm.movq(deoptimizedFrame, registerConfig.getFrameRegister());
+
+            /* Pass the address of the frame to deoptimize as first argument. */
+            asm.movq(firstArgument, registerConfig.getFrameRegister());
 
             /* Copy the original return registers values into the argument registers. */
             asm.movq(ValueUtil.asRegister(callingConvention.getArgument(1)), gpReturnReg);
             asm.movdq(ValueUtil.asRegister(callingConvention.getArgument(2)), fpReturnReg);
 
-            /* Add a dummy return address to the stack so that RSP is properly aligned. */
+            /*
+             * Keep the return address slot. This keeps the stack walkable, which is crucial for the
+             * interruptible phase of lazy deoptimization. (The return address points to the deopt
+             * stub, while the original return address is stored in the deopt slot.)
+             *
+             * This also ensures that the stack pointer is aligned properly.
+             */
             asm.subq(registerConfig.getFrameRegister(), FrameAccess.returnAddressSize());
             super.enter(tasm);
         }
     }
 
     /**
-     * Generates the epilog of a {@link com.oracle.svm.core.deopt.Deoptimizer.StubType#ExitStub}
+     * Generates the epilogue of a {@link com.oracle.svm.core.deopt.Deoptimizer.StubType#ExitStub}
      * method.
      *
      * Note no special handling is necessary for CFI as this will be a direct call from the

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/DeoptimizationRuntime.java

@@ -71,6 +71,9 @@ private static void deoptimize(long actionAndReason, SpeculationReason speculati
 
             if (Deoptimizer.Options.TraceDeoptimization.getValue()) {
                 CodePointer ip = KnownIntrinsics.readReturnAddress();
+                if (Deoptimizer.checkLazyDeoptimized(ip)) {
+                    ip = Deoptimizer.readLazyDeoptOriginalReturnAddress(CurrentIsolate.getCurrentThread(), sp);
+                }
                 traceDeoptimization(actionAndReason, speculation, action, sp, ip);
             }