diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/EconomyLowTier.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/EconomyLowTier.java index 3213daf336d7..39ac0e655bce 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/EconomyLowTier.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/EconomyLowTier.java @@ -26,6 +26,7 @@ import jdk.graal.compiler.debug.Assertions; import jdk.graal.compiler.graph.Graph; +import jdk.graal.compiler.nodes.GraphState; import jdk.graal.compiler.options.OptionValues; import jdk.graal.compiler.phases.PlaceholderPhase; import jdk.graal.compiler.phases.common.AddressLoweringPhase; @@ -36,6 +37,7 @@ import jdk.graal.compiler.phases.common.LowTierLoweringPhase; import jdk.graal.compiler.phases.common.RemoveOpaqueValuePhase; import jdk.graal.compiler.phases.common.TransplantGraphsPhase; +import jdk.graal.compiler.phases.common.WriteBarrierAdditionPhase; import jdk.graal.compiler.phases.schedule.SchedulePhase; import jdk.graal.compiler.phases.tiers.LowTierContext; @@ -50,6 +52,8 @@ public EconomyLowTier(OptionValues options) { appendPhase(new LowTierLoweringPhase(canonicalizer)); appendPhase(new ExpandLogicPhase(canonicalizer)); + appendPhase(new WriteBarrierAdditionPhase(GraphState.StageFlag.LOW_TIER_BARRIER_ADDITION)); + if (Assertions.assertionsEnabled()) { appendPhase(new BarrierSetVerificationPhase()); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/LowTier.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/LowTier.java index dd551716ec40..c3db9dd65c47 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/LowTier.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/LowTier.java @@ -28,6 +28,7 @@ import jdk.graal.compiler.core.common.GraalOptions; import jdk.graal.compiler.graph.Graph; +import jdk.graal.compiler.nodes.GraphState; import jdk.graal.compiler.options.Option; import jdk.graal.compiler.options.OptionKey; import jdk.graal.compiler.options.OptionType; @@ -47,6 +48,7 @@ import jdk.graal.compiler.phases.common.PropagateDeoptimizeProbabilityPhase; import jdk.graal.compiler.phases.common.RemoveOpaqueValuePhase; import jdk.graal.compiler.phases.common.TransplantGraphsPhase; +import jdk.graal.compiler.phases.common.WriteBarrierAdditionPhase; import jdk.graal.compiler.phases.schedule.SchedulePhase; import jdk.graal.compiler.phases.schedule.SchedulePhase.SchedulingStrategy; import jdk.graal.compiler.phases.tiers.LowTierContext; @@ -87,6 +89,8 @@ public LowTier(OptionValues options) { appendPhase(new FixReadsPhase(true, new SchedulePhase(GraalOptions.StressTestEarlyReads.getValue(options) ? SchedulingStrategy.EARLIEST : SchedulingStrategy.LATEST_OUT_OF_LOOPS_IMPLICIT_NULL_CHECKS))); + appendPhase(new WriteBarrierAdditionPhase(GraphState.StageFlag.LOW_TIER_BARRIER_ADDITION)); + appendPhase(canonicalizerWithoutGVN); /* diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java index 991f7e7496d1..226cf69d6fe2 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java @@ -622,6 +622,44 @@ private long getZGCAddressField(String name) { public final long zBarrierSetRuntimeLoadBarrierOnOopArray = getZGCAddressField("ZBarrierSetRuntime::load_barrier_on_oop_array"); public final int zPointerLoadShift = getConstant("ZPointerLoadShift", Integer.class, -1, osArch.equals("aarch64") && zgcSupport); + /* + * Shenandoah GC support. + */ + /** + * Indicates whether or not the HotSpot VM has been built with Shenandoah support. If not, then + * we don't expect the Shenandoah symbols to be present in JVMCI. + */ + public final boolean hasShenandoahGC = getStore().getConstants().containsKey("INCLUDE_SHENANDOAHGC") && getConstant("INCLUDE_SHENANDOAHGC", Boolean.class); + + /* + * Various Shenandoah GC constants. + */ + public final int shenandoahGCStateOffset = getConstant("ShenandoahThreadLocalData::gc_state_offset", Integer.class, -1, hasShenandoahGC); + public final int shenandoahSATBIndexOffset = getConstant("ShenandoahThreadLocalData::satb_mark_queue_index_offset", Integer.class, -1, hasShenandoahGC); + public final int shenandoahSATBBufferOffset = getConstant("ShenandoahThreadLocalData::satb_mark_queue_buffer_offset", Integer.class, -1, hasShenandoahGC); + public final int shenandoahCardTableOffset = getConstant("ShenandoahThreadLocalData::card_table_offset", Integer.class, -1, hasShenandoahGC); + public final int shenandoahGCRegionSizeBytesShift = getFieldValue("CompilerToVM::Data::shenandoah_region_size_bytes_shift", Integer.class, "int", -1, hasShenandoahGC); + public final long shenandoahGCCSetFastTestAddress = getFieldValue("CompilerToVM::Data::shenandoah_in_cset_fast_test_addr", Long.class, "address", -1L, hasShenandoahGC); + + public final int shenandoahGCStateHasForwarded = getConstant("ShenandoahHeap::HAS_FORWARDED", Integer.class, -1, hasShenandoahGC); + public final int shenandoahGCStateMarking = getConstant("ShenandoahHeap::MARKING", Integer.class, -1, hasShenandoahGC); + public final int shenandoahGCStateEvacuation = getConstant("ShenandoahHeap::EVACUATION", Integer.class, -1, hasShenandoahGC); + public final int shenandoahGCStateUpdateRefs = getConstant("ShenandoahHeap::UPDATE_REFS", Integer.class, -1, hasShenandoahGC); + public final int shenandoahGCStateWeakRoots = getConstant("ShenandoahHeap::WEAK_ROOTS", Integer.class, -1, hasShenandoahGC); + public final int shenandoahGCStateYoungMarking = getConstant("ShenandoahHeap::YOUNG_MARKING", Integer.class, -1, hasShenandoahGC); + public final int shenandoahGCStateOldMarking = getConstant("ShenandoahHeap::OLD_MARKING", Integer.class, -1, hasShenandoahGC); + + /* + * Shenandoah barrier slow-paths. + */ + public final long shenandoahLoadBarrierStrong = getAddress("ShenandoahRuntime::load_reference_barrier_strong", -1L, hasShenandoahGC); + public final long shenandoahLoadBarrierStrongNarrow = getAddress("ShenandoahRuntime::load_reference_barrier_strong_narrow", -1L, hasShenandoahGC); + public final long shenandoahLoadBarrierWeak = getAddress("ShenandoahRuntime::load_reference_barrier_weak", -1L, hasShenandoahGC); + public final long shenandoahLoadBarrierWeakNarrow = getAddress("ShenandoahRuntime::load_reference_barrier_weak_narrow", -1L, hasShenandoahGC); + public final long shenandoahLoadBarrierPhantom = getAddress("ShenandoahRuntime::load_reference_barrier_phantom", -1L, hasShenandoahGC); + public final long shenandoahLoadBarrierPhantomNarrow = getAddress("ShenandoahRuntime::load_reference_barrier_phantom_narrow", -1L, hasShenandoahGC); + public final long shenandoahWriteBarrierPre = getAddress("ShenandoahRuntime::write_barrier_pre", -1L, hasShenandoahGC); + // aarch64 specific nmethod entry barrier support // @formatter:off public final int BarrierSetAssembler_nmethod_patching_type = getFieldValue("CompilerToVM::Data::BarrierSetAssembler_nmethod_patching_type", Integer.class, "int", -1, osArch.equals("aarch64")); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotBackendFactory.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotBackendFactory.java index 3f13486e7dc7..39ea346d0199 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotBackendFactory.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotBackendFactory.java @@ -259,6 +259,8 @@ private BarrierSet createBarrierSet(GraalHotSpotVMConfig config, MetaAccessProvi ResolvedJavaField referentField = HotSpotReplacementsUtil.referentField(metaAccess); if (config.gc == HotSpotGraalRuntime.HotSpotGC.Z) { return new HotSpotZBarrierSet(objectArrayType, referentField); + } else if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + return new HotSpotShenandoahBarrierSet(objectArrayType, referentField, config); } else if (config.gc == HotSpotGraalRuntime.HotSpotGC.Epsilon) { return new NoBarrierSet(); } else if (config.useG1GC()) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalRuntime.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalRuntime.java index faa5833f73e8..8417651cb243 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalRuntime.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalRuntime.java @@ -209,9 +209,7 @@ public enum HotSpotGC { G1("UseG1GC"), Z(true, true, flagIsSet("UseZGC")), Epsilon(true, true, flagIsSet("UseEpsilonGC")), - - // Unsupported GCs - Shenandoah(false, true, flagIsSet("UseShenandoahGC")); + Shenandoah(true, true, flagIsSet("UseShenandoahGC")); HotSpotGC(String flag) { this(true, true, flagIsSet(flag)); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotShenandoahBarrierSet.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotShenandoahBarrierSet.java new file mode 100644 index 000000000000..19cfd7cdd883 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotShenandoahBarrierSet.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot; + +import jdk.graal.compiler.core.common.CompressEncoding; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.nodes.HotSpotCompressionNode; +import org.graalvm.word.LocationIdentity; + +import jdk.graal.compiler.core.common.memory.BarrierType; +import jdk.graal.compiler.core.common.type.AbstractObjectStamp; +import jdk.graal.compiler.core.common.type.Stamp; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahBarrierSet; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.ResolvedJavaField; +import jdk.vm.ci.meta.ResolvedJavaType; + +/** + * Specialization of {@link ShenandoahBarrierSet} that adds support for read barriers on handle + * locations and compressed references. + */ +public class HotSpotShenandoahBarrierSet extends ShenandoahBarrierSet { + private final CompressEncoding oopEncoding; + + public HotSpotShenandoahBarrierSet(ResolvedJavaType objectArrayType, ResolvedJavaField referentField, GraalHotSpotVMConfig config) { + super(objectArrayType, referentField); + this.oopEncoding = config.getOopEncoding(); + this.useLoadRefBarrier = config.getFlag("ShenandoahLoadRefBarrier", Boolean.class); + this.useSATBBarrier = config.getFlag("ShenandoahSATBBarrier", Boolean.class); + this.useCASBarrier = config.getFlag("ShenandoahCASBarrier", Boolean.class); + this.useCardBarrier = config.getFlag("ShenandoahCardBarrier", Boolean.class); + } + + @Override + protected BarrierType barrierForLocation(BarrierType currentBarrier, LocationIdentity location, JavaKind storageKind) { + if (location instanceof HotSpotReplacementsUtil.OopHandleLocationIdentity) { + return BarrierType.READ; + } + return super.barrierForLocation(currentBarrier, location, storageKind); + } + + @Override + public BarrierType readBarrierType(LocationIdentity location, ValueNode address, Stamp loadStamp) { + if (location instanceof HotSpotReplacementsUtil.OopHandleLocationIdentity) { + GraalError.guarantee(loadStamp instanceof AbstractObjectStamp, "expect object, got: " + loadStamp); + return BarrierType.READ; + } + return super.readBarrierType(location, address, loadStamp); + } + + @Override + public BarrierType writeBarrierType(LocationIdentity location) { + if (location instanceof HotSpotReplacementsUtil.OopHandleLocationIdentity) { + return BarrierType.FIELD; + } + return BarrierType.NONE; + } + + @Override + protected ValueNode maybeUncompressReference(ValueNode value, boolean narrow) { + if (value != null && narrow) { + return HotSpotCompressionNode.uncompressWithoutUnique(value.graph(), value, oopEncoding); + } + return value; + } + + @Override + protected ValueNode maybeCompressReference(ValueNode value, boolean narrow) { + if (value != null && narrow) { + return HotSpotCompressionNode.compressWithoutUnique(value.graph(), value, oopEncoding); + } + return value; + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotLIRGenerator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotLIRGenerator.java index b7e65a470a07..6317a8cb22b1 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotLIRGenerator.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotLIRGenerator.java @@ -62,6 +62,7 @@ import jdk.graal.compiler.hotspot.HotSpotLIRGenerator; import jdk.graal.compiler.hotspot.HotSpotLockStack; import jdk.graal.compiler.hotspot.aarch64.g1.AArch64HotSpotG1BarrierSetLIRTool; +import jdk.graal.compiler.hotspot.aarch64.shenandoah.AArch64HotSpotShenandoahBarrierSetLIRGenerator; import jdk.graal.compiler.hotspot.aarch64.z.AArch64HotSpotZBarrierSetLIRGenerator; import jdk.graal.compiler.hotspot.debug.BenchmarkCounters; import jdk.graal.compiler.hotspot.meta.HotSpotProviders; @@ -119,6 +120,9 @@ protected static BarrierSetLIRGeneratorTool getBarrierSet(GraalHotSpotVMConfig c if (config.gc == HotSpotGraalRuntime.HotSpotGC.Z) { return new AArch64HotSpotZBarrierSetLIRGenerator(config, providers); } + if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + return new AArch64HotSpotShenandoahBarrierSetLIRGenerator(config, providers); + } return null; } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotMove.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotMove.java index a4f391c52c96..018ac4e93936 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotMove.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotMove.java @@ -112,6 +112,10 @@ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { Register resultRegister = asRegister(result); Register ptr = asRegister(input); Register base = (isRegister(baseRegister) ? asRegister(baseRegister) : zr); + emitCompressCode(masm, ptr, resultRegister, base, encoding, nonNull); + } + + public static void emitCompressCode(AArch64MacroAssembler masm, Register ptr, Register resultRegister, Register base, CompressEncoding encoding, boolean nonNull) { // result = (ptr - base) >> shift if (!encoding.hasBase()) { if (encoding.hasShift()) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahBarrierSetLIRGenerator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahBarrierSetLIRGenerator.java new file mode 100644 index 000000000000..1fba9c7d607e --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahBarrierSetLIRGenerator.java @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.aarch64.shenandoah; + +import jdk.graal.compiler.asm.aarch64.AArch64Address; +import jdk.graal.compiler.core.aarch64.AArch64LIRGenerator; +import jdk.graal.compiler.core.aarch64.AArch64ReadBarrierSetLIRGenerator; +import jdk.graal.compiler.core.common.LIRKind; +import jdk.graal.compiler.core.common.memory.BarrierType; +import jdk.graal.compiler.core.common.memory.MemoryExtendKind; +import jdk.graal.compiler.core.common.memory.MemoryOrderMode; +import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.lir.LIRFrameState; +import jdk.graal.compiler.lir.Variable; +import jdk.graal.compiler.lir.aarch64.AArch64AddressValue; +import jdk.graal.compiler.lir.gen.LIRGeneratorTool; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadRefBarrierNode; +import jdk.vm.ci.aarch64.AArch64Kind; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.PlatformKind; +import jdk.vm.ci.meta.Value; + +/** + * Lowers Shenandoah barriers to AArch64 LIR. + */ +public class AArch64HotSpotShenandoahBarrierSetLIRGenerator implements ShenandoahBarrierSetLIRGeneratorTool, AArch64ReadBarrierSetLIRGenerator { + public AArch64HotSpotShenandoahBarrierSetLIRGenerator(GraalHotSpotVMConfig config, HotSpotProviders providers) { + this.config = config; + this.providers = providers; + } + + private final GraalHotSpotVMConfig config; + private final HotSpotProviders providers; + + private static ForeignCallLinkage getReadBarrierStub(LIRGeneratorTool tool, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, boolean narrow) { + return switch (strength) { + case STRONG -> + narrow ? tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_NARROW) + : tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER); + case WEAK -> + narrow ? tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_WEAK_NARROW) + : tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_WEAK); + case PHANTOM -> + narrow ? tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_PHANTOM_NARROW) + : tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_PHANTOM); + }; + } + + @Override + public Value emitLoadReferenceBarrier(LIRGeneratorTool tool, Value obj, Value address, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, boolean narrow, boolean notNull) { + PlatformKind platformKind = obj.getPlatformKind(); + LIRKind kind = LIRKind.reference(platformKind); + Value result = tool.newVariable(tool.toRegisterKind(kind)); + ForeignCallLinkage callTarget = getReadBarrierStub(tool, strength, narrow); + AllocatableValue object = tool.asAllocatable(obj); + AArch64AddressValue loadAddress = ((AArch64LIRGenerator) tool).asAddressValue(address, AArch64Address.ANY_SIZE); + tool.getResult().getFrameMapBuilder().callsMethod(callTarget.getOutgoingCallingConvention()); + tool.append(new AArch64HotSpotShenandoahLoadRefBarrierOp(config, providers, tool.asAllocatable(result), object, loadAddress, callTarget, strength, notNull)); + return result; + } + + @Override + public void emitPreWriteBarrier(LIRGeneratorTool lirTool, Value address, AllocatableValue expectedObject, boolean nonNull) { + AllocatableValue temp = lirTool.newVariable(LIRKind.value(AArch64Kind.QWORD)); + // If the assembly must load the value then it needs a temporary to store it. + AllocatableValue temp2 = expectedObject.equals(Value.ILLEGAL) ? lirTool.newVariable(LIRKind.value(AArch64Kind.QWORD)) : Value.ILLEGAL; + + // Load the address into a register + AllocatableValue addressValue = lirTool.newVariable(address.getValueKind()); + lirTool.emitMove(addressValue, address); + + ForeignCallLinkage callTarget = lirTool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_WRITE_BARRIER_PRE); + lirTool.getResult().getFrameMapBuilder().callsMethod(callTarget.getOutgoingCallingConvention()); + lirTool.append(new AArch64HotSpotShenandoahSATBBarrierOp(config, providers, addressValue, expectedObject, temp, temp2, callTarget, nonNull)); + } + + @Override + public void emitCardBarrier(LIRGeneratorTool lirTool, Value address) { + AArch64AddressValue addr = ((AArch64LIRGenerator) lirTool).asAddressValue(address, AArch64Address.ANY_SIZE); + AllocatableValue tmp = lirTool.newVariable(LIRKind.value(AArch64Kind.QWORD)); + lirTool.append(new AArch64HotSpotShenandoahCardBarrierOp(config, providers, addr, tmp)); + } + + @Override + public void emitCompareAndSwapOp(LIRGeneratorTool tool, boolean isLogic, Value address, MemoryOrderMode memoryOrder, AArch64Kind memKind, Variable result, + AllocatableValue allocatableExpectedValue, AllocatableValue allocatableNewValue, BarrierType barrierType) { + AllocatableValue tmp1 = tool.newVariable(LIRKind.value(AArch64Kind.QWORD)); + tool.append(new AArch64HotSpotShenandoahCompareAndSwapOp(config, providers, memKind, memoryOrder, isLogic, result, allocatableExpectedValue, allocatableNewValue, tool.asAllocatable(address), + tmp1)); + } + + @Override + public Value emitAtomicReadAndWrite(LIRGeneratorTool tool, LIRKind readKind, Value address, Value newValue, BarrierType barrierType) { + // We insert the necessary barriers in the node graph, at that level it + // is easier to handle compressed object references. No need to do anything + // special here. + return tool.emitAtomicReadAndWrite(readKind, address, newValue, BarrierType.NONE); + } + + @Override + public Variable emitBarrieredLoad(LIRGeneratorTool tool, LIRKind kind, Value address, LIRFrameState state, MemoryOrderMode memoryOrder, BarrierType barrierType) { + // We insert the necessary barriers in the node graph, at that level it + // is easier to handle compressed object references. No need to do anything + // special here. + return tool.getArithmetic().emitLoad(kind, address, state, memoryOrder, MemoryExtendKind.DEFAULT); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahCardBarrierOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahCardBarrierOp.java new file mode 100644 index 000000000000..8e5cbced7a6a --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahCardBarrierOp.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.aarch64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.aarch64.AArch64Address; +import jdk.graal.compiler.asm.aarch64.AArch64MacroAssembler; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.aarch64.AArch64AddressValue; +import jdk.graal.compiler.lir.aarch64.AArch64LIRInstruction; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; + +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.REG; +import static jdk.vm.ci.aarch64.AArch64.zr; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +/** + * AArch64 backend for the Shenandoah card barrier. + */ +public class AArch64HotSpotShenandoahCardBarrierOp extends AArch64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AArch64HotSpotShenandoahCardBarrierOp.class); + + private final GraalHotSpotVMConfig config; + private final HotSpotProviders providers; + + /** + * The store address. + */ + @Alive({COMPOSITE}) private AArch64AddressValue address; + + @Temp({REG}) private AllocatableValue tmp; + + protected AArch64HotSpotShenandoahCardBarrierOp(GraalHotSpotVMConfig config, HotSpotProviders providers, AArch64AddressValue addr, AllocatableValue tmp) { + super(TYPE); + this.config = config; + this.providers = providers; + this.address = addr; + this.tmp = tmp; + } + + @Override + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp#L373-L392", + sha1 = "1c3e544b6fdec2f4ca0f07b2a1d5261d55754cb9") + // @formatter:on + protected void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { + try (AArch64MacroAssembler.ScratchRegister tmp2 = masm.getScratchRegister(); + AArch64MacroAssembler.ScratchRegister tmp3 = masm.getScratchRegister()) { + Register rtmp1 = asRegister(tmp); + Register rtmp2 = tmp2.getRegister(); + Register rtmp3 = tmp3.getRegister(); + AArch64Address storeAddr = address.toAddress(); + Register rthread = providers.getRegisters().getThreadRegister(); + + // Flatten address if necessary. + Register rAddr; + if (storeAddr.isBaseRegisterOnly()) { + rAddr = storeAddr.getBase(); + } else { + rAddr = rtmp1; + masm.loadAddress(rAddr, storeAddr); + } + + masm.lsr(64, rAddr, rAddr, HotSpotReplacementsUtil.cardTableShift(config)); + + AArch64Address currCTHolderAddr = AArch64Address.createImmediateAddress(64, AArch64Address.AddressingMode.IMMEDIATE_SIGNED_UNSCALED, rthread, + HotSpotReplacementsUtil.shenandoahCardTableOffset(config)); + masm.ldr(64, rtmp2, currCTHolderAddr); + + AArch64Address cardAddr = AArch64Address.createRegisterOffsetAddress(8, rAddr, rtmp2, false); + if (HotSpotReplacementsUtil.useCondCardMark(config)) { + Label alreadyDirty = new Label(); + masm.ldr(8, rtmp3, cardAddr); + masm.cbz(8, rtmp3, alreadyDirty); + masm.str(8, zr, cardAddr); + masm.bind(alreadyDirty); + } else { + masm.str(8, zr, cardAddr); + } + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahCompareAndSwapOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahCompareAndSwapOp.java new file mode 100644 index 000000000000..54afde076c07 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahCompareAndSwapOp.java @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.aarch64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.aarch64.AArch64Address; +import jdk.graal.compiler.asm.aarch64.AArch64Assembler; +import jdk.graal.compiler.asm.aarch64.AArch64MacroAssembler; +import jdk.graal.compiler.core.common.memory.MemoryOrderMode; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.aarch64.AArch64HotSpotMove; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.Variable; +import jdk.graal.compiler.lir.aarch64.AArch64AtomicMove; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.aarch64.AArch64Kind; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; + +import static jdk.vm.ci.aarch64.AArch64.zr; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +/** + * Special Shenandoah CAS implementation that handles false negatives due to concurrent evacuation. + * The service is more complex than a traditional CAS operation because the CAS operation is + * intended to succeed if the reference at addr exactly matches expected or if the reference at addr + * holds a pointer to a from-space object that has been relocated to the location named by expected. + * There are two races that must be addressed: a) A parallel thread may mutate the contents of addr + * so that it points to a different object. In this case, the CAS operation should fail. b) A + * parallel thread may heal the contents of addr, replacing a from-space pointer held in addr with + * the to-space pointer representing the new location of the object. Upon entry to cmpxchg_oop, it + * is assured that new_val equals null or it refers to an object that is not being evacuated out of + * from-space, or it refers to the to-space version of an object that is being evacuated out of + * from-space. + */ +public class AArch64HotSpotShenandoahCompareAndSwapOp extends AArch64AtomicMove.CompareAndSwapOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AArch64HotSpotShenandoahCompareAndSwapOp.class); + + private final HotSpotProviders providers; + private final GraalHotSpotVMConfig config; + + @Temp private AllocatableValue tmp1Value; + + public AArch64HotSpotShenandoahCompareAndSwapOp(GraalHotSpotVMConfig config, HotSpotProviders providers, AArch64Kind accessKind, MemoryOrderMode memoryOrder, boolean isLogicVariant, + Variable result, AllocatableValue expectedValue, AllocatableValue newValue, AllocatableValue address, AllocatableValue tmp1) { + super(TYPE, accessKind, memoryOrder, isLogicVariant, result, expectedValue, newValue, address); + this.providers = providers; + this.config = config; + + this.tmp1Value = tmp1; + } + + @Override + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp#L471-L606", + sha1 = "553a2fb0d37f39016eda85331e8cd2421153cbfe") + // @formatter:on + public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { + Register address = asRegister(addressValue); + Register result = asRegister(resultValue); + Register expected = asRegister(expectedValue); + Register newVal = asRegister(newValue); + + Register tmp1 = asRegister(tmp1Value); + Label step2 = new Label(); + Label done = new Label(); + GraalError.guarantee(accessKind == AArch64Kind.QWORD || accessKind == AArch64Kind.DWORD, "must be 64 or 32 bit access"); + int size = (accessKind == AArch64Kind.QWORD) ? 64 : 32; + + // Step 1. Fast-path. + // + // Try to CAS with given arguments. If successful, then we are done. + + emitCompareAndSwap(masm, accessKind, address, result, expected, newVal, memoryOrder, true); + // EQ flag set iff success. result holds value fetched. + + // If expected equals null but result does not equal null, the + // step2 branches to done to report failure of CAS. If both + // expected and tmp2 equal null, the following branches to done to + // report success of CAS. There's no need for a special test of + // expected equal to null. + + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, step2); + + masm.bind(done); + + crb.getLIR().addSlowPath(this, () -> { + // Step 2. CAS has failed because the value held at addr does not + // match expected. This may be a false negative because the value fetched + // from addr (now held in result) may be a from-space pointer to the + // original copy of same object referenced by to-space pointer expected. + // + // To resolve this, it suffices to find the forward pointer associated + // with fetched value. If this matches expected, retry CAS with new + // parameters. If this mismatches, then we have a legitimate + // failure, and we're done. + masm.bind(step2); + + // Check for null. If we get null, then we have a legitimate failure. + masm.tst(size, result, result); + Label resultNullFailure = setConditionFlags ? new Label() : done; + masm.branchConditionally(AArch64Assembler.ConditionFlag.EQ, resultNullFailure); + + // overwrite tmp1 with from-space pointer fetched from memory + masm.mov(size, tmp1, result); + + // Decode tmp1 in order to resolve its forward pointer + uncompress(masm, tmp1); + + // Load mark-word (i.e. potential forwarding pointer). + masm.ldr(64, tmp1, AArch64Address.createImmediateAddress(64, AArch64Address.AddressingMode.IMMEDIATE_SIGNED_UNSCALED, tmp1, config.markOffset)); + // Invert the mark-word, so that we can test the two lowest bits for 11, while + // preserving the upper bits. + masm.eon(64, tmp1, tmp1, zr); + // Check lowest bits for 00, which would have been originally 11. + // Original 11 indicates a forwarded object. + masm.tst(64, tmp1, config.markWordLockMaskInPlace); + // If not forwarded, then we're done. It must be a legitimate failure. + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, done); + // Set lowest two bits, which will result in actual clearing the bits + // after the following inversion. + masm.orr(64, tmp1, tmp1, config.markWordLockMaskInPlace); + // ... and invert all bits back to get the forwarding pointer into tmp1. + masm.eon(64, tmp1, tmp1, zr); + + // Encode tmp1 to compare against expected. + compress(masm, tmp1); + + // Does forwarded value of fetched from-space pointer match original + // value of expected? + masm.cmp(size, tmp1, expected); + + // If not, then the failure was legitimate and we're done. + // Branching to done with NE condition denotes failure. + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, done); + + // Fall through to step 3. No need for step3 label. + + // Step 3. We've confirmed that the value originally held in memory + // (now held in result) pointed to from-space version of original + // expected value. Try the CAS again with the from-space expected + // value. If it now succeeds, we're good. + // + // Note: result holds encoded from-space pointer that matches to-space + // object residing at expected. result is the new "expected". + masm.mov(size, tmp1, result); + emitCompareAndSwap(masm, accessKind, address, result, tmp1, newVal, memoryOrder, true); + // EQ flag set iff success. result holds value fetched. + + // If fetched value did not equal the new expected, this could + // still be a false negative because some other thread may have + // newly overwritten the memory value with its to-space equivalent. + masm.branchConditionally(AArch64Assembler.ConditionFlag.EQ, done); + + // In the rare case that four steps are required to perform the + // requested operation, the fourth step is the same as the first. + + // Step 4. CAS has failed because the value most recently fetched + // from addr is no longer the from-space pointer held in result. If a + // different thread replaced the in-memory value with its equivalent + // to-space pointer, then CAS may still be able to succeed. The + // value held in the expected register has not changed. + // + // It is extremely rare we reach this point. + + emitCompareAndSwap(masm, accessKind, address, result, expected, newVal, memoryOrder, setConditionFlags); + // EQ flag set iff success. result holds value fetched. + + masm.jmp(done); + + if (setConditionFlags) { + masm.bind(resultNullFailure); + // Clear zero flag to indicate failure. + masm.subs(32, zr, zr, 1); + masm.jmp(done); + } + }); + } + + void uncompress(AArch64MacroAssembler masm, Register obj) { + if (accessKind == AArch64Kind.DWORD) { + Register heapBase = providers.getRegisters().getHeapBaseRegister(); + AArch64HotSpotMove.UncompressPointer.emitUncompressCode(masm, obj, obj, heapBase, config.getOopEncoding(), false); + } + } + + void compress(AArch64MacroAssembler masm, Register obj) { + if (accessKind == AArch64Kind.DWORD) { + Register heapBase = providers.getRegisters().getHeapBaseRegister(); + AArch64HotSpotMove.CompressPointer.emitCompressCode(masm, obj, obj, heapBase, config.getOopEncoding(), false); + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahLoadRefBarrierOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahLoadRefBarrierOp.java new file mode 100644 index 000000000000..3ac7c9d4b31f --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahLoadRefBarrierOp.java @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.aarch64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.aarch64.AArch64Address; +import jdk.graal.compiler.asm.aarch64.AArch64Assembler; +import jdk.graal.compiler.asm.aarch64.AArch64MacroAssembler; +import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.lir.LIRInstruction; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.aarch64.AArch64AddressValue; +import jdk.graal.compiler.lir.aarch64.AArch64Call; +import jdk.graal.compiler.lir.aarch64.AArch64LIRInstruction; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadRefBarrierNode; + +import jdk.vm.ci.code.CallingConvention; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; + +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.REG; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +/** + * AArch64 backend for the Shenandoah load-reference barrier. + */ +public class AArch64HotSpotShenandoahLoadRefBarrierOp extends AArch64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AArch64HotSpotShenandoahLoadRefBarrierOp.class); + + private final HotSpotProviders providers; + private final GraalHotSpotVMConfig config; + + /** + * The slow-path entry for the load-reference-barrier. + */ + private final ForeignCallLinkage callTarget; + + /** + * Strength (strong, weak, phantom) of incoming object reference. This affects whether or not + * the barrier needs to be active in the weak-roots phase, and whether or not we need to check + * for the object to be in the collection set. + */ + private final ShenandoahLoadRefBarrierNode.ReferenceStrength strength; + + /** + * If we know that the incoming object is not null, then we don't need to emit a null-check. + */ + private final boolean notNull; + + /** + * The output of the LRB. Passes the canonicalized reference to the consumer. + */ + @Def({REG}) private AllocatableValue result; + + /** + * The input of the LRB. This is typically a reference that has just been loaded. + */ + @Use({REG}) private AllocatableValue object; + + @Alive({COMPOSITE}) private AArch64AddressValue loadAddress; + + public AArch64HotSpotShenandoahLoadRefBarrierOp(GraalHotSpotVMConfig config, HotSpotProviders providers, + AllocatableValue result, AllocatableValue object, AArch64AddressValue loadAddress, + ForeignCallLinkage callTarget, + ShenandoahLoadRefBarrierNode.ReferenceStrength strength, + boolean notNull) { + super(TYPE); + this.providers = providers; + this.config = config; + this.result = result; + this.object = object; + this.loadAddress = loadAddress; + this.callTarget = callTarget; + this.strength = strength; + this.notNull = notNull; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { + Register thread = providers.getRegisters().getThreadRegister(); + emitCode(config, crb, masm, this, thread, asRegister(result), asRegister(object), loadAddress.toAddress(), callTarget, strength, notNull); + } + + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp#L232-L309", + sha1 = "4ed44f985dfdca39bf93c6d306a378be4bf88fe7") + // @formatter:on + public static void emitCode(GraalHotSpotVMConfig config, CompilationResultBuilder crb, AArch64MacroAssembler masm, LIRInstruction op, Register thread, Register result, Register object, + AArch64Address loadAddress, ForeignCallLinkage callTarget, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, boolean notNull) { + try (AArch64MacroAssembler.ScratchRegister sc1 = masm.getScratchRegister()) { + Register rscratch1 = sc1.getRegister(); + + Label done = new Label(); + Label csetCheck = new Label(); + Label slowPath = new Label(); + + // Move object to result, in case the heap is stable and no barrier needs to be called. + masm.mov(64, result, object); + + if (!notNull) { + // Check for object being null. + masm.cbz(64, result, done); + } + + // Check for heap stability + int gcStateOffset = HotSpotReplacementsUtil.shenandoahGCStateOffset(config); + AArch64Address gcState = masm.makeAddress(8, thread, gcStateOffset); + masm.ldr(8, rscratch1, gcState); + if (strength != ShenandoahLoadRefBarrierNode.ReferenceStrength.STRONG) { + // This is needed because in a short-cut cycle we may get a trailing + // weak-roots phase but no evacuation/update-refs phase, and during that, + // we need to take the LRB to report null for unreachable weak-refs. + // This is true even for non-cset objects. + // Two tests because HAS_FORWARDED | WEAK_ROOTS currently is not representable + // as a single immediate. + masm.tst(64, rscratch1, config.shenandoahGCStateHasForwarded); + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, slowPath); + masm.tst(64, rscratch1, config.shenandoahGCStateWeakRoots); + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, slowPath); + } else { + masm.tst(64, rscratch1, config.shenandoahGCStateHasForwarded); + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, csetCheck); + } + masm.bind(done); + + // Check for object in collection set in an out-of-line mid-path. + if (strength == ShenandoahLoadRefBarrierNode.ReferenceStrength.STRONG) { + crb.getLIR().addSlowPath(op, () -> { + try (AArch64MacroAssembler.ScratchRegister tmp1 = masm.getScratchRegister(); AArch64MacroAssembler.ScratchRegister tmp2 = masm.getScratchRegister()) { + Register rtmp1 = tmp1.getRegister(); + Register rtmp2 = tmp2.getRegister(); + masm.bind(csetCheck); + masm.mov(rtmp1, HotSpotReplacementsUtil.shenandoahGCCSetFastTestAddr(config)); + masm.lsr(64, rtmp2, object, HotSpotReplacementsUtil.shenandoahGCRegionSizeBytesShift(config)); + masm.ldr(8, rtmp2, AArch64Address.createRegisterOffsetAddress(8, rtmp1, rtmp2, false)); + masm.cbnz(32, rtmp2, slowPath); + masm.jmp(done); + } + }); + } + // Call runtime slow-path LRB in out-of-line slow-path. + crb.getLIR().addSlowPath(op, () -> { + try (AArch64MacroAssembler.ScratchRegister tmp1 = masm.getScratchRegister(); AArch64MacroAssembler.ScratchRegister tmp2 = masm.getScratchRegister()) { + Register rtmp1 = tmp1.getRegister(); + Register rtmp2 = tmp2.getRegister(); + masm.bind(slowPath); + CallingConvention cc = callTarget.getOutgoingCallingConvention(); + GraalError.guarantee(cc.getArgumentCount() == 2, "Expecting callTarget to have only 2 parameters. It has " + cc.getArgumentCount()); + + // Store first argument + AArch64Address cArg0 = (AArch64Address) crb.asAddress(cc.getArgument(0)); + masm.str(64, object, cArg0); + + // Store second argument + Register addressReg; + if (loadAddress.isBaseRegisterOnly()) { + // Can directly use the base register as the address + addressReg = loadAddress.getBase(); + } else { + addressReg = rtmp1; + masm.loadAddress(addressReg, loadAddress); + } + AArch64Address cArg1 = (AArch64Address) crb.asAddress(cc.getArgument(1)); + masm.str(64, addressReg, cArg1); + + // Make the call + AArch64Call.directCall(crb, masm, callTarget, AArch64Call.isNearCall(callTarget) ? null : rtmp2, null); + + // Retrieve result and move to the result register. + AArch64Address cRet = (AArch64Address) crb.asAddress(cc.getReturn()); + masm.ldr(64, result, cRet); + masm.jmp(done); + } + }); + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahSATBBarrierOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahSATBBarrierOp.java new file mode 100644 index 000000000000..64f80b7a4c88 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahSATBBarrierOp.java @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.aarch64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.aarch64.AArch64Address; +import jdk.graal.compiler.asm.aarch64.AArch64Assembler; +import jdk.graal.compiler.asm.aarch64.AArch64MacroAssembler; +import jdk.graal.compiler.core.common.CompressEncoding; +import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.aarch64.AArch64HotSpotMacroAssembler; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.aarch64.AArch64Call; +import jdk.graal.compiler.lir.aarch64.AArch64LIRInstruction; +import jdk.graal.compiler.lir.aarch64.AArch64Move; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.code.CallingConvention; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.Value; + +import static jdk.graal.compiler.asm.Assembler.guaranteeDifferentRegisters; +import static jdk.graal.compiler.asm.aarch64.AArch64Address.AddressingMode.IMMEDIATE_SIGNED_UNSCALED; +import static jdk.graal.compiler.core.common.GraalOptions.AssemblyGCBarriersSlowPathOnly; +import static jdk.graal.compiler.core.common.GraalOptions.VerifyAssemblyGCBarriers; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +/** + * AArch64 backend for the Shenandoah SATB barrier. + */ +public class AArch64HotSpotShenandoahSATBBarrierOp extends AArch64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AArch64HotSpotShenandoahSATBBarrierOp.class); + + private final GraalHotSpotVMConfig config; + private final HotSpotProviders providers; + + /** + * The SATB slow-path runtime entry. + */ + private final ForeignCallLinkage callTarget; + + /** + * If we know that the previous value is not null, then we don't need to emit a null-check. + */ + private final boolean nonNull; + + /** + * The store address. + */ + @Alive private Value address; + + /** + * The pre-loaded previous value, if any. + */ + @Alive({OperandFlag.REG, OperandFlag.ILLEGAL}) private Value expectedObject; + + @Temp private Value temp; + + @Temp({OperandFlag.REG, OperandFlag.ILLEGAL}) private Value temp2; + + public AArch64HotSpotShenandoahSATBBarrierOp(GraalHotSpotVMConfig config, HotSpotProviders providers, + AllocatableValue address, AllocatableValue expectedObject, AllocatableValue temp, AllocatableValue temp2, ForeignCallLinkage callTarget, boolean nonNull) { + super(TYPE); + this.config = config; + this.providers = providers; + this.address = address; + GraalError.guarantee(expectedObject.equals(Value.ILLEGAL) ^ temp2.equals(Value.ILLEGAL), "only one register is necessary"); + this.expectedObject = expectedObject; + this.temp = temp; + this.temp2 = temp2; + this.callTarget = callTarget; + this.nonNull = nonNull; + GraalError.guarantee(expectedObject.equals(Value.ILLEGAL) || expectedObject.getPlatformKind().getSizeInBytes() == 8, "expected uncompressed pointer"); + } + + public void loadObject(AArch64MacroAssembler masm, Register preVal, Register immediateAddress) { + if (config.useCompressedOops) { + masm.ldr(32, preVal, AArch64Address.createImmediateAddress(32, IMMEDIATE_SIGNED_UNSCALED, immediateAddress, 0)); + CompressEncoding encoding = config.getOopEncoding(); + AArch64Move.UncompressPointerOp.emitUncompressCode(masm, preVal, preVal, encoding, false, providers.getRegisters().getHeapBaseRegister(), false); + } else { + masm.ldr(64, preVal, AArch64Address.createImmediateAddress(64, IMMEDIATE_SIGNED_UNSCALED, immediateAddress, 0)); + } + } + + @Override + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp#L100-L183", + sha1 = "7b3d183187ff6578e0d14eb54e4b5007ff4d5e1e") + // @formatter:on + protected void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { + Register storeAddress = asRegister(address); + Register thread = providers.getRegisters().getThreadRegister(); + Register tmp = asRegister(temp); + Register previousValue = expectedObject.equals(Value.ILLEGAL) ? asRegister(temp2) : asRegister(expectedObject); + + guaranteeDifferentRegisters(storeAddress, thread, tmp, previousValue); + + Label done = new Label(); + Label midPath = new Label(); + Label runtime = new Label(); + + // Is marking active? + int gcStateOffset = HotSpotReplacementsUtil.shenandoahGCStateOffset(config); + AArch64Address gcState = masm.makeAddress(8, thread, gcStateOffset); + masm.ldr(8, tmp, gcState); + masm.tst(64, tmp, config.shenandoahGCStateMarking); + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, midPath); + masm.bind(done); + + // Out of line mid-path. + crb.getLIR().addSlowPath(this, () -> { + masm.bind(midPath); + + // Do we need to load the previous value? + if (expectedObject.equals(Value.ILLEGAL)) { + loadObject(masm, previousValue, storeAddress); + } + + if (!nonNull) { + // Is the previous value null? + masm.cbz(64, previousValue, done); + } + + if (VerifyAssemblyGCBarriers.getValue(crb.getOptions())) { + try (AArch64MacroAssembler.ScratchRegister sc1 = masm.getScratchRegister()) { + Register tmp2 = sc1.getRegister(); + verifyOop(masm, previousValue, tmp, tmp2, false); + } + } + + if (AssemblyGCBarriersSlowPathOnly.getValue(crb.getOptions())) { + masm.jmp(runtime); + } else { + int satbQueueIndexOffset = HotSpotReplacementsUtil.shenandoahSATBIndexOffset(config); + AArch64Address satbQueueIndex = masm.makeAddress(64, thread, satbQueueIndexOffset); + // tmp := *index_adr + // if tmp == 0 then goto runtime + masm.ldr(64, tmp, satbQueueIndex); + masm.cbz(64, tmp, runtime); + + // tmp := tmp - wordSize + // *index_adr := tmp + // tmp := tmp + *buffer_adr + masm.sub(64, tmp, tmp, 8); + masm.str(64, tmp, satbQueueIndex); + try (AArch64MacroAssembler.ScratchRegister sc1 = masm.getScratchRegister()) { + Register scratch1 = sc1.getRegister(); + int satbQueueBufferOffset = HotSpotReplacementsUtil.shenandoahSATBBufferOffset(config); + AArch64Address satbQueueBuffer = masm.makeAddress(64, thread, satbQueueBufferOffset); + masm.ldr(64, scratch1, satbQueueBuffer); + masm.add(64, tmp, tmp, scratch1); + } + + // Record the previous value + masm.str(64, previousValue, masm.makeAddress(64, tmp, 0)); + masm.jmp(done); + } + }); + + // Out of line slow path + crb.getLIR().addSlowPath(this, () -> { + try (AArch64MacroAssembler.ScratchRegister sc1 = masm.getScratchRegister()) { + Register scratch1 = sc1.getRegister(); + masm.bind(runtime); + CallingConvention cc = callTarget.getOutgoingCallingConvention(); + AArch64Address cArg0 = (AArch64Address) crb.asAddress(cc.getArgument(0)); + masm.str(64, previousValue, cArg0); + AArch64Call.directCall(crb, masm, callTarget, AArch64Call.isNearCall(callTarget) ? null : scratch1, null); + masm.jmp(done); + } + }); + } + + private static void verifyOop(AArch64MacroAssembler masm, Register previousValue, Register tmp, Register tmp2, boolean compressed) { + ((AArch64HotSpotMacroAssembler) masm).verifyOop(previousValue, tmp, tmp2, compressed, true); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java index 59b9ab0ea770..1a4861b996ea 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java @@ -55,6 +55,7 @@ import jdk.graal.compiler.hotspot.HotSpotLIRGenerator; import jdk.graal.compiler.hotspot.HotSpotLockStack; import jdk.graal.compiler.hotspot.amd64.g1.AMD64HotSpotG1BarrierSetLIRTool; +import jdk.graal.compiler.hotspot.amd64.shenandoah.AMD64HotSpotShenandoahBarrierSetLIRGenerator; import jdk.graal.compiler.hotspot.amd64.z.AMD64HotSpotZBarrierSetLIRGenerator; import jdk.graal.compiler.hotspot.debug.BenchmarkCounters; import jdk.graal.compiler.hotspot.meta.HotSpotForeignCallDescriptor; @@ -120,6 +121,9 @@ protected static BarrierSetLIRGeneratorTool getBarrierSet(GraalHotSpotVMConfig c if (config.gc == HotSpotGraalRuntime.HotSpotGC.Z) { return new AMD64HotSpotZBarrierSetLIRGenerator(config, providers); } + if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + return new AMD64HotSpotShenandoahBarrierSetLIRGenerator(config, providers); + } return null; } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahBarrierSetLIRGenerator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahBarrierSetLIRGenerator.java new file mode 100644 index 000000000000..083223b2126b --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahBarrierSetLIRGenerator.java @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.amd64.shenandoah; + +import jdk.graal.compiler.core.amd64.AMD64LIRGenerator; +import jdk.graal.compiler.core.amd64.AMD64ReadBarrierSetLIRGenerator; +import jdk.graal.compiler.core.common.LIRKind; +import jdk.graal.compiler.core.common.memory.BarrierType; +import jdk.graal.compiler.core.common.memory.MemoryExtendKind; +import jdk.graal.compiler.core.common.memory.MemoryOrderMode; +import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.lir.LIRFrameState; +import jdk.graal.compiler.lir.Variable; +import jdk.graal.compiler.lir.amd64.AMD64AddressValue; +import jdk.graal.compiler.lir.gen.LIRGeneratorTool; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadRefBarrierNode; +import jdk.vm.ci.amd64.AMD64Kind; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.PlatformKind; +import jdk.vm.ci.code.RegisterValue; +import jdk.vm.ci.meta.Value; + +public class AMD64HotSpotShenandoahBarrierSetLIRGenerator implements ShenandoahBarrierSetLIRGeneratorTool, AMD64ReadBarrierSetLIRGenerator { + public AMD64HotSpotShenandoahBarrierSetLIRGenerator(GraalHotSpotVMConfig config, HotSpotProviders providers) { + this.config = config; + this.providers = providers; + } + + private final GraalHotSpotVMConfig config; + private final HotSpotProviders providers; + + private static ForeignCallLinkage getReadBarrierStub(LIRGeneratorTool tool, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, boolean narrow) { + return switch (strength) { + case STRONG -> narrow ? tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_NARROW) + : tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER); + case WEAK -> narrow ? tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_WEAK_NARROW) + : tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_WEAK); + case PHANTOM -> narrow ? tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_PHANTOM_NARROW) + : tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_PHANTOM); + }; + } + + @Override + public Value emitLoadReferenceBarrier(LIRGeneratorTool tool, Value obj, Value address, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, boolean narrow, boolean notNull) { + PlatformKind platformKind = obj.getPlatformKind(); + LIRKind kind = LIRKind.reference(platformKind); + Value result = tool.newVariable(tool.toRegisterKind(kind)); + ForeignCallLinkage callTarget = getReadBarrierStub(tool, strength, narrow); + AllocatableValue object = tool.asAllocatable(obj); + AMD64AddressValue loadAddress = ((AMD64LIRGenerator) tool).asAddressValue(address); + AllocatableValue tmp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + AllocatableValue tmp2 = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + tool.getResult().getFrameMapBuilder().callsMethod(callTarget.getOutgoingCallingConvention()); + tool.append(new AMD64HotSpotShenandoahLoadRefBarrierOp(config, providers, tool.asAllocatable(result), object, loadAddress, callTarget, strength, tmp, tmp2, notNull)); + return result; + } + + @Override + public void emitPreWriteBarrier(LIRGeneratorTool lirTool, Value address, AllocatableValue expectedObject, boolean nonNull) { + AllocatableValue temp = lirTool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + // If the assembly must load the value then it's needs a temporary to store it + AllocatableValue temp2 = expectedObject.equals(Value.ILLEGAL) ? lirTool.newVariable(LIRKind.value(AMD64Kind.QWORD)) : Value.ILLEGAL; + AllocatableValue temp3 = lirTool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + + // Load the address into a register + AllocatableValue addressValue = lirTool.newVariable(address.getValueKind()); + lirTool.emitMove(addressValue, address); + + ForeignCallLinkage callTarget = lirTool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_WRITE_BARRIER_PRE); + lirTool.getResult().getFrameMapBuilder().callsMethod(callTarget.getOutgoingCallingConvention()); + lirTool.append(new AMD64HotSpotShenandoahSATBBarrierOp(config, providers, addressValue, expectedObject, temp, temp2, temp3, callTarget, nonNull)); + } + + @Override + public void emitCardBarrier(LIRGeneratorTool lirTool, Value address) { + AMD64AddressValue addr = ((AMD64LIRGenerator) lirTool).asAddressValue(address); + AllocatableValue tmp = lirTool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + AllocatableValue tmp2 = lirTool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + lirTool.append(new AMD64HotSpotShenandoahCardBarrierOp(config, providers, addr, tmp, tmp2)); + } + + @Override + public void emitCompareAndSwapOp(LIRGeneratorTool tool, boolean isLogic, LIRKind accessKind, AMD64Kind memKind, RegisterValue raxValue, AMD64AddressValue address, AllocatableValue newValue, + BarrierType barrierType) { + GraalError.guarantee(barrierType != BarrierType.NONE, "must have barrier type != NONE"); + AllocatableValue tmp1 = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + AllocatableValue tmp2 = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + tool.append(new AMD64HotSpotShenandoahCompareAndSwapOp(config, providers, memKind, raxValue, address, raxValue, newValue, tmp1, tmp2, isLogic)); + } + + @Override + public Value emitAtomicReadAndWrite(LIRGeneratorTool tool, LIRKind readKind, Value address, Value newValue, BarrierType barrierType) { + // We insert the necessary barriers in the node graph, at that level it + // is easier to handle compressed object references. No need to do anything + // special here. + return tool.emitAtomicReadAndWrite(readKind, address, newValue, BarrierType.NONE); + } + + @Override + public Variable emitBarrieredLoad(LIRGeneratorTool tool, LIRKind kind, Value address, LIRFrameState state, MemoryOrderMode memoryOrder, BarrierType barrierType) { + // We insert the necessary barriers in the node graph, at that level it + // is easier to handle compressed object references. No need to do anything + // special here. + return tool.getArithmetic().emitLoad(kind, address, state, memoryOrder, MemoryExtendKind.DEFAULT); + } + +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahCardBarrierOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahCardBarrierOp.java new file mode 100644 index 000000000000..25d483efcaf5 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahCardBarrierOp.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.amd64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.amd64.AMD64Address; +import jdk.graal.compiler.asm.amd64.AMD64MacroAssembler; +import jdk.graal.compiler.asm.amd64.AMD64Assembler; +import jdk.graal.compiler.core.common.Stride; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.amd64.AMD64AddressValue; +import jdk.graal.compiler.lir.amd64.AMD64LIRInstruction; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; + +import static jdk.graal.compiler.asm.Assembler.guaranteeDifferentRegisters; +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.REG; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +/** + * X86 backend for the Shenandoah card barrier. + */ +public class AMD64HotSpotShenandoahCardBarrierOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64HotSpotShenandoahCardBarrierOp.class); + private final GraalHotSpotVMConfig config; + private final HotSpotProviders providers; + + /** + * The store address. + */ + @Alive({COMPOSITE}) private AMD64AddressValue address; + + @Temp({REG}) private AllocatableValue tmp; + + @Temp({REG}) private AllocatableValue tmp2; + + protected AMD64HotSpotShenandoahCardBarrierOp(GraalHotSpotVMConfig config, HotSpotProviders providers, AMD64AddressValue addr, AllocatableValue tmp, AllocatableValue tmp2) { + super(TYPE); + this.config = config; + this.providers = providers; + this.address = addr; + this.tmp = tmp; + this.tmp2 = tmp2; + } + + @Override + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp#L509-L535", + sha1 = "ad163e79b0707221700bb3b2230581fb711ded61") + // @formatter:on + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + Register rtmp1 = asRegister(tmp); + Register rtmp2 = asRegister(tmp2); + Register rthread = providers.getRegisters().getThreadRegister(); + guaranteeDifferentRegisters(rtmp1, rtmp2, rthread); + + masm.leaq(rtmp1, address.toAddress(masm)); + masm.shrq(rtmp1, HotSpotReplacementsUtil.cardTableShift(config)); + + AMD64Address currCTHolderAddr = new AMD64Address(rthread, HotSpotReplacementsUtil.shenandoahCardTableOffset(config)); + masm.movq(rtmp2, currCTHolderAddr); + + AMD64Address cardAddr = new AMD64Address(rtmp1, rtmp2, Stride.S1); + if (HotSpotReplacementsUtil.useCondCardMark(config)) { + Label alreadyDirty = new Label(); + masm.cmpb(cardAddr, 0 /* dirtyCardValue */); + masm.jccb(AMD64Assembler.ConditionFlag.Equal, alreadyDirty); + masm.movb(cardAddr, 0 /* dirtyCardValue */); + masm.bind(alreadyDirty); + } else { + masm.movb(cardAddr, 0 /* dirtyCardValue */); + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahCompareAndSwapOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahCompareAndSwapOp.java new file mode 100644 index 000000000000..ddf8321343f6 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahCompareAndSwapOp.java @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.amd64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.amd64.AMD64Address; +import jdk.graal.compiler.asm.amd64.AMD64Assembler; +import jdk.graal.compiler.asm.amd64.AMD64BaseAssembler; +import jdk.graal.compiler.asm.amd64.AMD64MacroAssembler; +import jdk.graal.compiler.core.common.CompressEncoding; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.lir.amd64.AMD64LIRInstruction; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.Opcode; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.amd64.AMD64AddressValue; +import jdk.graal.compiler.lir.amd64.AMD64Move; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.amd64.AMD64; +import jdk.vm.ci.amd64.AMD64Kind; + +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; +import static jdk.vm.ci.code.ValueUtil.asRegister; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; + +/** + * Special Shenandoah CAS implementation that handles false negatives due to concurrent evacuation. + * The service is more complex than a traditional CAS operation because the CAS operation is + * intended to succeed if the reference at addr exactly matches expected or if the reference at addr + * holds a pointer to a from-space object that has been relocated to the location named by expected. + * There are two races that must be addressed: a) A parallel thread may mutate the contents of addr + * so that it points to a different object. In this case, the CAS operation should fail. b) A + * parallel thread may heal the contents of addr, replacing a from-space pointer held in addr with + * the to-space pointer representing the new location of the object. Upon entry to cmpxchg_oop, it + * is assured that new_val equals null or it refers to an object that is not being evacuated out of + * from-space, or it refers to the to-space version of an object that is being evacuated out of + * from-space. + */ +@Opcode("CAS_Shenandoah") +public class AMD64HotSpotShenandoahCompareAndSwapOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64HotSpotShenandoahCompareAndSwapOp.class); + + private final HotSpotProviders providers; + private final GraalHotSpotVMConfig config; + + private final AMD64Kind accessKind; + private final boolean isLogic; + + @Def private AllocatableValue result; + @Alive({COMPOSITE}) private AMD64AddressValue address; + @Alive private AllocatableValue cmpValue; + @Alive private AllocatableValue newValue; + + @Temp private AllocatableValue tmp1Value; + @Temp private AllocatableValue tmp2Value; + + public AMD64HotSpotShenandoahCompareAndSwapOp(GraalHotSpotVMConfig config, HotSpotProviders providers, AMD64Kind accessKind, AllocatableValue result, AMD64AddressValue address, + AllocatableValue cmpValue, AllocatableValue newValue, AllocatableValue tmp1, AllocatableValue tmp2, boolean isLogic) { + super(TYPE); + this.providers = providers; + this.config = config; + this.accessKind = accessKind; + this.isLogic = isLogic; + this.result = result; + this.address = address; + this.cmpValue = cmpValue; + this.newValue = newValue; + this.tmp1Value = tmp1; + this.tmp2Value = tmp2; + } + + @Override + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp#L596-L737", + sha1 = "c236c9ca8ccf7e45757d6ac04d16a230df9475a0") + // @formatter:on + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + AMD64Address addr = address.toAddress(masm); + Register resultReg = asRegister(result); + Register expectedReg = asRegister(cmpValue); + Register newVal = asRegister(newValue); + + Register tmp1 = asRegister(tmp1Value); + Register tmp2 = asRegister(tmp2Value); + + GraalError.guarantee(resultReg.equals(expectedReg), "same registers"); + GraalError.guarantee(!tmp1.equals(tmp2), "different registers"); + GraalError.guarantee(!tmp1.equals(newVal), "different registers"); + GraalError.guarantee(!tmp1.equals(expectedReg), "different registers"); + GraalError.guarantee(!tmp2.equals(newVal), "different registers"); + GraalError.guarantee(!tmp2.equals(expectedReg), "different registers"); + GraalError.guarantee(!newVal.equals(expectedReg), "different registers"); + + Label done = new Label(); + Label step2 = new Label(); + + AMD64BaseAssembler.OperandSize opSize = AMD64BaseAssembler.OperandSize.get(accessKind); + + // Remember expectedReg for retry logic below + mov(masm, opSize, tmp1, expectedReg); + + // Step 1. Fast-path. + // + // Try to CAS with given arguments. If successful, then we are done. + + // There are two ways to reach this label. Initial entry into the + // cmpxchg_oop code expansion starts at step1 (which is equivalent + // to label step4). Additionally, in the rare case that four steps + // are required to perform the requested operation, the fourth step + // is the same as the first. On a second pass through step 1, + // control may flow through step 2 on its way to failure. It will + // not flow from step 2 to step 3 since we are assured that the + // memory at addr no longer holds a from-space pointer. + emitCompareAndSwap(crb, masm, accessKind, addr, expectedReg, newVal, resultReg); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, step2); + masm.bind(done); + + crb.getLIR().addSlowPath(this, () -> { + // Step 2. CAS had failed. This may be a false negative. + // + // The trouble comes when we compare the to-space pointer with the from-space + // pointer to the same object. To resolve this, it will suffice to resolve + // the value from memory -- this will give both to-space pointers. + // If they mismatch, then it was a legitimate failure. + masm.bind(step2); + + // First we check the value that we just fetched for null. + // If the value is null, then we have a legitimate failure. + // We need the correct ZF at the done label. That is, upon seeing null, + // we need to have ZF=0 to indicate CAS failure. Unfortunately, the + // null-test comes out with ZF=1. Therefore we branch to a small block + // at the end, which clears ZF, and then jumps to done. + Label resultNullFailure = isLogic ? new Label() : done; + masm.testAndJcc(opSize, resultReg, resultReg, AMD64Assembler.ConditionFlag.Zero, resultNullFailure, isLogic); + + // We need to preserve resultReg for the failure paths. + mov(masm, opSize, tmp2, resultReg); + // Uncompress the offending value that we just fetched, + // so that we can read its mark-word. + uncompress(masm, opSize, tmp2); + + // Resolve forwarding ptr into tmp2. + masm.movq(tmp2, new AMD64Address(tmp2, config.markOffset)); + // Invert the whole mark-word, to preserve the upper bits. + // If the object has been forwarded, i.e. the lowest + // two bits have been 11, then these become 00, which is + // easily testable. + masm.notq(tmp2); + // Test lowest two bits for 00. + GraalError.guarantee(config.markWordLockMaskInPlace == 3, "mask must be 3 (11)"); + // If not forwarded, then we have a legitimate failure of the CAS. + masm.testAndJcc(AMD64BaseAssembler.OperandSize.QWORD, tmp2, (int) config.markWordLockMaskInPlace, AMD64Assembler.ConditionFlag.NotZero, done, false); + // Now set the two lowest bits. Upon re-inversion, these become 00. + masm.orq(tmp2, (int) config.markWordLockMaskInPlace); + // Invert again to get the resolved forwardee in tmp2. + masm.notq(tmp2); + // We need to compress it for comparing with original + // expectedReg value. + compress(masm, opSize, tmp2); + + // Now we have the forwarded offender in tmp2. + // Compare with original expectedReg value, and if they don't match, we have legitimate + // failure + masm.cmpAndJcc(opSize, tmp1, tmp2, AMD64Assembler.ConditionFlag.NotEqual, done, false); + + // Step 3. We've confirmed that the value originally held in memory + // (now held in resultReg/expectedReg) pointed to from-space version of original + // expectedReg value. Try the CAS again with the from-space expectedReg + // value. If it now succeeds, we're good. + // + // Note: resultReg holds encoded from-space pointer that matches to-space + // object residing at expectedReg. resultReg is the new "expectedReg". + emitCompareAndSwap(crb, masm, accessKind, addr, expectedReg, newVal, resultReg); + + // If fetched value did not equal the new expectedReg, this could + // still be a false negative because some other thread may have + // newly overwritten the memory value with its to-space equivalent. + masm.jcc(AMD64Assembler.ConditionFlag.Equal, done); + + // Step 4. CAS has failed because the value most recently fetched + // from addr is no longer the from-space pointer held in tmp2. If a + // different thread replaced the in-memory value with its equivalent + // to-space pointer, then CAS may still be able to succeed. The + // value held in the expectedReg register must be updated to expect + // the original expectedReg value (cmpxchg uses the same register for + // expectedReg and resultReg, and it might hold a wrong value now). + mov(masm, opSize, expectedReg, tmp1); + emitCompareAndSwap(crb, masm, accessKind, addr, expectedReg, newVal, resultReg); + masm.jmp(done); + + if (isLogic) { + masm.bind(resultNullFailure); + // Clear 0 register to indicate failure. We arrive here with + // resultReg == 0, but need the ZR flag to be cleared to + // indicate failure on the done label. + masm.cmpl(resultReg, 1); + masm.jmp(done); + } + }); + } + + private static void mov(AMD64MacroAssembler masm, AMD64BaseAssembler.OperandSize opSize, Register dst, Register src) { + if (opSize == AMD64BaseAssembler.OperandSize.QWORD) { + masm.movq(dst, src); + } else if (opSize == AMD64BaseAssembler.OperandSize.DWORD) { + masm.movl(dst, src); + } else { + GraalError.shouldNotReachHereUnexpectedValue(opSize); + } + } + + private void compress(AMD64MacroAssembler masm, AMD64BaseAssembler.OperandSize opSize, Register dst) { + if (opSize == AMD64BaseAssembler.OperandSize.DWORD) { + CompressEncoding encoding = config.getOopEncoding(); + Register heapBase = providers.getRegisters().getHeapBaseRegister(); + AMD64Move.CompressPointerOp.emitCompressCode(masm, dst, encoding.getShift(), heapBase, true); + } else { + GraalError.guarantee(opSize == AMD64BaseAssembler.OperandSize.QWORD, "unexpected opSize"); + } + } + + private void uncompress(AMD64MacroAssembler masm, AMD64BaseAssembler.OperandSize opSize, Register dst) { + if (opSize == AMD64BaseAssembler.OperandSize.DWORD) { + CompressEncoding encoding = config.getOopEncoding(); + Register heapBase = providers.getRegisters().getHeapBaseRegister(); + AMD64Move.UncompressPointerOp.emitUncompressCode(masm, dst, encoding.getShift(), heapBase, true); + } else { + GraalError.guarantee(opSize == AMD64BaseAssembler.OperandSize.QWORD, "unexpected opSize"); + } + } + + private static void emitCompareAndSwap(CompilationResultBuilder crb, AMD64MacroAssembler masm, AMD64Kind accessKind, AMD64Address address, Register expected, Register newValue, Register result) { + GraalError.guarantee(expected.equals(AMD64.rax), "expected must be in rax"); + GraalError.guarantee(result.equals(AMD64.rax), "result must be in rax"); + + if (crb.target.isMP) { + masm.lock(); + } + switch (accessKind) { + case DWORD: + masm.cmpxchgl(address, newValue); + break; + case QWORD: + masm.cmpxchgq(newValue, address); + break; + default: + throw GraalError.shouldNotReachHereUnexpectedValue(accessKind); // ExcludeFromJacocoGeneratedReport + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahLoadRefBarrierOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahLoadRefBarrierOp.java new file mode 100644 index 000000000000..de5a489a075a --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahLoadRefBarrierOp.java @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.amd64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.amd64.AMD64Address; +import jdk.graal.compiler.asm.amd64.AMD64Assembler; +import jdk.graal.compiler.asm.amd64.AMD64BaseAssembler; +import jdk.graal.compiler.asm.amd64.AMD64MacroAssembler; +import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.lir.LIRInstruction; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.amd64.AMD64AddressValue; +import jdk.graal.compiler.lir.amd64.AMD64Call; +import jdk.graal.compiler.lir.amd64.AMD64LIRInstruction; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadRefBarrierNode; + +import jdk.vm.ci.code.CallingConvention; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; + +import static jdk.graal.compiler.asm.Assembler.guaranteeDifferentRegisters; +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.REG; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +/** + * X86 backend for the Shenandoah load-reference barrier. + */ +public class AMD64HotSpotShenandoahLoadRefBarrierOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64HotSpotShenandoahLoadRefBarrierOp.class); + + private final HotSpotProviders providers; + private final GraalHotSpotVMConfig config; + + /** + * The slow-path entry for the load-reference-barrier. + */ + private final ForeignCallLinkage callTarget; + + /** + * Strength (strong, weak, phantom) of incoming object reference. This affects whether or not + * the barrier needs to be active in the weak-roots phase, and whether or not we need to check + * for the object to be in the collection set. + */ + private final ShenandoahLoadRefBarrierNode.ReferenceStrength strength; + + /** + * If we know that the incoming object is not null, then we don't need to emit a null-check. + */ + private final boolean notNull; + + @Temp({REG}) private AllocatableValue tmp; + + @Temp({REG}) private AllocatableValue tmp2; + + /** + * The output of the LRB. Passes the canonicalized reference to the consumer. + */ + @Def({REG}) private AllocatableValue result; + + /** + * The input of the LRB. This is typically a reference that has just been loaded. + */ + @Alive({REG}) private AllocatableValue object; + + /** + * The address from where the reference has been loaded, if any. + */ + @Alive({COMPOSITE}) private AMD64AddressValue loadAddress; + + public AMD64HotSpotShenandoahLoadRefBarrierOp(GraalHotSpotVMConfig config, HotSpotProviders providers, + AllocatableValue result, AllocatableValue object, AMD64AddressValue loadAddress, + ForeignCallLinkage callTarget, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, + AllocatableValue tmp, AllocatableValue tmp2, boolean notNull) { + super(TYPE); + this.providers = providers; + this.config = config; + this.result = result; + this.object = object; + this.loadAddress = loadAddress; + this.callTarget = callTarget; + this.strength = strength; + this.notNull = notNull; + this.tmp = tmp; + this.tmp2 = tmp2; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + Register thread = providers.getRegisters().getThreadRegister(); + Register rtmp1 = asRegister(tmp); + Register rtmp2 = asRegister(tmp2); + Register objectRegister = asRegister(object); + Register resultRegister = asRegister(result); + AMD64Address loadAddr = loadAddress.toAddress(masm); + emitCode(config, crb, masm, this, thread, resultRegister, objectRegister, rtmp1, rtmp2, loadAddr, callTarget, strength, notNull); + } + + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp#L296-L430", + sha1 = "a039ddb87ee03446a7d015f2d955eb3014c9413e") + // @formatter:on + public static void emitCode(GraalHotSpotVMConfig config, CompilationResultBuilder crb, AMD64MacroAssembler masm, LIRInstruction op, Register thread, Register resultRegister, + Register objectRegister, Register rtmp1, Register rtmp2, + AMD64Address loadAddr, ForeignCallLinkage callTarget, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, boolean notNull) { + guaranteeDifferentRegisters(thread, rtmp1, rtmp2, objectRegister, resultRegister); + + Label done = new Label(); + Label csetCheck = new Label(); + Label slowPath = new Label(); + + // Move object to result, in case the heap is stable and no barrier needs to be called. + masm.movq(resultRegister, objectRegister); + + if (!notNull) { + // Check for object being null. + masm.testAndJcc(AMD64BaseAssembler.OperandSize.QWORD, resultRegister, resultRegister, AMD64Assembler.ConditionFlag.Zero, done, true); + } + + // Check for heap stability + masm.movb(rtmp1, new AMD64Address(thread, HotSpotReplacementsUtil.shenandoahGCStateOffset(config))); + if (strength != ShenandoahLoadRefBarrierNode.ReferenceStrength.STRONG) { + // This is needed because in a short-cut cycle we may get a trailing + // weak-roots phase but no evacuation/update-refs phase, and during that, + // we need to take the LRB to report null for unreachable weak-refs. + // This is true even for non-cset objects. + masm.testlAndJcc(rtmp1, config.shenandoahGCStateHasForwarded | config.shenandoahGCStateWeakRoots, AMD64Assembler.ConditionFlag.NotZero, slowPath, false); + } else { + masm.testlAndJcc(rtmp1, config.shenandoahGCStateHasForwarded, AMD64Assembler.ConditionFlag.NotZero, csetCheck, false); + } + masm.bind(done); + + // Check for object in collection set in an out-of-line mid-path. + if (strength == ShenandoahLoadRefBarrierNode.ReferenceStrength.STRONG) { + crb.getLIR().addSlowPath(op, () -> { + masm.bind(csetCheck); + + masm.movq(rtmp1, HotSpotReplacementsUtil.shenandoahGCCSetFastTestAddr(config)); + masm.movq(rtmp2, objectRegister); + masm.shrq(rtmp2, HotSpotReplacementsUtil.shenandoahGCRegionSizeBytesShift(config)); + + masm.addq(rtmp2, rtmp1); + masm.cmpb(new AMD64Address(rtmp2), 0); + masm.jcc(AMD64Assembler.ConditionFlag.NotZero, slowPath); + + masm.jmp(done); + }); + } + + // Call runtime slow-path LRB in out-of-line slow-path. + crb.getLIR().addSlowPath(op, () -> { + masm.bind(slowPath); + CallingConvention cc = callTarget.getOutgoingCallingConvention(); + GraalError.guarantee(cc.getArgumentCount() == 2, "Expecting callTarget to have only 2 parameters. It has " + cc.getArgumentCount()); + + AMD64Address cArg0 = (AMD64Address) crb.asAddress(cc.getArgument(0)); + AMD64Address cArg1 = (AMD64Address) crb.asAddress(cc.getArgument(1)); + + // Store first argument + masm.movq(cArg0, objectRegister); + + // Store second argument + masm.leaq(rtmp1, loadAddr); + masm.movq(cArg1, rtmp1); + + // Make the call + AMD64Call.directCall(crb, masm, callTarget, null, false, null); + + // Retrieve result and move to the result register. + AMD64Address cRet = (AMD64Address) crb.asAddress(cc.getReturn()); + masm.movq(resultRegister, cRet); + masm.jmp(done); + }); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahSATBBarrierOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahSATBBarrierOp.java new file mode 100644 index 000000000000..2809053b4327 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahSATBBarrierOp.java @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.amd64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.amd64.AMD64Address; +import jdk.graal.compiler.asm.amd64.AMD64Assembler; +import jdk.graal.compiler.asm.amd64.AMD64BaseAssembler; +import jdk.graal.compiler.asm.amd64.AMD64MacroAssembler; +import jdk.graal.compiler.core.common.CompressEncoding; +import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.amd64.AMD64HotSpotMacroAssembler; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.amd64.AMD64Call; +import jdk.graal.compiler.lir.amd64.AMD64LIRInstruction; +import jdk.graal.compiler.lir.amd64.AMD64Move; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.code.CallingConvention; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.Value; + +import static jdk.graal.compiler.asm.Assembler.guaranteeDifferentRegisters; +import static jdk.graal.compiler.core.common.GraalOptions.AssemblyGCBarriersSlowPathOnly; +import static jdk.graal.compiler.core.common.GraalOptions.VerifyAssemblyGCBarriers; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +/** + * X86 backend for the Shenandoah SATB barrier. + */ +public class AMD64HotSpotShenandoahSATBBarrierOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64HotSpotShenandoahSATBBarrierOp.class); + + private final GraalHotSpotVMConfig config; + private final HotSpotProviders providers; + + /** + * The SATB slow-path runtime entry. + */ + private final ForeignCallLinkage callTarget; + + /** + * If we know that the previous value is not null, then we don't need to emit a null-check. + */ + private final boolean nonNull; + + /** + * The store address. + */ + @Alive private Value address; + + /** + * The pre-loaded previous value, if any. + */ + @Alive({OperandFlag.REG, OperandFlag.ILLEGAL}) private Value expectedObject; + + @Temp private Value temp; + + @Temp({OperandFlag.REG, OperandFlag.ILLEGAL}) private Value temp2; + + @Temp private Value temp3; + + public AMD64HotSpotShenandoahSATBBarrierOp(GraalHotSpotVMConfig config, HotSpotProviders providers, + AllocatableValue address, AllocatableValue expectedObject, + AllocatableValue temp, AllocatableValue temp2, AllocatableValue temp3, + ForeignCallLinkage callTarget, boolean nonNull) { + super(TYPE); + this.config = config; + this.providers = providers; + this.address = address; + GraalError.guarantee(expectedObject.equals(Value.ILLEGAL) ^ temp2.equals(Value.ILLEGAL), "only one register is necessary"); + this.expectedObject = expectedObject; + this.temp = temp; + this.temp2 = temp2; + this.temp3 = temp3; + this.callTarget = callTarget; + this.nonNull = nonNull; + GraalError.guarantee(expectedObject.equals(Value.ILLEGAL) || expectedObject.getPlatformKind().getSizeInBytes() == 8, "expected uncompressed pointer"); + } + + public void loadObject(AMD64MacroAssembler masm, Register preVal, Register immediateAddress) { + if (config.useCompressedOops) { + masm.movl(preVal, new AMD64Address(immediateAddress)); + CompressEncoding encoding = config.getOopEncoding(); + AMD64Move.UncompressPointerOp.emitUncompressCode(masm, preVal, encoding.getShift(), providers.getRegisters().getHeapBaseRegister(), false); + } else { + masm.movq(preVal, new AMD64Address(immediateAddress)); + } + } + + @Override + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp#L189-L294", + sha1 = "8fb9ab816c4ee89723e0cf3efb15dcea87128044") + // @formatter:on + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + Register storeAddress = asRegister(address); + Register thread = providers.getRegisters().getThreadRegister(); + Register tmp = asRegister(temp); + Register tmp3 = asRegister(temp3); + Register previousValue = expectedObject.equals(Value.ILLEGAL) ? asRegister(temp2) : asRegister(expectedObject); + guaranteeDifferentRegisters(storeAddress, thread, tmp, tmp3, previousValue); + + Label done = new Label(); + Label runtime = new Label(); + + // Is marking active? + masm.movb(tmp, new AMD64Address(thread, HotSpotReplacementsUtil.shenandoahGCStateOffset(config))); + masm.testlAndJcc(tmp, config.shenandoahGCStateMarking, AMD64Assembler.ConditionFlag.Zero, done, true); + + // Do we need to load the previous value? + if (expectedObject.equals(Value.ILLEGAL)) { + loadObject(masm, previousValue, storeAddress); + } + + if (!nonNull) { + // Is the previous value null? + masm.testAndJcc(AMD64BaseAssembler.OperandSize.QWORD, previousValue, previousValue, AMD64Assembler.ConditionFlag.Zero, done, true); + } + + if (VerifyAssemblyGCBarriers.getValue(crb.getOptions())) { + verifyOop(masm, previousValue, tmp, tmp3); + } + + if (AssemblyGCBarriersSlowPathOnly.getValue(crb.getOptions())) { + masm.jmp(runtime); + } else { + int satbQueueIndexOffset = HotSpotReplacementsUtil.shenandoahSATBIndexOffset(config); + AMD64Address satbQueueIndex = new AMD64Address(thread, satbQueueIndexOffset); + // tmp := *index_adr + // if tmp == 0 then goto runtime + masm.movq(tmp, satbQueueIndex); + masm.cmpq(tmp, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, runtime); + + // tmp := tmp - wordSize + // *index_adr := tmp + masm.subq(tmp, 8); + masm.movq(satbQueueIndex, tmp); + + // tmp := tmp + *buffer_adr + int satbQueueBufferOffset = HotSpotReplacementsUtil.shenandoahSATBBufferOffset(config); + AMD64Address satbQueueBuffer = new AMD64Address(thread, satbQueueBufferOffset); + masm.movq(tmp3, satbQueueBuffer); + masm.addq(tmp, tmp3); + + // Record the previous value + masm.movq(new AMD64Address(tmp), previousValue); + } + masm.bind(done); + + // Out of line slow path + crb.getLIR().addSlowPath(this, () -> { + masm.bind(runtime); + CallingConvention cc = callTarget.getOutgoingCallingConvention(); + GraalError.guarantee(cc.getArgumentCount() == 1, "Expecting callTarget to have only 1 parameters. It has " + cc.getArgumentCount()); + + AMD64Address cArg0 = (AMD64Address) crb.asAddress(cc.getArgument(0)); + masm.movq(cArg0, previousValue); + AMD64Call.directCall(crb, masm, callTarget, null, false, null); + masm.jmp(done); + }); + } + + private static void verifyOop(AMD64MacroAssembler masm, Register previousValue, Register tmp, Register tmp2) { + ((AMD64HotSpotMacroAssembler) masm).verifyOop(previousValue, tmp, tmp2, false, true); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java index 59ceb0ace38d..6dd168149ed2 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java @@ -79,6 +79,7 @@ import static jdk.graal.compiler.hotspot.HotSpotBackend.VM_ERROR; import static jdk.graal.compiler.hotspot.HotSpotForeignCallLinkage.RegisterEffect.COMPUTES_REGISTERS_KILLED; import static jdk.graal.compiler.hotspot.HotSpotForeignCallLinkage.RegisterEffect.DESTROYS_ALL_CALLER_SAVE_REGISTERS; +import static jdk.graal.compiler.hotspot.HotSpotGraalRuntime.HotSpotGC.Shenandoah; import static jdk.graal.compiler.hotspot.HotSpotGraalRuntime.HotSpotGC.Z; import static jdk.graal.compiler.hotspot.HotSpotHostBackend.DEOPT_BLOB_UNCOMMON_TRAP; import static jdk.graal.compiler.hotspot.HotSpotHostBackend.DEOPT_BLOB_UNPACK; @@ -237,6 +238,35 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall public static final HotSpotForeignCallDescriptor Z_ARRAY_BARRIER = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, NO_LOCATIONS, "load_barrier_on_oop_array", void.class, long.class, long.class); + /** + * Shenandoah runtime function entries. + */ + + // oopDesc* ShenandoahRuntime::load_reference_barrier_strong(oopDesc* o, oop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_strong", Object.class, Object.class, Word.class); + // oopDesc* ShenandoahRuntime::load_reference_barrier_strong_narrow(oopDesc* o, narrowOop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER_NARROW = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_strong_narrow", Object.class, Object.class, Word.class); + + // oopDesc* ShenandoahRuntime::load_reference_barrier_weak(oopDesc* o, oop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER_WEAK = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_weak", Object.class, Object.class, Word.class); + // oopDesc* ShenandoahRuntime::load_reference_barrier_weak_narrow(oopDesc* o, narrowOop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER_WEAK_NARROW = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_weak_narrow", Object.class, Object.class, Word.class); + + // oopDesc* ShenandoahRuntime::load_reference_barrier_phantom(oopDesc* o, oop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER_PHANTOM = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_phantom", Object.class, Object.class, Word.class); + // oopDesc* ShenandoahRuntime::load_reference_barrier_phantom_narrow(oopDesc* o, narrowOop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER_PHANTOM_NARROW = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_phantom_narrow", Object.class, Object.class, Word.class); + + // void ShenandoahRuntime::write_barrier_pre(oopDesc*) + public static final HotSpotForeignCallDescriptor SHENANDOAH_WRITE_BARRIER_PRE = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, NO_SIDE_EFFECT, NO_LOCATIONS, + "ShenandoahRuntime::write_barrier_pre_stack_only", void.class, Object.class); + /** * Signature of an unsafe {@link System#arraycopy} stub. * @@ -547,6 +577,14 @@ public void initialize(HotSpotProviders providers, OptionValues options) { linkStackOnlyForeignCall(options, providers, G1WBPRECALL_STACK_ONLY, c.writeBarrierPreAddress, PREPEND_THREAD); linkStackOnlyForeignCall(options, providers, G1WBPOSTCALL_STACK_ONLY, c.writeBarrierPostAddress, PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER, c.shenandoahLoadBarrierStrong, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER_NARROW, c.shenandoahLoadBarrierStrongNarrow, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER_WEAK, c.shenandoahLoadBarrierWeak, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER_WEAK_NARROW, c.shenandoahLoadBarrierWeakNarrow, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER_PHANTOM, c.shenandoahLoadBarrierPhantom, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER_PHANTOM_NARROW, c.shenandoahLoadBarrierPhantomNarrow, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_WRITE_BARRIER_PRE, c.shenandoahWriteBarrierPre, DONT_PREPEND_THREAD); + linkForeignCall(options, providers, LOG_PRINTF, c.logPrintfAddress, PREPEND_THREAD); linkForeignCall(options, providers, LOG_OBJECT, c.logObjectAddress, PREPEND_THREAD); linkForeignCall(options, providers, LOG_PRIMITIVE, c.logPrimitiveAddress, PREPEND_THREAD); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/nodes/HotSpotCompressionNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/nodes/HotSpotCompressionNode.java index 5cf32565dbff..72ec01582775 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/nodes/HotSpotCompressionNode.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/nodes/HotSpotCompressionNode.java @@ -63,6 +63,14 @@ public static CompressionNode uncompress(StructuredGraph graph, ValueNode input, return graph.unique(uncompress(input, encoding)); } + public static HotSpotCompressionNode compressWithoutUnique(StructuredGraph graph, ValueNode input, CompressEncoding encoding) { + return graph.addWithoutUnique(compress(input, encoding)); + } + + public static CompressionNode uncompressWithoutUnique(StructuredGraph graph, ValueNode input, CompressEncoding encoding) { + return graph.addWithoutUnique(uncompress(input, encoding)); + } + private static HotSpotCompressionNode compress(ValueNode input, CompressEncoding encoding) { return new HotSpotCompressionNode(CompressionOp.Compress, input, encoding); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java index 7c90aceacccf..d6ff8f1acbe9 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java @@ -701,6 +701,36 @@ public static boolean useCondCardMark(@InjectedParameter GraalHotSpotVMConfig co return config.useCondCardMark; } + @Fold + public static int shenandoahGCStateOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahGCStateOffset; + } + + @Fold + public static int shenandoahSATBIndexOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahSATBIndexOffset; + } + + @Fold + public static int shenandoahSATBBufferOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahSATBBufferOffset; + } + + @Fold + public static int shenandoahCardTableOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahCardTableOffset; + } + + @Fold + public static int shenandoahGCRegionSizeBytesShift(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahGCRegionSizeBytesShift; + } + + @Fold + public static long shenandoahGCCSetFastTestAddr(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahGCCSetFastTestAddress; + } + public static final LocationIdentity KLASS_SUPER_CHECK_OFFSET_LOCATION = NamedLocationIdentity.immutable("Klass::_super_check_offset"); @Fold diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/aarch64/AArch64AtomicMove.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/aarch64/AArch64AtomicMove.java index b91576855f24..fe3520b732ba 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/aarch64/AArch64AtomicMove.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/aarch64/AArch64AtomicMove.java @@ -139,7 +139,7 @@ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { emitCompareAndSwap(masm, accessKind, address, result, expected, asRegister(newValue), memoryOrder, setConditionFlags); } - protected static void emitCompareAndSwap(AArch64MacroAssembler masm, AArch64Kind accessKind, Register address, Register result, Register expected, Register newValue, + public static void emitCompareAndSwap(AArch64MacroAssembler masm, AArch64Kind accessKind, Register address, Register result, Register expected, Register newValue, MemoryOrderMode memoryOrder, boolean setConditionFlags) { assert accessKind.isInteger(); final int memAccessSize = accessKind.getSizeInBytes() * Byte.SIZE; diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64Move.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64Move.java index 5309f24115a4..b685224c4aa0 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64Move.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64Move.java @@ -986,6 +986,10 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { final Register resReg = getResultRegister(); final Register baseReg = getBaseRegister(); + emitCompressCode(masm, resReg, getShift(), baseReg, nonNull); + } + + public static void emitCompressCode(AMD64MacroAssembler masm, Register resReg, int shift, Register baseReg, boolean nonNull) { if (!baseReg.equals(Register.None)) { if (!nonNull) { masm.testq(resReg, resReg); @@ -994,7 +998,6 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.subq(resReg, baseReg); } - int shift = getShift(); if (shift != 0) { masm.shrq(resReg, shift); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/ShenandoahBarrierSetLIRGeneratorTool.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/ShenandoahBarrierSetLIRGeneratorTool.java new file mode 100644 index 000000000000..257b3c85e024 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/ShenandoahBarrierSetLIRGeneratorTool.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.lir.gen; + +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadRefBarrierNode; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.Value; + +/** + * Lowers Shenandoah barriers to LIR. + */ +public interface ShenandoahBarrierSetLIRGeneratorTool extends BarrierSetLIRGeneratorTool { + Value emitLoadReferenceBarrier(LIRGeneratorTool tool, Value obj, Value address, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, boolean narrow, boolean notNull); + + void emitPreWriteBarrier(LIRGeneratorTool lirTool, Value address, AllocatableValue expectedObject, boolean nonNull); + + void emitCardBarrier(LIRGeneratorTool lirTool, Value address); +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java index a8e130b32b44..7f7a068bd89d 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java @@ -68,7 +68,8 @@ public final class GraphState { StageFlag.EXPAND_LOGIC, StageFlag.ADDRESS_LOWERING, StageFlag.REMOVE_OPAQUE_VALUES, - StageFlag.FINAL_SCHEDULE); + StageFlag.FINAL_SCHEDULE, + StageFlag.LOW_TIER_BARRIER_ADDITION); private static final EnumSet ENTERPRISE_MID_TIER_MANDATORY_STAGES = EnumSet.of( StageFlag.OPTIMISTIC_ALIASING, StageFlag.GUARD_LOWERING, @@ -630,6 +631,7 @@ public enum StageFlag { VECTOR_LOWERING, EXPAND_LOGIC, FIXED_READS, + LOW_TIER_BARRIER_ADDITION, PARTIAL_REDUNDANCY_SCHEDULE, ADDRESS_LOWERING, FINAL_CANONICALIZATION, diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/BarrierSet.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/BarrierSet.java index 86871a540585..4118cd123155 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/BarrierSet.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/BarrierSet.java @@ -25,6 +25,7 @@ */ package jdk.graal.compiler.nodes.gc; +import jdk.graal.compiler.nodes.GraphState; import org.graalvm.word.LocationIdentity; import jdk.graal.compiler.core.common.memory.BarrierType; @@ -88,4 +89,10 @@ default BarrierType writeBarrierType(LocationIdentity location) { */ default void verifyBarriers(StructuredGraph graph) { } + + default boolean shouldAddBarriersInStage(GraphState.StageFlag stage) { + // Most barrier sets should be added in mid-tier, some might also + // wish to add in low-tier (e.g. Shenandoah GC). + return stage == GraphState.StageFlag.BARRIER_ADDITION; + } } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahBarrierSet.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahBarrierSet.java new file mode 100644 index 000000000000..ddb537791e2c --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahBarrierSet.java @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.nodes.gc.shenandoah; + +import static jdk.graal.compiler.nodes.NamedLocationIdentity.OFF_HEAP_LOCATION; + +import jdk.graal.compiler.core.common.type.AbstractObjectStamp; +import jdk.graal.compiler.nodes.GraphState; +import jdk.graal.compiler.nodes.extended.ArrayRangeWrite; +import jdk.graal.compiler.nodes.gc.BarrierSet; +import jdk.graal.compiler.nodes.java.ValueCompareAndSwapNode; +import jdk.graal.compiler.nodes.spi.CoreProviders; +import jdk.graal.compiler.nodes.type.NarrowOopStamp; +import org.graalvm.word.LocationIdentity; + +import jdk.graal.compiler.core.common.memory.BarrierType; +import jdk.graal.compiler.core.common.type.Stamp; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.graph.Node; +import jdk.graal.compiler.nodes.FieldLocationIdentity; +import jdk.graal.compiler.nodes.NamedLocationIdentity; +import jdk.graal.compiler.nodes.NodeView; +import jdk.graal.compiler.nodes.StructuredGraph; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.extended.RawStoreNode; +import jdk.graal.compiler.nodes.java.AbstractCompareAndSwapNode; +import jdk.graal.compiler.nodes.java.LoweredAtomicReadAndWriteNode; +import jdk.graal.compiler.nodes.memory.AddressableMemoryAccess; +import jdk.graal.compiler.nodes.memory.FixedAccessNode; +import jdk.graal.compiler.nodes.memory.LIRLowerableAccess; +import jdk.graal.compiler.nodes.memory.ReadNode; +import jdk.graal.compiler.nodes.memory.WriteNode; +import jdk.graal.compiler.nodes.memory.address.AddressNode; +import jdk.graal.compiler.nodes.type.StampTool; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.ResolvedJavaField; +import jdk.vm.ci.meta.ResolvedJavaType; + +/** + * Shenandoah barrier set implementation. + * + * This generates 3 kinds of barriers: + * + *
    + *
  • Load-reference barriers after reference-loads. The purpose is to canonicalize references + * during concurrent collection, where we might otherwise see both from- and to-space references to + * the same object.
    • + * + *
  • SATB barriers before reference writes. Those support concurrent marking, similar to how this + * is done in G1. Reference.get-barriers are a special form of this, to support (weak,soft,phantom-) + * references.
    • + * + *
  • Card barriers, only needed for generational Shenandoah. Those are inserted after + * reference-writes and dirty cards. Similar to their counterparts in Serial and Parallel GC.
    • + *
+ */ +public class ShenandoahBarrierSet implements BarrierSet { + + private final ResolvedJavaType objectArrayType; + private final ResolvedJavaField referentField; + protected boolean useLoadRefBarrier; + protected boolean useSATBBarrier; + protected boolean useCASBarrier; + protected boolean useCardBarrier; + + public ShenandoahBarrierSet(ResolvedJavaType objectArrayType, ResolvedJavaField referentField) { + this.referentField = referentField; + this.objectArrayType = objectArrayType; + this.useLoadRefBarrier = true; + this.useSATBBarrier = true; + this.useCASBarrier = true; + this.useCardBarrier = true; + } + + @Override + public BarrierType postAllocationInitBarrier(BarrierType original) { + assert original == BarrierType.FIELD || original == BarrierType.ARRAY : "only for write barriers: " + original; + return BarrierType.POST_INIT_WRITE; + } + + @Override + public BarrierType readBarrierType(LocationIdentity location, ValueNode address, Stamp loadStamp) { + if (location.equals(OFF_HEAP_LOCATION)) { + // Off heap locations are never expected to contain objects + GraalError.guarantee(!loadStamp.isObjectStamp(), "off-heap location not expected to be object: " + location); + return BarrierType.NONE; + } + + if (loadStamp.isObjectStamp()) { + if (address.stamp(NodeView.DEFAULT).isObjectStamp()) { + // A read of an Object from an Object requires a barrier + return BarrierType.READ; + } + + if (address instanceof AddressNode addr) { + if (addr.getBase().stamp(NodeView.DEFAULT).isObjectStamp()) { + // A read of an Object from an Object requires a barrier + return BarrierType.READ; + } + } + // Objects aren't expected to be read from non-heap locations. + throw GraalError.shouldNotReachHere("Unexpected location type " + loadStamp); + } + + GraalError.guarantee(!(location instanceof FieldLocationIdentity fieldLocationIdentity) || fieldLocationIdentity.getField().getJavaKind() != JavaKind.Object, + "must not be a reference location: " + address); + return BarrierType.NONE; + } + + @Override + public BarrierType writeBarrierType(RawStoreNode store) { + if (store.object().isNullConstant()) { + return BarrierType.NONE; + } + return store.needsBarrier() ? readWriteBarrier(store.object(), store.value()) : BarrierType.NONE; + } + + @Override + public BarrierType fieldReadBarrierType(ResolvedJavaField field, JavaKind storageKind) { + if (field.getJavaKind() == JavaKind.Object && field.equals(referentField)) { + return BarrierType.REFERENCE_GET; + } + if (storageKind.isObject()) { + return BarrierType.READ; + } + return BarrierType.NONE; + } + + @Override + public BarrierType fieldWriteBarrierType(ResolvedJavaField field, JavaKind storageKind) { + return storageKind == JavaKind.Object ? BarrierType.FIELD : BarrierType.NONE; + } + + @Override + public BarrierType arrayWriteBarrierType(JavaKind storageKind) { + return storageKind == JavaKind.Object ? BarrierType.ARRAY : BarrierType.NONE; + } + + @Override + public BarrierType readWriteBarrier(ValueNode object, ValueNode value) { + if (value.stamp(NodeView.DEFAULT).isObjectStamp()) { + ResolvedJavaType type = StampTool.typeOrNull(object); + if (type != null && type.isArray()) { + return BarrierType.ARRAY; + } else if (type == null || type.isAssignableFrom(objectArrayType)) { + return BarrierType.ARRAY; + } else { + return BarrierType.FIELD; + } + } + return BarrierType.NONE; + } + + @Override + public boolean hasWriteBarrier() { + return true; + } + + @Override + public boolean hasReadBarrier() { + return true; + } + + @Override + public void addBarriers(FixedAccessNode n, CoreProviders context) { + switch (n) { + case ReadNode readNode -> addReadNodeBarriers(readNode); + case WriteNode write -> addWriteBarriers(write, write.value(), null); + case LoweredAtomicReadAndWriteNode atomic -> { + if (useCASBarrier) { + addWriteBarriers(atomic, atomic.getNewValue(), null); + addReadNodeBarriers(atomic); + } + } + case AbstractCompareAndSwapNode cmpSwap -> { + if (useCASBarrier) { + addWriteBarriers(cmpSwap, cmpSwap.getNewValue(), cmpSwap.getExpectedValue()); + if (cmpSwap instanceof ValueCompareAndSwapNode) { + addReadNodeBarriers(cmpSwap); + } + } + } + case ArrayRangeWrite ignored -> GraalError.unimplemented("ArrayRangeWrite is not used"); + case null, default -> + GraalError.guarantee(n.getBarrierType() == BarrierType.NONE, "missed a node that requires a GC barrier: %s", n.getClass()); + } + } + + private void addWriteBarriers(FixedAccessNode node, ValueNode writtenValue, ValueNode expectedValue) { + BarrierType barrierType = node.getBarrierType(); + switch (barrierType) { + case NONE: + // nothing to do + break; + case FIELD: + case ARRAY: + case UNKNOWN: + case POST_INIT_WRITE: + case AS_NO_KEEPALIVE_WRITE: + if (isObjectValue(writtenValue)) { + StructuredGraph graph = node.graph(); + boolean init = node.getLocationIdentity().isInit(); + if (!init && barrierType != BarrierType.AS_NO_KEEPALIVE_WRITE && useSATBBarrier) { + // The pre barrier does nothing if the value being read is null, so it can + // be explicitly skipped when this is an initializing store. + // No keep-alive means no need for the pre-barrier. + addShenandoahSATBBarrier(node, node.getAddress(), expectedValue, graph); + } + if (!init && useCardBarrier && !StampTool.isPointerAlwaysNull(writtenValue)) { + graph.addAfterFixed(node, graph.add(new ShenandoahCardBarrierNode(node.getAddress()))); + } + } + break; + default: + throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + private void addLoadReferenceBarrier(FixedAccessNode node, AddressNode address, BarrierType barrierType) { + GraalError.guarantee(node != null, "input value must not be null"); + StructuredGraph graph = node.graph(); + boolean narrow = node.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp; + ValueNode uncompressed = maybeUncompressReference(node, narrow); + ShenandoahLoadRefBarrierNode lrb = graph.add(new ShenandoahLoadRefBarrierNode(uncompressed, address, barrierType, narrow)); + ValueNode compValue = maybeCompressReference(lrb, narrow); + ValueNode newUsage = uncompressed != node ? uncompressed : lrb; + node.replaceAtUsages(compValue, usage -> usage != newUsage); + } + + private void addReadNodeBarriers(FixedAccessNode node) { + + BarrierType barrierType = node.getBarrierType(); + StructuredGraph graph = node.graph(); + switch (barrierType) { + case NONE -> { + // No barriers required. + } + case REFERENCE_GET -> { + if (useLoadRefBarrier) { + addLoadReferenceBarrier(node, node.getAddress(), barrierType); + } + if (useSATBBarrier) { + boolean narrow = node.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp; + ShenandoahReferentFieldReadBarrierNode barrier = graph.add(new ShenandoahReferentFieldReadBarrierNode(node.getAddress(), maybeUncompressReference(node, narrow))); + graph.addAfterFixed(node, barrier); + } + } + case WEAK_REFERS_TO, PHANTOM_REFERS_TO, READ, ARRAY, FIELD, UNKNOWN -> { + if (useLoadRefBarrier) { + addLoadReferenceBarrier(node, node.getAddress(), barrierType); + } + } + default -> throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + protected ValueNode maybeUncompressReference(ValueNode value, @SuppressWarnings("unused") boolean narrow) { + return value; + } + + protected ValueNode maybeCompressReference(ValueNode value, @SuppressWarnings("unused") boolean narrow) { + return value; + } + + private void addShenandoahSATBBarrier(FixedAccessNode node, AddressNode address, ValueNode value, StructuredGraph graph) { + boolean narrow = value != null && value.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp; + ShenandoahSATBBarrierNode preBarrier = graph.add(new ShenandoahSATBBarrierNode(address, maybeUncompressReference(value, narrow))); + GraalError.guarantee(!node.getUsedAsNullCheck(), "trapping null checks are inserted after write barrier insertion: ", node); + node.setStateBefore(null); + graph.addBeforeFixed(node, preBarrier); + } + + private static boolean isObjectValue(ValueNode value) { + return value.stamp(NodeView.DEFAULT) instanceof AbstractObjectStamp; + } + + @Override + public boolean mayNeedPreWriteBarrier(JavaKind storageKind) { + return false; + } + + @Override + public void verifyBarriers(StructuredGraph graph) { + for (Node node : graph.getNodes()) { + if (node instanceof WriteNode write) { + Stamp stamp = write.getAccessStamp(NodeView.DEFAULT); + if (!stamp.isObjectStamp()) { + GraalError.guarantee(write.getBarrierType() == BarrierType.NONE, "no barriers for primitive writes: %s", write); + } + } else if (node instanceof ReadNode || + node instanceof AbstractCompareAndSwapNode || + node instanceof LoweredAtomicReadAndWriteNode) { + LIRLowerableAccess read = (LIRLowerableAccess) node; + Stamp stamp = read.getAccessStamp(NodeView.DEFAULT); + if (!stamp.isObjectStamp()) { + GraalError.guarantee(read.getBarrierType() == BarrierType.NONE, "no barriers for primitive reads: %s", read); + continue; + } + + BarrierType expectedBarrier = barrierForLocation(read.getBarrierType(), read.getLocationIdentity(), JavaKind.Object); + if (expectedBarrier != null) { + GraalError.guarantee(expectedBarrier == read.getBarrierType(), "expected %s but found %s in %s", expectedBarrier, read.getBarrierType(), read); + continue; + } + + ValueNode base = read.getAddress().getBase(); + if (!base.stamp(NodeView.DEFAULT).isObjectStamp()) { + GraalError.guarantee(read.getBarrierType() == BarrierType.NONE, "no barrier for non-heap read: %s", read); + } else { + GraalError.guarantee(read.getBarrierType() == BarrierType.READ, "missing barriers for heap read: %s", read); + } + } else if (node instanceof AddressableMemoryAccess access) { + if (access.getBarrierType() != BarrierType.NONE) { + throw new GraalError("Unexpected memory access with barrier : " + node); + } + } + } + } + + protected BarrierType barrierForLocation(BarrierType currentBarrier, LocationIdentity location, JavaKind storageKind) { + if (location instanceof FieldLocationIdentity fieldLocationIdentity) { + BarrierType barrierType = fieldReadBarrierType(fieldLocationIdentity.getField(), storageKind); + if (barrierType != currentBarrier && barrierType == BarrierType.REFERENCE_GET) { + if (currentBarrier == BarrierType.WEAK_REFERS_TO || currentBarrier == BarrierType.PHANTOM_REFERS_TO) { + return currentBarrier; + } + } + return barrierType; + } + if (location.equals(NamedLocationIdentity.getArrayLocation(JavaKind.Object))) { + return BarrierType.READ; + } + return null; + } + + @Override + public boolean shouldAddBarriersInStage(GraphState.StageFlag stage) { + return stage == GraphState.StageFlag.LOW_TIER_BARRIER_ADDITION; + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahCardBarrierNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahCardBarrierNode.java new file mode 100644 index 000000000000..94a99444be58 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahCardBarrierNode.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.nodes.gc.shenandoah; + +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodes.gc.ObjectWriteBarrierNode; +import jdk.graal.compiler.nodes.memory.address.AddressNode; +import jdk.graal.compiler.nodes.spi.LIRLowerable; +import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; + +import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_8; +import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_4; + +/** + * Shenandoah card barriers. Those are added after reference-writes and serve to dirty cards in the + * card-table. Only needed for generational Shenandoah. + */ +@NodeInfo(cycles = CYCLES_8, size = SIZE_4) +public class ShenandoahCardBarrierNode extends ObjectWriteBarrierNode implements LIRLowerable { + public static final NodeClass TYPE = NodeClass.create(ShenandoahCardBarrierNode.class); + + public ShenandoahCardBarrierNode(AddressNode address) { + super(TYPE, address, null, false); + } + + @Override + public Kind getKind() { + return Kind.POST_BARRIER; + } + + @Override + public void generate(NodeLIRBuilderTool gen) { + ShenandoahBarrierSetLIRGeneratorTool tool = (ShenandoahBarrierSetLIRGeneratorTool) gen.getLIRGeneratorTool().getBarrierSet(); + tool.emitCardBarrier(gen.getLIRGeneratorTool(), gen.operand(getAddress())); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahLoadRefBarrierNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahLoadRefBarrierNode.java new file mode 100644 index 000000000000..f943594cc8ae --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahLoadRefBarrierNode.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.nodes.gc.shenandoah; + +import jdk.graal.compiler.core.common.memory.BarrierType; +import jdk.graal.compiler.core.common.type.AbstractObjectStamp; +import jdk.graal.compiler.core.common.type.Stamp; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodeinfo.InputType; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodes.NodeView; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.memory.LIRLowerableAccess; +import jdk.graal.compiler.nodes.memory.address.AddressNode; +import jdk.graal.compiler.nodes.spi.LIRLowerable; +import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; + +import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_64; + +/** + * Shenandoah load-reference barriers. Those are added after reference-loads, and are used to + * canonicalize references during concurrent evacuation. During concurrent evacuation we might see + * both from- and to-space references to the same objects, and this barrier ensures that we only see + * to-space references. (a.k.a. To-space invariant). + */ +@NodeInfo(cycles = CYCLES_64, size = SIZE_64) +public final class ShenandoahLoadRefBarrierNode extends ValueNode implements LIRLowerable { + public static final NodeClass TYPE = NodeClass.create(ShenandoahLoadRefBarrierNode.class); + + /** + * Strength of the input reference, determines generated code and slow-path call. + */ + public enum ReferenceStrength { + STRONG, + WEAK, + PHANTOM; + } + + /** + * The input value. Typically this is a reference that has just been loaded. The barrier output + * represents the canonicalized reference. + */ + @Input private ValueNode value; + + /** + * The address from which the input value has been loaded, if any/known. + */ + @Input(InputType.Association) private AddressNode address; + + /** + * The strength of the loaded reference. + */ + private final ReferenceStrength strength; + + /** + * Whether the reference is compressed. + */ + private final boolean narrow; + + private static ReferenceStrength getReferenceStrength(BarrierType barrierType) { + return switch (barrierType) { + case READ, FIELD, ARRAY, NONE -> ReferenceStrength.STRONG; + case REFERENCE_GET, WEAK_REFERS_TO -> ReferenceStrength.WEAK; + case PHANTOM_REFERS_TO -> ReferenceStrength.PHANTOM; + case UNKNOWN, POST_INIT_WRITE, AS_NO_KEEPALIVE_WRITE -> throw GraalError.shouldNotReachHere("Unexpected barrier type: " + barrierType); + }; + } + + public ShenandoahLoadRefBarrierNode(ValueNode value, AddressNode address, BarrierType barrierType, boolean narrow) { + super(TYPE, value.stamp(NodeView.DEFAULT)); + this.value = value; + this.address = address; + this.strength = getReferenceStrength(barrierType); + this.narrow = narrow; + } + + @Override + public void generate(NodeLIRBuilderTool gen) { + Stamp valueStamp; + if (value instanceof LIRLowerableAccess accessValue) { + valueStamp = accessValue.getAccessStamp(NodeView.DEFAULT); + } else { + valueStamp = value.stamp(NodeView.DEFAULT); + } + GraalError.guarantee(valueStamp.isObjectStamp(), "LRB value must be object"); + boolean notNull = ((AbstractObjectStamp) valueStamp).nonNull(); + ShenandoahBarrierSetLIRGeneratorTool tool = (ShenandoahBarrierSetLIRGeneratorTool) gen.getLIRGeneratorTool().getBarrierSet(); + gen.setResult(this, tool.emitLoadReferenceBarrier(gen.getLIRGeneratorTool(), gen.operand(value), gen.operand(address), strength, narrow, notNull)); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahReferentFieldReadBarrierNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahReferentFieldReadBarrierNode.java new file mode 100644 index 000000000000..7504d489e154 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahReferentFieldReadBarrierNode.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.nodes.gc.shenandoah; + +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.lir.gen.LIRGeneratorTool; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.gc.ObjectWriteBarrierNode; +import jdk.graal.compiler.nodes.memory.address.AddressNode; +import jdk.graal.compiler.nodes.spi.LIRLowerable; +import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; + +import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_64; + +/** + * A special case of the SATB barrier, needed to support soft and weak references. They are added + * after reads of referents of SoftReference and WeakReference objects, and ensure that such + * referents are marked live during concurrent marking. + */ +@NodeInfo(cycles = CYCLES_64, size = SIZE_64) +public class ShenandoahReferentFieldReadBarrierNode extends ObjectWriteBarrierNode implements LIRLowerable { + public static final NodeClass TYPE = NodeClass.create(ShenandoahReferentFieldReadBarrierNode.class); + + public ShenandoahReferentFieldReadBarrierNode(AddressNode address, ValueNode expectedObject) { + super(TYPE, address, expectedObject, true); + } + + public ValueNode getExpectedObject() { + return getValue(); + } + + @Override + public Kind getKind() { + return Kind.PRE_BARRIER; + } + + @Override + public void generate(NodeLIRBuilderTool generator) { + LIRGeneratorTool lirGen = generator.getLIRGeneratorTool(); + ShenandoahBarrierSetLIRGeneratorTool tool = (ShenandoahBarrierSetLIRGeneratorTool) generator.getLIRGeneratorTool().getBarrierSet(); + tool.emitPreWriteBarrier(lirGen, generator.operand(getAddress()), lirGen.asAllocatable(generator.operand(getExpectedObject())), false); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahSATBBarrierNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahSATBBarrierNode.java new file mode 100644 index 000000000000..9aab7bc99c57 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahSATBBarrierNode.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.nodes.gc.shenandoah; + +import jdk.graal.compiler.core.common.type.ObjectStamp; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.lir.gen.LIRGeneratorTool; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodes.NodeView; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.gc.ObjectWriteBarrierNode; +import jdk.graal.compiler.nodes.memory.address.AddressNode; +import jdk.graal.compiler.nodes.spi.LIRLowerable; +import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.Value; + +import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_64; + +/** + * Shenandoah SATB barrier. Supports concurrent marking, by implementing the so-called + * snapshot-at-the-beginning (SATB). The barrier ensures that we see a consistent and complete + * marking bitmap after concurrent marking, that has at least all objects marked live that have been + * live at the beginning of marking (hence the name). This barrier is very similar to G1's + * pre-write-barrier. + */ +@NodeInfo(cycles = CYCLES_64, size = SIZE_64) +public final class ShenandoahSATBBarrierNode extends ObjectWriteBarrierNode implements LIRLowerable { + public static final NodeClass TYPE = NodeClass.create(ShenandoahSATBBarrierNode.class); + + public ShenandoahSATBBarrierNode(AddressNode address, ValueNode expectedObject) { + super(TYPE, address, expectedObject, true); + } + + public ValueNode getExpectedObject() { + return getValue(); + } + + @Override + public Kind getKind() { + return Kind.PRE_BARRIER; + } + + @Override + public void generate(NodeLIRBuilderTool generator) { + ValueNode expectedObject = getExpectedObject(); + if (expectedObject == null || !expectedObject.isJavaConstant() || !expectedObject.asJavaConstant().isNull()) { + AllocatableValue operand = Value.ILLEGAL; + boolean nonNull = false; + LIRGeneratorTool lirGen = generator.getLIRGeneratorTool(); + if (expectedObject != null) { + operand = lirGen.asAllocatable(generator.operand(expectedObject)); + nonNull = ((ObjectStamp) expectedObject.stamp(NodeView.DEFAULT)).nonNull(); + GraalError.guarantee(expectedObject.stamp(NodeView.DEFAULT) instanceof ObjectStamp, "expecting full size object"); + } + ShenandoahBarrierSetLIRGeneratorTool tool = (ShenandoahBarrierSetLIRGeneratorTool) generator.getLIRGeneratorTool().getBarrierSet(); + tool.emitPreWriteBarrier(lirGen, generator.operand(getAddress()), operand, nonNull); + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/WriteBarrierAdditionPhase.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/WriteBarrierAdditionPhase.java index 374224ea0672..1e9b49bf4f07 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/WriteBarrierAdditionPhase.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/WriteBarrierAdditionPhase.java @@ -37,11 +37,21 @@ public class WriteBarrierAdditionPhase extends BasePhase { + private final StageFlag stage; + + public WriteBarrierAdditionPhase() { + this(StageFlag.BARRIER_ADDITION); + } + + public WriteBarrierAdditionPhase(StageFlag stage) { + this.stage = stage; + } + @Override public Optional notApplicableTo(GraphState graphState) { return NotApplicable.ifAny( - NotApplicable.ifApplied(this, StageFlag.BARRIER_ADDITION, graphState), - NotApplicable.unlessRunAfter(this, StageFlag.MID_TIER_LOWERING, graphState), + NotApplicable.ifApplied(this, stage, graphState), + NotApplicable.unlessRunAfter(this, stage == StageFlag.BARRIER_ADDITION ? StageFlag.MID_TIER_LOWERING : StageFlag.LOW_TIER_LOWERING, graphState), NotApplicable.unlessRunAfter(this, StageFlag.FSA, graphState)); } @@ -49,7 +59,7 @@ public Optional notApplicableTo(GraphState graphState) { @Override protected void run(StructuredGraph graph, CoreProviders context) { BarrierSet barrierSet = context.getPlatformConfigurationProvider().getBarrierSet(); - if (barrierSet.hasWriteBarrier()) { + if (barrierSet.hasWriteBarrier() && barrierSet.shouldAddBarriersInStage(stage)) { for (FixedAccessNode n : graph.getNodes(FixedAccessNode.TYPE)) { try (DebugCloseable scope = n.graph().withNodeSourcePosition(n)) { barrierSet.addBarriers(n, context); @@ -61,7 +71,7 @@ protected void run(StructuredGraph graph, CoreProviders context) { @Override public void updateGraphState(GraphState graphState) { super.updateGraphState(graphState); - graphState.setAfterStage(StageFlag.BARRIER_ADDITION); + graphState.setAfterStage(stage); } @Override diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/TruffleEntryPointDecorator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/TruffleEntryPointDecorator.java index e6b31813498d..d2a9fa28358e 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/TruffleEntryPointDecorator.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/TruffleEntryPointDecorator.java @@ -24,6 +24,7 @@ */ package jdk.graal.compiler.truffle.hotspot; +import static jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER; import static jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider.Z_LOAD_BARRIER; import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; @@ -65,5 +66,9 @@ public void initialize(CoreProviders providers, LIRGenerationResult lirGenRes) { ForeignCallLinkage callTarget = providers.getForeignCalls().lookupForeignCall(Z_LOAD_BARRIER); lirGenRes.getFrameMapBuilder().callsMethod(callTarget.getOutgoingCallingConvention()); } + if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + ForeignCallLinkage callTarget = providers.getForeignCalls().lookupForeignCall(SHENANDOAH_LOAD_BARRIER); + lirGenRes.getFrameMapBuilder().callsMethod(callTarget.getOutgoingCallingConvention()); + } } } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/aarch64/AArch64TruffleCallBoundaryInstrumentationFactory.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/aarch64/AArch64TruffleCallBoundaryInstrumentationFactory.java index cedf561593f1..bfaaf8e8e596 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/aarch64/AArch64TruffleCallBoundaryInstrumentationFactory.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/aarch64/AArch64TruffleCallBoundaryInstrumentationFactory.java @@ -24,6 +24,7 @@ */ package jdk.graal.compiler.truffle.hotspot.aarch64; +import static jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER; import static jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider.Z_LOAD_BARRIER; import static jdk.vm.ci.hotspot.HotSpotCallingConventionType.JavaCall; import static jdk.vm.ci.meta.JavaKind.Object; @@ -38,10 +39,12 @@ import jdk.graal.compiler.hotspot.HotSpotGraalRuntime; import jdk.graal.compiler.hotspot.aarch64.AArch64HotSpotBackend; import jdk.graal.compiler.hotspot.aarch64.AArch64HotSpotMove; +import jdk.graal.compiler.hotspot.aarch64.shenandoah.AArch64HotSpotShenandoahLoadRefBarrierOp; import jdk.graal.compiler.hotspot.aarch64.z.AArch64HotSpotZBarrierSetLIRGenerator; import jdk.graal.compiler.hotspot.meta.HotSpotRegistersProvider; import jdk.graal.compiler.lir.asm.CompilationResultBuilder; import jdk.graal.compiler.lir.asm.EntryPointDecorator; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadRefBarrierNode; import jdk.graal.compiler.serviceprovider.ServiceProvider; import jdk.graal.compiler.truffle.TruffleCompilerConfiguration; import jdk.graal.compiler.truffle.hotspot.TruffleCallBoundaryInstrumentationFactory; @@ -56,7 +59,7 @@ public EntryPointDecorator create(TruffleCompilerConfiguration compilerConfig, G return new TruffleEntryPointDecorator(compilerConfig, config, registers) { @Override public void emitEntryPoint(CompilationResultBuilder crb, boolean beforeFrameSetup) { - if (beforeFrameSetup == (config.gc == HotSpotGraalRuntime.HotSpotGC.Z)) { + if (beforeFrameSetup == (config.gc == HotSpotGraalRuntime.HotSpotGC.Z || config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah)) { // The Z load barrier must be performed after the nmethod entry barrier which is // part of the frame setup. The other GCs don't have a read barrier so it's // safe to do this dispatch before the frame is set up. @@ -72,9 +75,16 @@ public void emitEntryPoint(CompilationResultBuilder crb, boolean beforeFrameSetu Label doProlog = new Label(); if (config.useCompressedOops) { CompressEncoding encoding = config.getOopEncoding(); - masm.ldr(32, spillRegister, AArch64Address.createImmediateAddress(32, AArch64Address.AddressingMode.IMMEDIATE_UNSIGNED_SCALED, thisRegister, installedCodeOffset)); + AArch64Address address = AArch64Address.createImmediateAddress(32, AArch64Address.AddressingMode.IMMEDIATE_UNSIGNED_SCALED, thisRegister, installedCodeOffset); + masm.ldr(32, spillRegister, address); Register base = encoding.hasBase() ? registers.getHeapBaseRegister() : null; AArch64HotSpotMove.UncompressPointer.emitUncompressCode(masm, spillRegister, spillRegister, base, encoding, true); + if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + Register thread = registers.getThreadRegister(); + ForeignCallLinkage callTarget = crb.getForeignCalls().lookupForeignCall(SHENANDOAH_LOAD_BARRIER); + AArch64HotSpotShenandoahLoadRefBarrierOp.emitCode(config, crb, masm, null, thread, spillRegister, spillRegister, address, callTarget, + ShenandoahLoadRefBarrierNode.ReferenceStrength.STRONG, false); + } } else { AArch64Address address = AArch64Address.createImmediateAddress(64, AArch64Address.AddressingMode.IMMEDIATE_UNSIGNED_SCALED, thisRegister, installedCodeOffset); masm.ldr(64, spillRegister, address); @@ -82,6 +92,12 @@ public void emitEntryPoint(CompilationResultBuilder crb, boolean beforeFrameSetu ForeignCallLinkage callTarget = crb.getForeignCalls().lookupForeignCall(Z_LOAD_BARRIER); AArch64HotSpotZBarrierSetLIRGenerator.emitLoadBarrier(crb, masm, config, spillRegister, callTarget, address, null, false, false); } + if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + Register thread = registers.getThreadRegister(); + ForeignCallLinkage callTarget = crb.getForeignCalls().lookupForeignCall(SHENANDOAH_LOAD_BARRIER); + AArch64HotSpotShenandoahLoadRefBarrierOp.emitCode(config, crb, masm, null, thread, spillRegister, spillRegister, address, callTarget, + ShenandoahLoadRefBarrierNode.ReferenceStrength.STRONG, false); + } } masm.ldr(64, spillRegister, AArch64Address.createImmediateAddress(64, AArch64Address.AddressingMode.IMMEDIATE_UNSIGNED_SCALED, spillRegister, entryPointOffset)); masm.cbz(64, spillRegister, doProlog); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/amd64/AMD64TruffleCallBoundaryInstrumentationFactory.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/amd64/AMD64TruffleCallBoundaryInstrumentationFactory.java index e45fcbd170ee..ae2313cb565c 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/amd64/AMD64TruffleCallBoundaryInstrumentationFactory.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/amd64/AMD64TruffleCallBoundaryInstrumentationFactory.java @@ -24,6 +24,7 @@ */ package jdk.graal.compiler.truffle.hotspot.amd64; +import static jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER; import static jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider.Z_LOAD_BARRIER; import static jdk.vm.ci.hotspot.HotSpotCallingConventionType.JavaCall; @@ -36,11 +37,13 @@ import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; import jdk.graal.compiler.hotspot.HotSpotGraalRuntime; import jdk.graal.compiler.hotspot.amd64.AMD64HotSpotBackend; +import jdk.graal.compiler.hotspot.amd64.shenandoah.AMD64HotSpotShenandoahLoadRefBarrierOp; import jdk.graal.compiler.hotspot.amd64.z.AMD64HotSpotZBarrierSetLIRGenerator; import jdk.graal.compiler.hotspot.meta.HotSpotRegistersProvider; import jdk.graal.compiler.lir.amd64.AMD64Move; import jdk.graal.compiler.lir.asm.CompilationResultBuilder; import jdk.graal.compiler.lir.asm.EntryPointDecorator; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadRefBarrierNode; import jdk.graal.compiler.serviceprovider.ServiceProvider; import jdk.graal.compiler.truffle.TruffleCompilerConfiguration; import jdk.graal.compiler.truffle.hotspot.TruffleCallBoundaryInstrumentationFactory; @@ -78,6 +81,14 @@ public void emitEntryPoint(CompilationResultBuilder crb, boolean beforeFrameSetu CompressEncoding encoding = config.getOopEncoding(); Register heapBaseRegister = AMD64Move.UncompressPointerOp.hasBase(encoding) ? registers.getHeapBaseRegister() : Register.None; AMD64Move.UncompressPointerOp.emitUncompressCode(masm, spillRegister, encoding.getShift(), heapBaseRegister, true); + if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + Register thread = registers.getThreadRegister(); + ForeignCallLinkage callTarget = crb.getForeignCalls().lookupForeignCall(SHENANDOAH_LOAD_BARRIER); + Register tmp1 = AMD64.r9; // TODO: Can we use this? Should be caller-saved. + Register tmp2 = AMD64.r11; // TODO: Can we use this? Should be caller-saved. + AMD64HotSpotShenandoahLoadRefBarrierOp.emitCode(config, crb, masm, null, thread, spillRegister, spillRegister, tmp1, tmp2, address, callTarget, + ShenandoahLoadRefBarrierNode.ReferenceStrength.STRONG, false); + } } else { // First instruction must be at least 5 bytes long to be safe for // patching @@ -88,6 +99,14 @@ public void emitEntryPoint(CompilationResultBuilder crb, boolean beforeFrameSetu AMD64HotSpotZBarrierSetLIRGenerator.emitLoadBarrier(crb, masm, spillRegister, callTarget, address, null, false); } + if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + Register thread = registers.getThreadRegister(); + ForeignCallLinkage callTarget = crb.getForeignCalls().lookupForeignCall(SHENANDOAH_LOAD_BARRIER); + Register tmp1 = AMD64.r9; // TODO: Can we use this? Should be caller-saved. + Register tmp2 = AMD64.r11; // TODO: Can we use this? Should be caller-saved. + AMD64HotSpotShenandoahLoadRefBarrierOp.emitCode(config, crb, masm, null, thread, spillRegister, spillRegister, tmp1, tmp2, address, callTarget, + ShenandoahLoadRefBarrierNode.ReferenceStrength.STRONG, false); + } } masm.movq(spillRegister, new AMD64Address(spillRegister, entryPointOffset)); masm.testqAndJcc(spillRegister, spillRegister, ConditionFlag.Equal, doProlog, true);