diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractImageHeapLayouter.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractImageHeapLayouter.java deleted file mode 100644 index f5ead5771f61..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractImageHeapLayouter.java +++ /dev/null @@ -1,274 +0,0 @@ -/* - * Copyright (c) 2019, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.svm.core.genscavenge; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import com.oracle.svm.core.config.ConfigurationValues; -import com.oracle.svm.core.hub.DynamicHub; -import com.oracle.svm.core.image.ImageHeap; -import com.oracle.svm.core.image.ImageHeapLayoutInfo; -import com.oracle.svm.core.image.ImageHeapLayouter; -import com.oracle.svm.core.image.ImageHeapObject; -import com.oracle.svm.core.image.ImageHeapPartition; -import com.oracle.svm.core.util.VMError; - -public abstract class AbstractImageHeapLayouter implements ImageHeapLayouter { - /** A partition holding objects with only read-only primitive values, but no references. */ - private static final int READ_ONLY_PRIMITIVE = 0; - /** A partition holding objects with read-only references and primitive values. */ - private static final int READ_ONLY_REFERENCE = 1; - /** - * A pseudo-partition used during image building to consolidate objects that contain relocatable - * references. - *

- * Collecting the relocations together means the dynamic linker has to operate on less of the - * image heap during image startup, and it means that less of the image heap has to be - * copied-on-write if the image heap is relocated in a new process. - *

- * A relocated reference is read-only once relocated, e.g., at runtime. The read-only relocation - * partition does not exist as a separate partition in the generated image. Instead, the - * read-only reference partition is resized to include the read-only relocation partition as - * well. - */ - private static final int READ_ONLY_RELOCATABLE = 2; - /** A partition holding objects with writable primitive values, but no references. */ - private static final int WRITABLE_PRIMITIVE = 3; - /** A partition holding objects with writable references and primitive values. */ - private static final int WRITABLE_REFERENCE = 4; - /** A partition holding very large writable objects with or without references. */ - private static final int WRITABLE_HUGE = 5; - /** - * A partition holding very large read-only objects with or without references, but never with - * relocatable references. - */ - private static final int READ_ONLY_HUGE = 6; - - private static final int PARTITION_COUNT = 7; - - private final T[] partitions; - - @Override - public T[] getPartitions() { - return partitions; - } - - private T getLastPartition() { - return getPartitions()[PARTITION_COUNT - 1]; - } - - @SuppressWarnings("this-escape") - public AbstractImageHeapLayouter() { - this.partitions = createPartitionsArray(PARTITION_COUNT); - this.partitions[READ_ONLY_PRIMITIVE] = createPartition("readOnlyPrimitive", false, false, false); - this.partitions[READ_ONLY_REFERENCE] = createPartition("readOnlyReference", true, false, false); - this.partitions[READ_ONLY_RELOCATABLE] = createPartition("readOnlyRelocatable", true, false, false); - this.partitions[WRITABLE_PRIMITIVE] = createPartition("writablePrimitive", false, true, false); - this.partitions[WRITABLE_REFERENCE] = createPartition("writableReference", true, true, false); - this.partitions[WRITABLE_HUGE] = createPartition("writableHuge", true, true, true); - this.partitions[READ_ONLY_HUGE] = createPartition("readOnlyHuge", true, false, true); - } - - @Override - public void assignObjectToPartition(ImageHeapObject info, boolean immutable, boolean references, boolean relocatable) { - T partition = choosePartition(info, immutable, references, relocatable); - info.setHeapPartition(partition); - partition.assign(info); - } - - @Override - public ImageHeapLayoutInfo layout(ImageHeap imageHeap, int pageSize) { - int objectAlignment = ConfigurationValues.getObjectLayout().getAlignment(); - assert pageSize % objectAlignment == 0 : "Page size does not match object alignment"; - - for (T partition : getPartitions()) { - int startAlignment = objectAlignment; - int endAlignment = objectAlignment; - if (partition == getReadOnlyRelocatable()) { - startAlignment = pageSize; - endAlignment = pageSize; - } else if (partition == getWritablePrimitive()) { - startAlignment = pageSize; - } else if (partition == getWritableHuge()) { - endAlignment = pageSize; - } - - /* Make sure the image heap size is a multiple of the page size. */ - if (partition == getLastPartition()) { - endAlignment = pageSize; - } - - partition.setStartAlignment(startAlignment); - partition.setEndAlignment(endAlignment); - } - - ImageHeapLayoutInfo layoutInfo = doLayout(imageHeap); - - for (T partition : getPartitions()) { - assert partition.getStartOffset() % partition.getStartAlignment() == 0 : partition; - assert (partition.getStartOffset() + partition.getSize()) % partition.getEndAlignment() == 0 : partition; - } - - assert layoutInfo.getReadOnlyRelocatableOffset() % pageSize == 0 && layoutInfo.getReadOnlyRelocatableSize() % pageSize == 0 : layoutInfo; - assert layoutInfo.getWritableOffset() % pageSize == 0 && layoutInfo.getWritableSize() % pageSize == 0 : layoutInfo; - - return layoutInfo; - } - - @Override - public void writeMetadata(ByteBuffer imageHeapBytes, long imageHeapOffsetInBuffer) { - // For implementation in subclasses, if necessary. - } - - protected abstract ImageHeapLayoutInfo doLayout(ImageHeap imageHeap); - - protected T getReadOnlyPrimitive() { - return getPartitions()[READ_ONLY_PRIMITIVE]; - } - - protected T getReadOnlyReference() { - return getPartitions()[READ_ONLY_REFERENCE]; - } - - protected T getReadOnlyRelocatable() { - return getPartitions()[READ_ONLY_RELOCATABLE]; - } - - protected T getWritablePrimitive() { - return getPartitions()[WRITABLE_PRIMITIVE]; - } - - protected T getWritableReference() { - return getPartitions()[WRITABLE_REFERENCE]; - } - - protected T getWritableHuge() { - return getPartitions()[WRITABLE_HUGE]; - } - - protected T getReadOnlyHuge() { - return getPartitions()[READ_ONLY_HUGE]; - } - - /** The size in bytes at and above which an object should be assigned to the huge partitions. */ - protected long getHugeObjectThreshold() { - // Do not use huge partitions by default, they remain empty and should not consume space - return Long.MAX_VALUE; - } - - private T choosePartition(@SuppressWarnings("unused") ImageHeapObject info, boolean immutable, boolean hasReferences, boolean hasRelocatables) { - if (immutable) { - if (hasRelocatables) { - VMError.guarantee(info.getSize() < getHugeObjectThreshold(), "Objects with relocatable pointers cannot be huge objects"); - return getReadOnlyRelocatable(); - } - if (info.getSize() >= getHugeObjectThreshold()) { - VMError.guarantee(info.getObjectClass() != DynamicHub.class, "Class metadata (dynamic hubs) cannot be huge objects"); - return getReadOnlyHuge(); - } - return hasReferences ? getReadOnlyReference() : getReadOnlyPrimitive(); - } else { - assert info.getObjectClass() != DynamicHub.class : "Class metadata (dynamic hubs) cannot be writable"; - if (info.getSize() >= getHugeObjectThreshold()) { - return getWritableHuge(); - } - return hasReferences ? getWritableReference() : getWritablePrimitive(); - } - } - - protected ImageHeapLayoutInfo createLayoutInfo(long heapStartOffset, long writableBeginOffset) { - long writableEnd = getWritableHuge().getStartOffset() + getWritableHuge().getSize(); - long writableSize = writableEnd - writableBeginOffset; - long imageHeapSize = getReadOnlyHuge().getStartOffset() + getReadOnlyHuge().getSize() - heapStartOffset; - return new ImageHeapLayoutInfo(writableBeginOffset, writableSize, getReadOnlyRelocatable().getStartOffset(), getReadOnlyRelocatable().getSize(), imageHeapSize); - } - - protected abstract T[] createPartitionsArray(int count); - - protected abstract T createPartition(String name, boolean containsReferences, boolean writable, boolean hugeObjects); - - /** - * The native image heap comes in partitions. Each partition holds objects with different - * properties (read-only/writable, primitives/objects). - */ - public abstract static class AbstractImageHeapPartition implements ImageHeapPartition { - private final String name; - private final boolean writable; - - private int startAlignment = -1; - private int endAlignment = -1; - private final List objects = new ArrayList<>(); - - public AbstractImageHeapPartition(String name, boolean writable) { - this.name = name; - this.writable = writable; - } - - public void assign(ImageHeapObject obj) { - assert obj.getPartition() == this : obj; - objects.add(obj); - } - - public void setStartAlignment(int alignment) { - assert this.startAlignment == -1 : "Start alignment already assigned: " + this.startAlignment; - this.startAlignment = alignment; - } - - public final int getStartAlignment() { - assert startAlignment >= 0 : "Start alignment not yet assigned"; - return startAlignment; - } - - public void setEndAlignment(int endAlignment) { - assert this.endAlignment == -1 : "End alignment already assigned: " + this.endAlignment; - this.endAlignment = endAlignment; - } - - public final int getEndAlignment() { - assert endAlignment >= 0 : "End alignment not yet assigned"; - return endAlignment; - } - - public List getObjects() { - return objects; - } - - @Override - public String getName() { - return name; - } - - public boolean isWritable() { - return writable; - } - - @Override - public String toString() { - return name; - } - } -} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index 9745a24a5f98..e45c244257d0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -24,8 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import jdk.graal.compiler.api.replacements.Fold; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -40,8 +38,13 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectVisitor; +import com.oracle.svm.core.os.CommittedMemoryProvider; import com.oracle.svm.core.util.PointerUtils; +import jdk.graal.compiler.api.directives.GraalDirectives; +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.word.Word; + /** * An AlignedHeapChunk can hold many Objects. *

@@ -127,6 +130,9 @@ public static AlignedHeader getEnclosingChunk(Object obj) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static AlignedHeader getEnclosingChunkFromObjectPointer(Pointer ptr) { + if (!GraalDirectives.inIntrinsic()) { + assert !HeapImpl.getHeapImpl().isInImageHeap(ptr) || CommittedMemoryProvider.get().guaranteesHeapPreferredAddressSpaceAlignment() : "can't be used because the image heap is unaligned"; + } return (AlignedHeader) PointerUtils.roundDown(ptr, HeapParameters.getAlignedHeapChunkAlignment()); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java index 001775088ae8..2f249e9f0fbc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java @@ -27,30 +27,77 @@ import java.nio.ByteBuffer; import java.util.List; -import jdk.graal.compiler.core.common.NumUtil; import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.ChunkedImageHeapAllocator.AlignedChunk; import com.oracle.svm.core.genscavenge.ChunkedImageHeapAllocator.Chunk; import com.oracle.svm.core.genscavenge.ChunkedImageHeapAllocator.UnalignedChunk; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.image.ImageHeap; import com.oracle.svm.core.image.ImageHeapLayoutInfo; +import com.oracle.svm.core.image.ImageHeapLayouter; +import com.oracle.svm.core.image.ImageHeapObject; +import com.oracle.svm.core.util.VMError; + +import jdk.graal.compiler.core.common.NumUtil; -public class ChunkedImageHeapLayouter extends AbstractImageHeapLayouter { +public class ChunkedImageHeapLayouter implements ImageHeapLayouter { + /** A partition holding objects with only read-only primitive values, but no references. */ + private static final int READ_ONLY_PRIMITIVE = 0; + /** A partition holding objects with read-only references and primitive values. */ + private static final int READ_ONLY_REFERENCE = READ_ONLY_PRIMITIVE + 1; + /** + * A pseudo-partition used during image building to consolidate objects that contain relocatable + * references. + *

+ * Collecting the relocations together means the dynamic linker has to operate on less of the + * image heap during image startup, and it means that less of the image heap has to be + * copied-on-write if the image heap is relocated in a new process. + *

+ * A relocated reference is read-only once relocated, e.g., at runtime. The read-only relocation + * partition does not exist as a separate partition in the generated image. Instead, the + * read-only reference partition is resized to include the read-only relocation partition as + * well. + */ + private static final int READ_ONLY_RELOCATABLE = READ_ONLY_REFERENCE + 1; + /** A partition holding objects with writable primitive values, but no references. */ + private static final int WRITABLE_PRIMITIVE = READ_ONLY_RELOCATABLE + 1; + /** A partition holding objects with writable references and primitive values. */ + private static final int WRITABLE_REFERENCE = WRITABLE_PRIMITIVE + 1; + /** A partition holding very large writable objects with or without references. */ + private static final int WRITABLE_HUGE = WRITABLE_REFERENCE + 1; + /** + * A partition holding very large read-only objects with or without references, but never with + * relocatable references. + */ + private static final int READ_ONLY_HUGE = WRITABLE_HUGE + 1; + private static final int PARTITION_COUNT = READ_ONLY_HUGE + 1; + + private final ChunkedImageHeapPartition[] partitions; private final ImageHeapInfo heapInfo; private final long startOffset; - private final int nullRegionSize; private final long hugeObjectThreshold; private ChunkedImageHeapAllocator allocator; - public ChunkedImageHeapLayouter(ImageHeapInfo heapInfo, long startOffset, int nullRegionSize) { + @SuppressWarnings("this-escape") + public ChunkedImageHeapLayouter(ImageHeapInfo heapInfo, long startOffset) { assert startOffset == 0 || startOffset >= Heap.getHeap().getImageHeapOffsetInAddressSpace() : "must be relative to the heap base"; + + this.partitions = new ChunkedImageHeapPartition[PARTITION_COUNT]; + this.partitions[READ_ONLY_PRIMITIVE] = new ChunkedImageHeapPartition("readOnlyPrimitive", false, false); + this.partitions[READ_ONLY_REFERENCE] = new ChunkedImageHeapPartition("readOnlyReference", false, false); + this.partitions[READ_ONLY_RELOCATABLE] = new ChunkedImageHeapPartition("readOnlyRelocatable", false, false); + this.partitions[WRITABLE_PRIMITIVE] = new ChunkedImageHeapPartition("writablePrimitive", true, false); + this.partitions[WRITABLE_REFERENCE] = new ChunkedImageHeapPartition("writableReference", true, false); + this.partitions[WRITABLE_HUGE] = new ChunkedImageHeapPartition("writableHuge", true, true); + this.partitions[READ_ONLY_HUGE] = new ChunkedImageHeapPartition("readOnlyHuge", false, true); + this.heapInfo = heapInfo; this.startOffset = startOffset; - this.nullRegionSize = nullRegionSize; UnsignedWord alignedHeaderSize = RememberedSet.get().getHeaderSizeOfAlignedChunk(); UnsignedWord unalignedHeaderSize = RememberedSet.get().getHeaderSizeOfUnalignedChunk(); UnsignedWord hugeThreshold = HeapParameters.getAlignedHeapChunkSize().subtract(alignedHeaderSize); @@ -61,24 +108,82 @@ public ChunkedImageHeapLayouter(ImageHeapInfo heapInfo, long startOffset, int nu } @Override - protected ChunkedImageHeapPartition[] createPartitionsArray(int count) { - return new ChunkedImageHeapPartition[count]; + public ChunkedImageHeapPartition[] getPartitions() { + return partitions; } - @Override - protected ChunkedImageHeapPartition createPartition(String name, boolean containsReferences, boolean writable, boolean hugeObjects) { - return new ChunkedImageHeapPartition(name, writable, hugeObjects); + private ChunkedImageHeapPartition getLastPartition() { + return partitions[PARTITION_COUNT - 1]; } @Override - protected long getHugeObjectThreshold() { - return hugeObjectThreshold; + public void assignObjectToPartition(ImageHeapObject info, boolean immutable, boolean references, boolean relocatable) { + ChunkedImageHeapPartition partition = choosePartition(info, immutable, references, relocatable); + info.setHeapPartition(partition); + partition.assign(info); + } + + private ChunkedImageHeapPartition choosePartition(@SuppressWarnings("unused") ImageHeapObject info, boolean immutable, boolean hasReferences, boolean hasRelocatables) { + if (immutable) { + if (hasRelocatables) { + VMError.guarantee(info.getSize() < hugeObjectThreshold, "Objects with relocatable pointers cannot be huge objects"); + return getReadOnlyRelocatable(); + } + if (info.getSize() >= hugeObjectThreshold) { + VMError.guarantee(info.getObjectClass() != DynamicHub.class, "Class metadata (dynamic hubs) cannot be huge objects"); + return getReadOnlyHuge(); + } + return hasReferences ? getReadOnlyReference() : getReadOnlyPrimitive(); + } else { + assert info.getObjectClass() != DynamicHub.class : "Class metadata (dynamic hubs) cannot be writable"; + if (info.getSize() >= hugeObjectThreshold) { + return getWritableHuge(); + } + return hasReferences ? getWritableReference() : getWritablePrimitive(); + } } @Override - protected ImageHeapLayoutInfo doLayout(ImageHeap imageHeap) { - long position = startOffset + nullRegionSize; - allocator = new ChunkedImageHeapAllocator(imageHeap, position); + public ImageHeapLayoutInfo layout(ImageHeap imageHeap, int pageSize) { + int objectAlignment = ConfigurationValues.getObjectLayout().getAlignment(); + assert pageSize % objectAlignment == 0 : "Page size does not match object alignment"; + + for (ChunkedImageHeapPartition partition : getPartitions()) { + int startAlignment = objectAlignment; + int endAlignment = objectAlignment; + if (partition == getReadOnlyRelocatable()) { + startAlignment = pageSize; + endAlignment = pageSize; + } else if (partition == getWritablePrimitive()) { + startAlignment = pageSize; + } else if (partition == getWritableHuge()) { + endAlignment = pageSize; + } + + /* Make sure the image heap size is a multiple of the page size. */ + if (partition == getLastPartition()) { + endAlignment = pageSize; + } + + partition.setStartAlignment(startAlignment); + partition.setEndAlignment(endAlignment); + } + + ImageHeapLayoutInfo layoutInfo = doLayout(imageHeap); + + for (ChunkedImageHeapPartition partition : getPartitions()) { + assert partition.getStartOffset() % partition.getStartAlignment() == 0 : partition; + assert (partition.getStartOffset() + partition.getSize()) % partition.getEndAlignment() == 0 : partition; + } + + assert layoutInfo.getReadOnlyRelocatableOffset() % pageSize == 0 && layoutInfo.getReadOnlyRelocatableSize() % pageSize == 0 : layoutInfo; + assert layoutInfo.getWritableOffset() % pageSize == 0 && layoutInfo.getWritableSize() % pageSize == 0 : layoutInfo; + + return layoutInfo; + } + + private ImageHeapLayoutInfo doLayout(ImageHeap imageHeap) { + allocator = new ChunkedImageHeapAllocator(imageHeap, startOffset); for (ChunkedImageHeapPartition partition : getPartitions()) { partition.layout(allocator); } @@ -127,6 +232,13 @@ private void initializeHeapInfo(int dynamicHubCount, long offsetOfFirstWritableA getReadOnlyHuge().firstObject, getReadOnlyHuge().lastObject, writableAligned, writableUnaligned, dynamicHubCount); } + private ImageHeapLayoutInfo createLayoutInfo(long heapStartOffset, long writableBeginOffset) { + long writableEnd = getWritableHuge().getStartOffset() + getWritableHuge().getSize(); + long writableSize = writableEnd - writableBeginOffset; + long imageHeapSize = getReadOnlyHuge().getStartOffset() + getReadOnlyHuge().getSize() - heapStartOffset; + return new ImageHeapLayoutInfo(writableBeginOffset, writableSize, getReadOnlyRelocatable().getStartOffset(), getReadOnlyRelocatable().getSize(), imageHeapSize); + } + @Override public void writeMetadata(ByteBuffer imageHeapBytes, long imageHeapOffsetInBuffer) { long layoutToBufferOffsetAddend = imageHeapOffsetInBuffer - startOffset; @@ -152,8 +264,7 @@ private static void writeHeader(ImageHeapChunkWriter writer, Chunk previous, Chu long offsetToPrevious = (previous != null) ? (previous.getBegin() - current.getBegin()) : 0; long offsetToNext = (next != null) ? (next.getBegin() - current.getBegin()) : 0; int chunkPosition = NumUtil.safeToInt(current.getBegin()); - if (current instanceof AlignedChunk) { - AlignedChunk aligned = (AlignedChunk) current; + if (current instanceof AlignedChunk aligned) { writer.initializeAlignedChunk(chunkPosition, current.getTopOffset(), current.getEndOffset(), offsetToPrevious, offsetToNext); writer.enableRememberedSetForAlignedChunk(chunkPosition, aligned.getObjects()); } else { @@ -163,4 +274,32 @@ private static void writeHeader(ImageHeapChunkWriter writer, Chunk previous, Chu } } } + + private ChunkedImageHeapPartition getReadOnlyPrimitive() { + return partitions[READ_ONLY_PRIMITIVE]; + } + + private ChunkedImageHeapPartition getReadOnlyReference() { + return partitions[READ_ONLY_REFERENCE]; + } + + private ChunkedImageHeapPartition getReadOnlyRelocatable() { + return partitions[READ_ONLY_RELOCATABLE]; + } + + private ChunkedImageHeapPartition getWritablePrimitive() { + return partitions[WRITABLE_PRIMITIVE]; + } + + private ChunkedImageHeapPartition getWritableReference() { + return partitions[WRITABLE_REFERENCE]; + } + + private ChunkedImageHeapPartition getWritableHuge() { + return partitions[WRITABLE_HUGE]; + } + + private ChunkedImageHeapPartition getReadOnlyHuge() { + return partitions[READ_ONLY_HUGE]; + } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapPartition.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapPartition.java index 1e23506521d0..ee65ca25caa1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapPartition.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapPartition.java @@ -25,6 +25,7 @@ package com.oracle.svm.core.genscavenge; import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.List; @@ -34,15 +35,20 @@ import java.util.TreeMap; import com.oracle.svm.core.config.ConfigurationValues; -import com.oracle.svm.core.genscavenge.AbstractImageHeapLayouter.AbstractImageHeapPartition; import com.oracle.svm.core.image.ImageHeapObject; +import com.oracle.svm.core.image.ImageHeapPartition; import com.oracle.svm.core.meta.SubstrateObjectConstant; /** - * An unstructured image heap partition that just contains a linear sequence of image heap objects. + * The image heap comes in partitions. Each partition holds objects with different properties + * (read-only/writable, primitives/objects). */ -public class ChunkedImageHeapPartition extends AbstractImageHeapPartition { +public class ChunkedImageHeapPartition implements ImageHeapPartition { + private final String name; + private final boolean writable; private final boolean hugeObjects; + private final int minimumObjectSize; + private final List objects = new ArrayList<>(); Object firstObject; Object lastObject; @@ -50,18 +56,21 @@ public class ChunkedImageHeapPartition extends AbstractImageHeapPartition { long startOffset = -1; long endOffset = -1; - private final int minimumObjectSize; + private int startAlignment = -1; + private int endAlignment = -1; ChunkedImageHeapPartition(String name, boolean writable, boolean hugeObjects) { - super(name, writable); + this.name = name; + this.writable = writable; this.hugeObjects = hugeObjects; /* Cache to prevent frequent lookups of the object layout from ImageSingletons. */ this.minimumObjectSize = ConfigurationValues.getObjectLayout().getMinImageHeapObjectSize(); } - boolean usesUnalignedObjects() { - return hugeObjects; + void assign(ImageHeapObject obj) { + assert obj.getPartition() == this : obj; + objects.add(obj); } void layout(ChunkedImageHeapAllocator allocator) { @@ -77,7 +86,7 @@ private void layoutInUnalignedChunks(ChunkedImageHeapAllocator allocator) { allocator.alignBetweenChunks(getStartAlignment()); startOffset = allocator.getPosition(); - for (ImageHeapObject info : getObjects()) { // No need to sort by size + for (ImageHeapObject info : objects) { // No need to sort by size appendAllocatedObject(info, allocator.allocateUnalignedChunkForObject(info, isWritable())); } @@ -97,9 +106,9 @@ private void layoutInAlignedChunks(ChunkedImageHeapAllocator allocator) { } private void allocateObjectsInAlignedChunks(ChunkedImageHeapAllocator allocator) { - NavigableMap> objects = createSortedObjectsMap(getObjects()); - while (!objects.isEmpty()) { - ImageHeapObject info = dequeueBestFit(objects, allocator.getRemainingBytesInAlignedChunk()); + NavigableMap> sortedObjects = createSortedObjectsMap(); + while (!sortedObjects.isEmpty()) { + ImageHeapObject info = dequeueBestFit(sortedObjects, allocator.getRemainingBytesInAlignedChunk()); if (info == null) { allocator.startNewAlignedChunk(); } else { @@ -124,7 +133,7 @@ private ImageHeapObject dequeueBestFit(NavigableMap return info; } - private static NavigableMap> createSortedObjectsMap(List objects) { + private NavigableMap> createSortedObjectsMap() { ImageHeapObject[] sorted = objects.toArray(new ImageHeapObject[0]); Arrays.sort(sorted, new SizeComparator()); @@ -139,6 +148,7 @@ private static NavigableMap> createSortedObjectsMap currentQueue = new ArrayDeque<>(); map.put(currentObjectsSize, currentQueue); } + assert currentQueue != null; currentQueue.add(obj); } return map; @@ -170,13 +180,46 @@ private static Object extractObject(ImageHeapObject info) { } } + @Override + public String getName() { + return name; + } + + boolean isWritable() { + return writable; + } + + boolean usesUnalignedObjects() { + return hugeObjects; + } + + final int getStartAlignment() { + assert startAlignment >= 0 : "Start alignment not yet assigned"; + return startAlignment; + } + + void setStartAlignment(int alignment) { + assert this.startAlignment == -1 : "Start alignment already assigned: " + this.startAlignment; + this.startAlignment = alignment; + } + + final int getEndAlignment() { + assert endAlignment >= 0 : "End alignment not yet assigned"; + return endAlignment; + } + + void setEndAlignment(int endAlignment) { + assert this.endAlignment == -1 : "End alignment already assigned: " + this.endAlignment; + this.endAlignment = endAlignment; + } + @Override public long getStartOffset() { assert startOffset >= 0 : "Start offset not yet set"; return startOffset; } - public long getEndOffset() { + long getEndOffset() { assert endOffset >= 0 : "End offset not yet set"; return endOffset; } @@ -186,6 +229,10 @@ public long getSize() { return getEndOffset() - getStartOffset(); } + public String toString() { + return name; + } + private static class SizeComparator implements Comparator { @Override public int compare(ImageHeapObject o1, ImageHeapObject o2) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 0bfc16bfd4a7..82d65d75e344 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -331,7 +331,7 @@ private void verifyBeforeGC() { if (!success) { String kind = getGCKind(); Log.log().string("Heap verification failed before ").string(kind).string(" garbage collection.").newline(); - VMError.shouldNotReachHereAtRuntime(); + VMError.shouldNotReachHere("Heap verification failed"); } } finally { verifyBeforeTimer.close(); @@ -350,7 +350,7 @@ private void verifyAfterGC() { if (!success) { String kind = getGCKind(); Log.log().string("Heap verification failed after ").string(kind).string(" garbage collection.").newline(); - VMError.shouldNotReachHereAtRuntime(); + VMError.shouldNotReachHere("Heap verification failed"); } } finally { verifyAfterTime.close(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 44b7dcc2db9a..33d7acb9f0b6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -26,8 +26,6 @@ import java.util.function.IntUnaryOperator; -import jdk.graal.compiler.api.directives.GraalDirectives; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.struct.RawField; @@ -50,6 +48,9 @@ import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.identityhashcode.IdentityHashCodeSupport; +import jdk.graal.compiler.api.directives.GraalDirectives; +import jdk.graal.compiler.word.Word; + /** * The common structure of the chunks of memory which make up the heap. HeapChunks are aggregated * into {@linkplain Space spaces}. A specific "subtype" of chunk should be accessed via its own @@ -332,7 +333,6 @@ public static Pointer asPointer(Header that) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static HeapChunk.Header getEnclosingHeapChunk(Object obj) { if (!GraalDirectives.inIntrinsic()) { - assert !HeapImpl.getHeapImpl().isInImageHeap(obj) || HeapImpl.usesImageHeapChunks() : "Must be checked before calling this method"; assert !ObjectHeaderImpl.isPointerToForwardedObject(Word.objectToUntrackedPointer(obj)) : "Forwarded objects must be a pointer and not an object"; } if (ObjectHeaderImpl.isAlignedObject(obj)) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index 2374e7bb7e38..9ec3bd2c2b10 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -28,13 +28,6 @@ import java.util.ArrayList; import java.util.List; -import jdk.graal.compiler.api.directives.GraalDirectives; -import jdk.graal.compiler.api.replacements.Fold; -import jdk.graal.compiler.core.common.NumUtil; -import jdk.graal.compiler.core.common.SuppressFBWarnings; -import jdk.graal.compiler.nodes.extended.MembarNode; -import jdk.graal.compiler.nodes.memory.address.OffsetAddressNode; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; @@ -42,6 +35,7 @@ import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; +import com.oracle.svm.core.Isolates; import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.SubstrateDiagnostics; @@ -92,6 +86,14 @@ import com.oracle.svm.core.util.UnsignedUtils; import com.oracle.svm.core.util.UserError; +import jdk.graal.compiler.api.directives.GraalDirectives; +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.core.common.NumUtil; +import jdk.graal.compiler.core.common.SuppressFBWarnings; +import jdk.graal.compiler.nodes.extended.MembarNode; +import jdk.graal.compiler.nodes.memory.address.OffsetAddressNode; +import jdk.graal.compiler.word.Word; + public final class HeapImpl extends Heap { /** Synchronization means for notifying {@link #refPendingList} waiters without deadlocks. */ private static final VMMutex REF_MUTEX = new VMMutex("referencePendingList"); @@ -390,12 +392,6 @@ public void detachThread(IsolateThread isolateThread) { ThreadLocalAllocation.disableAndFlushForThread(isolateThread); } - @Fold - public static boolean usesImageHeapChunks() { - // Chunks are needed for card marking and not very useful without it - return usesImageHeapCardMarking(); - } - @Fold public static boolean usesImageHeapCardMarking() { Boolean enabled = SerialGCOptions.ImageHeapCardMarking.getValue(); @@ -430,20 +426,6 @@ public int getImageHeapOffsetInAddressSpace() { return 0; } - @Fold - @Override - public int getImageHeapNullRegionSize() { - if (SubstrateOptions.SpawnIsolates.getValue() && SubstrateOptions.UseNullRegion.getValue() && !CommittedMemoryProvider.get().guaranteesHeapPreferredAddressSpaceAlignment()) { - /* - * Prepend a single null page to the image heap so that there is a memory protected gap - * between the heap base and the start of the image heap. The null page is placed - * directly into the native image file, so it makes the file slightly larger. - */ - return pageSize; - } - return 0; - } - @Fold @Override public boolean allowPageSizeMismatch() { @@ -713,13 +695,8 @@ public long getIdentityHashSalt(Object obj) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static Pointer getImageHeapStart() { - int imageHeapOffsetInAddressSpace = Heap.getHeap().getImageHeapOffsetInAddressSpace(); - if (imageHeapOffsetInAddressSpace > 0) { - return KnownIntrinsics.heapBase().add(imageHeapOffsetInAddressSpace); - } else { - int nullRegionSize = Heap.getHeap().getImageHeapNullRegionSize(); - return KnownIntrinsics.heapBase().add(nullRegionSize); - } + Pointer heapBase = (Pointer) Isolates.getHeapBase(CurrentIsolate.getIsolate()); + return heapBase.add(Heap.getHeap().getImageHeapOffsetInAddressSpace()); } private boolean printLocationInfo(Log log, Pointer ptr, boolean allowJavaHeapAccess, boolean allowUnsafeOperations) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java index 48054b4d9b04..6e098e8497fe 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java @@ -26,13 +26,11 @@ import java.lang.ref.Reference; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; @@ -46,11 +44,13 @@ import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.InteriorObjRefWalker; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.os.CommittedMemoryProvider; import com.oracle.svm.core.snippets.KnownIntrinsics; +import jdk.graal.compiler.word.Word; + public final class HeapVerifier { private static final ObjectVerifier OBJECT_VERIFIER = new ObjectVerifier(); - private static final ImageHeapRegionVerifier IMAGE_HEAP_OBJECT_VERIFIER = new ImageHeapRegionVerifier(); private static final ObjectReferenceVerifier REFERENCE_VERIFIER = new ObjectReferenceVerifier(); @Platforms(Platform.HOSTED_ONLY.class) @@ -59,22 +59,14 @@ private HeapVerifier() { public static boolean verify(Occasion occasion) { boolean success = true; - success &= verifyImageHeapObjects(); + success &= verifyImageHeap(); success &= verifyYoungGeneration(occasion); success &= verifyOldGeneration(); success &= verifyRememberedSets(); return success; } - private static boolean verifyImageHeapObjects() { - if (HeapImpl.usesImageHeapChunks()) { - return verifyChunkedImageHeap(); - } else { - return verifyNonChunkedImageHeap(); - } - } - - private static boolean verifyChunkedImageHeap() { + private static boolean verifyImageHeap() { boolean success = true; ImageHeapInfo info = HeapImpl.getImageHeapInfo(); success &= verifyAlignedChunks(null, info.getFirstWritableAlignedChunk()); @@ -82,12 +74,6 @@ private static boolean verifyChunkedImageHeap() { return success; } - private static boolean verifyNonChunkedImageHeap() { - IMAGE_HEAP_OBJECT_VERIFIER.initialize(); - ImageHeapWalker.walkRegions(HeapImpl.getImageHeapInfo(), IMAGE_HEAP_OBJECT_VERIFIER); - return IMAGE_HEAP_OBJECT_VERIFIER.getResult(); - } - private static boolean verifyYoungGeneration(Occasion occasion) { boolean success = true; YoungGeneration youngGeneration = HeapImpl.getHeapImpl().getYoungGeneration(); @@ -155,15 +141,13 @@ private static boolean verifyRememberedSets() { boolean success = true; RememberedSet rememberedSet = RememberedSet.get(); - if (HeapImpl.usesImageHeapChunks()) { - /* - * For the image heap, we can't verify that all cards are clean after a GC because the - * GC itself may result in dirty cards. - */ - ImageHeapInfo info = HeapImpl.getImageHeapInfo(); - success &= rememberedSet.verify(info.getFirstWritableAlignedChunk()); - success &= rememberedSet.verify(info.getFirstWritableUnalignedChunk()); - } + /* + * For the image heap, we can't verify that all cards are clean after a GC because the GC + * itself may result in dirty cards. + */ + ImageHeapInfo info = HeapImpl.getImageHeapInfo(); + success &= rememberedSet.verify(info.getFirstWritableAlignedChunk()); + success &= rememberedSet.verify(info.getFirstWritableUnalignedChunk()); OldGeneration oldGeneration = HeapImpl.getHeapImpl().getOldGeneration(); Space toSpace = oldGeneration.getToSpace(); @@ -269,49 +253,49 @@ private static boolean verifyObject(Object obj, AlignedHeader aChunk, UnalignedH return false; } - if (HeapImpl.usesImageHeapChunks() || !HeapImpl.getHeapImpl().isInImageHeap(obj)) { - assert aChunk.isNonNull() ^ uChunk.isNonNull(); - HeapChunk.Header expectedChunk = aChunk.isNonNull() ? aChunk : uChunk; - HeapChunk.Header chunk = HeapChunk.getEnclosingHeapChunk(obj); - if (chunk.notEqual(expectedChunk)) { - Log.log().string("Object ").zhex(ptr).string(" should have ").zhex(expectedChunk).string(" as its enclosing chunk but getEnclosingHeapChunk returned ").zhex(chunk).newline(); + assert aChunk.isNonNull() ^ uChunk.isNonNull(); + HeapChunk.Header chunk = aChunk.isNonNull() ? aChunk : uChunk; + if (CommittedMemoryProvider.get().guaranteesHeapPreferredAddressSpaceAlignment()) { + HeapChunk.Header enclosingHeapChunk = HeapChunk.getEnclosingHeapChunk(obj); + if (chunk.notEqual(enclosingHeapChunk)) { + Log.log().string("Object ").zhex(ptr).string(" should have ").zhex(chunk).string(" as its enclosing chunk but getEnclosingHeapChunk returned ").zhex(enclosingHeapChunk).newline(); return false; } + } + + Pointer chunkStart = HeapChunk.asPointer(chunk); + Pointer chunkTop = HeapChunk.getTopPointer(chunk); + if (chunkStart.aboveOrEqual(ptr) || chunkTop.belowOrEqual(ptr)) { + Log.log().string("Object ").zhex(ptr).string(" is not within the allocated part of the chunk: ").zhex(chunkStart).string(" - ").zhex(chunkTop).string("").newline(); + return false; + } - Pointer chunkStart = HeapChunk.asPointer(chunk); - Pointer chunkTop = HeapChunk.getTopPointer(chunk); - if (chunkStart.aboveOrEqual(ptr) || chunkTop.belowOrEqual(ptr)) { - Log.log().string("Object ").zhex(ptr).string(" is not within the allocated part of the chunk: ").zhex(chunkStart).string(" - ").zhex(chunkTop).string("").newline(); + if (aChunk.isNonNull()) { + if (!ObjectHeaderImpl.isAlignedHeader(header)) { + Log.log().string("Header of object ").zhex(ptr).string(" is not marked as aligned: ").zhex(header).newline(); + return false; + } + } else { + assert uChunk.isNonNull(); + if (!ObjectHeaderImpl.isUnalignedHeader(header)) { + Log.log().string("Header of object ").zhex(ptr).string(" is not marked as unaligned: ").zhex(header).newline(); return false; } + } - if (aChunk.isNonNull()) { - if (!ObjectHeaderImpl.isAlignedHeader(header)) { - Log.log().string("Header of object ").zhex(ptr).string(" is not marked as aligned: ").zhex(header).newline(); - return false; - } - } else { - assert uChunk.isNonNull(); - if (!ObjectHeaderImpl.isUnalignedHeader(header)) { - Log.log().string("Header of object ").zhex(ptr).string(" is not marked as unaligned: ").zhex(header).newline(); - return false; - } + Space space = chunk.getSpace(); + if (space == null) { + if (!HeapImpl.getHeapImpl().isInImageHeap(obj)) { + Log.log().string("Object ").zhex(ptr).string(" is not an image heap object even though the space of the parent chunk ").zhex(chunk).string(" is null.").newline(); + return false; } + // Not all objects in the image heap have the remembered set bit in the header, so + // we can't verify that this bit is set. - Space space = chunk.getSpace(); - if (space == null) { - if (!HeapImpl.getHeapImpl().isInImageHeap(obj)) { - Log.log().string("Object ").zhex(ptr).string(" is not an image heap object even though the space of the parent chunk ").zhex(chunk).string(" is null.").newline(); - return false; - } - // Not all objects in the image heap have the remembered set bit in the header, so - // we can't verify that this bit is set. - - } else if (space.isOldSpace()) { - if (SubstrateOptions.useRememberedSet() && !RememberedSet.get().hasRememberedSet(header)) { - Log.log().string("Object ").zhex(ptr).string(" is in old generation chunk ").zhex(chunk).string(" but does not have a remembered set.").newline(); - return false; - } + } else if (space.isOldSpace()) { + if (SubstrateOptions.useRememberedSet() && !RememberedSet.get().hasRememberedSet(header)) { + Log.log().string("Object ").zhex(ptr).string(" is in old generation chunk ").zhex(chunk).string(" but does not have a remembered set.").newline(); + return false; } } @@ -383,29 +367,6 @@ private static void printParent(Object parentObject) { } } - private static class ImageHeapRegionVerifier implements MemoryWalker.ImageHeapRegionVisitor { - private final ImageHeapObjectVerifier objectVerifier; - - @Platforms(Platform.HOSTED_ONLY.class) - ImageHeapRegionVerifier() { - objectVerifier = new ImageHeapObjectVerifier(); - } - - public void initialize() { - objectVerifier.initialize(WordFactory.nullPointer(), WordFactory.nullPointer()); - } - - public boolean getResult() { - return objectVerifier.result; - } - - @Override - public boolean visitNativeImageHeapRegion(T region, MemoryWalker.NativeImageHeapRegionAccess access) { - access.visitObjects(region, objectVerifier); - return true; - } - } - private static class ObjectVerifier implements ObjectVisitor { protected boolean result; private AlignedHeader aChunk; @@ -429,23 +390,6 @@ public boolean visitObject(Object object) { } } - private static class ImageHeapObjectVerifier extends ObjectVerifier { - @Platforms(Platform.HOSTED_ONLY.class) - ImageHeapObjectVerifier() { - } - - @Override - public boolean visitObject(Object object) { - Word pointer = Word.objectToUntrackedPointer(object); - if (!HeapImpl.getHeapImpl().isInImageHeap(object)) { - Log.log().string("Image heap object ").zhex(pointer).string(" is not considered as part of the image heap.").newline(); - result = false; - } - - return super.visitObject(object); - } - } - private static class ObjectReferenceVerifier implements ObjectReferenceVisitor { private boolean result; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java index e7dddf3ed31f..637b534b64d9 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java @@ -24,12 +24,13 @@ */ package com.oracle.svm.core.genscavenge; -import jdk.graal.compiler.word.Word; +import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.BuildPhaseProvider.AfterHeapLayout; +import com.oracle.svm.core.Isolates; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; @@ -37,7 +38,8 @@ import com.oracle.svm.core.heap.UnknownPrimitiveField; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.snippets.KnownIntrinsics; + +import jdk.graal.compiler.word.Word; /** * Information on the multiple partitions that make up the image heap, which don't necessarily form @@ -213,7 +215,8 @@ private static > T asImageHeapChunk(long offsetInI return (T) WordFactory.nullPointer(); } UnsignedWord offset = WordFactory.unsigned(offsetInImageHeap); - return (T) KnownIntrinsics.heapBase().add(offset); + Pointer heapBase = (Pointer) Isolates.getHeapBase(CurrentIsolate.getIsolate()); + return (T) heapBase.add(offset); } public void print(Log log) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java index ec12e0b1e200..504f477c9776 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; @@ -36,9 +35,10 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.hub.LayoutEncoding; -import com.oracle.svm.core.os.CommittedMemoryProvider; import com.oracle.svm.core.util.UnsignedUtils; +import jdk.graal.compiler.word.Word; + public final class ImageHeapWalker { private static final MemoryWalker.NativeImageHeapRegionAccess READ_ONLY_PRIMITIVE_WALKER = new ReadOnlyPrimitiveMemoryWalkerAccess(); private static final MemoryWalker.NativeImageHeapRegionAccess READ_ONLY_REFERENCE_WALKER = new ReadOnlyReferenceMemoryWalkerAccess(); @@ -91,27 +91,22 @@ private static boolean walkPartitionInline(Object firstObject, Object lastObject Pointer firstPointer = Word.objectToUntrackedPointer(firstObject); Pointer lastPointer = Word.objectToUntrackedPointer(lastObject); Pointer current = firstPointer; - HeapChunk.Header currentChunk = WordFactory.nullPointer(); - if (HeapImpl.usesImageHeapChunks()) { - Pointer base = WordFactory.zero(); - if (!CommittedMemoryProvider.get().guaranteesHeapPreferredAddressSpaceAlignment()) { - base = HeapImpl.getImageHeapStart(); - } - Pointer offset = current.subtract(base); - UnsignedWord chunkOffset = alignedChunks ? UnsignedUtils.roundDown(offset, HeapParameters.getAlignedHeapChunkAlignment()) - : offset.subtract(UnalignedHeapChunk.getObjectStartOffset()); - currentChunk = (HeapChunk.Header) chunkOffset.add(base); - // Assumption: the order of chunks in their linked list is the same order as in memory, - // and objects are laid out as a continuous sequence without any gaps. - } + /* Compute the enclosing chunk without assuming that the image heap is aligned. */ + Pointer base = HeapImpl.getImageHeapStart(); + Pointer offset = current.subtract(base); + UnsignedWord chunkOffset = alignedChunks ? UnsignedUtils.roundDown(offset, HeapParameters.getAlignedHeapChunkAlignment()) + : offset.subtract(UnalignedHeapChunk.getObjectStartOffset()); + HeapChunk.Header currentChunk = (HeapChunk.Header) chunkOffset.add(base); + + // Assumption: the order of chunks in their linked list is the same order as in memory, + // and objects are laid out as a continuous sequence without any gaps. + do { Pointer limit = lastPointer; - if (HeapImpl.usesImageHeapChunks()) { - Pointer chunkTop = HeapChunk.getTopPointer(currentChunk); - if (lastPointer.aboveThan(chunkTop)) { - limit = chunkTop.subtract(1); // lastObject in another chunk, visit all objects - } + Pointer chunkTop = HeapChunk.getTopPointer(currentChunk); + if (lastPointer.aboveThan(chunkTop)) { + limit = chunkTop.subtract(1); // lastObject in another chunk, visit all objects } while (current.belowOrEqual(limit)) { Object currentObject = current.toObject(); @@ -124,7 +119,7 @@ private static boolean walkPartitionInline(Object firstObject, Object lastObject } current = LayoutEncoding.getImageHeapObjectEnd(current.toObject()); } - if (HeapImpl.usesImageHeapChunks() && current.belowThan(lastPointer)) { + if (current.belowThan(lastPointer)) { currentChunk = HeapChunk.getNext(currentChunk); current = alignedChunks ? AlignedHeapChunk.getObjectsStart((AlignedHeapChunk.AlignedHeader) currentChunk) : UnalignedHeapChunk.getObjectStart((UnalignedHeapChunk.UnalignedHeader) currentChunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LinearImageHeapAllocator.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LinearImageHeapAllocator.java deleted file mode 100644 index 8f2bde217382..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LinearImageHeapAllocator.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2020, 2020, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.svm.core.genscavenge; - -public class LinearImageHeapAllocator { - private long position; - - public LinearImageHeapAllocator(long position) { - this.position = position; - } - - public long getPosition() { - return position; - } - - public long allocate(long size) { - long begin = position; - position += size; - return begin; - } - - public void align(int multiple) { - allocate(computePadding(position, multiple)); - } - - static long computePadding(long offset, int alignment) { - long remainder = offset % alignment; - return remainder == 0 ? 0 : alignment - remainder; - } -} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LinearImageHeapLayouter.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LinearImageHeapLayouter.java deleted file mode 100644 index 569a999b8c63..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LinearImageHeapLayouter.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2019, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.svm.core.genscavenge; - -import com.oracle.svm.core.config.ConfigurationValues; -import com.oracle.svm.core.heap.Heap; -import com.oracle.svm.core.image.ImageHeap; -import com.oracle.svm.core.image.ImageHeapLayoutInfo; - -public class LinearImageHeapLayouter extends AbstractImageHeapLayouter { - private final ImageHeapInfo heapInfo; - private final long startOffset; - private final int nullRegionSize; - - public LinearImageHeapLayouter(ImageHeapInfo heapInfo, long startOffset, int nullRegionSize) { - assert startOffset >= 0; - assert startOffset == 0 || startOffset >= Heap.getHeap().getImageHeapOffsetInAddressSpace() : "must be relative to the heap base"; - assert nullRegionSize >= 0; - this.heapInfo = heapInfo; - this.startOffset = startOffset; - this.nullRegionSize = nullRegionSize; - } - - @Override - protected LinearImageHeapPartition[] createPartitionsArray(int count) { - return new LinearImageHeapPartition[count]; - } - - @Override - protected LinearImageHeapPartition createPartition(String name, boolean containsReferences, boolean writable, boolean hugeObjects) { - return new LinearImageHeapPartition(name, writable); - } - - @Override - protected ImageHeapLayoutInfo doLayout(ImageHeap imageHeap) { - long beginOffset = startOffset + spaceReservedForNull(); - assert beginOffset >= ConfigurationValues.getObjectLayout().getAlignment() : "Zero designates null"; - LinearImageHeapAllocator allocator = new LinearImageHeapAllocator(beginOffset); - for (LinearImageHeapPartition partition : getPartitions()) { - partition.allocateObjects(allocator); - } - initializeHeapInfo(imageHeap.countDynamicHubs()); - return createLayoutInfo(startOffset, getWritablePrimitive().getStartOffset()); - } - - private long spaceReservedForNull() { - if (startOffset == 0 && nullRegionSize == 0) { - return ConfigurationValues.getObjectLayout().getAlignment(); - } - return nullRegionSize; - } - - /** - * Store which objects are at the boundaries of the image heap partitions. Here, we also merge - * the read-only reference partition with the read-only relocatable partition. - */ - private void initializeHeapInfo(int dynamicHubCount) { - heapInfo.initialize(getReadOnlyPrimitive().firstObject, getReadOnlyPrimitive().lastObject, getReadOnlyReference().firstObject, getReadOnlyReference().lastObject, - getReadOnlyRelocatable().firstObject, getReadOnlyRelocatable().lastObject, getWritablePrimitive().firstObject, getWritablePrimitive().lastObject, - getWritableReference().firstObject, getWritableReference().lastObject, getWritableHuge().firstObject, getWritableHuge().lastObject, - getReadOnlyHuge().firstObject, getReadOnlyHuge().lastObject, ImageHeapInfo.NO_CHUNK, ImageHeapInfo.NO_CHUNK, dynamicHubCount); - } -} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LinearImageHeapPartition.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LinearImageHeapPartition.java deleted file mode 100644 index 5be426e4e560..000000000000 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LinearImageHeapPartition.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2019, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.svm.core.genscavenge; - -import com.oracle.svm.core.config.ConfigurationValues; -import com.oracle.svm.core.genscavenge.AbstractImageHeapLayouter.AbstractImageHeapPartition; -import com.oracle.svm.core.image.ImageHeapObject; - -/** - * An unstructured image heap partition that just contains a linear sequence of image heap objects. - */ -public class LinearImageHeapPartition extends AbstractImageHeapPartition { - Object firstObject; - Object lastObject; - - long startOffset = -1; - long endOffset = -1; - - LinearImageHeapPartition(String name, boolean writable) { - super(name, writable); - } - - void allocateObjects(LinearImageHeapAllocator allocator) { - allocator.align(getStartAlignment()); - startOffset = allocator.getPosition(); - for (ImageHeapObject info : getObjects()) { - allocate(info, allocator); - } - allocator.align(getEndAlignment()); - endOffset = allocator.getPosition(); - } - - private void allocate(ImageHeapObject info, LinearImageHeapAllocator allocator) { - assert info.getPartition() == this; - if (firstObject == null) { - firstObject = info.getObject(); - } - long offsetInPartition = allocator.allocate(info.getSize()) - startOffset; - assert ConfigurationValues.getObjectLayout().isAligned(offsetInPartition) : "start: " + offsetInPartition + " must be aligned."; - info.setOffsetInPartition(offsetInPartition); - lastObject = info.getObject(); - } - - @Override - public void assign(ImageHeapObject obj) { - assert startOffset == -1 && endOffset == -1 : "Adding objects late is not supported"; - super.assign(obj); - } - - @Override - public long getStartOffset() { - assert startOffset >= 0 : "Start offset not yet set"; - return startOffset; - } - - public long getEndOffset() { - assert endOffset >= 0 : "End offset not yet set"; - return endOffset; - } - - @Override - public long getSize() { - return getEndOffset() - getStartOffset(); - } -} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 6c414ee63a13..c9f5af276171 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -305,17 +305,15 @@ public long encodeAsImageHeapObjectHeader(ImageHeapObject obj, long hubOffsetFro long header = hubOffsetFromHeapBase << numReservedExtraBits; VMError.guarantee((header >>> numReservedExtraBits) == hubOffsetFromHeapBase, "Hub is too far from heap base for encoding in object header"); assert (header & reservedBitsMask) == 0 : "Object header bits must be zero initially"; - if (HeapImpl.usesImageHeapCardMarking()) { - if (obj.getPartition() instanceof ChunkedImageHeapPartition partition) { - if (partition.isWritable()) { - header |= REMEMBERED_SET_BIT.rawValue(); - } - if (partition.usesUnalignedObjects()) { - header |= UNALIGNED_BIT.rawValue(); - } - } else { - assert obj.getPartition() instanceof FillerObjectDummyPartition; + if (obj.getPartition() instanceof ChunkedImageHeapPartition partition) { + if (partition.isWritable() && HeapImpl.usesImageHeapCardMarking()) { + header |= REMEMBERED_SET_BIT.rawValue(); + } + if (partition.usesUnalignedObjects()) { + header |= UNALIGNED_BIT.rawValue(); } + } else { + assert obj.getPartition() instanceof FillerObjectDummyPartition; } if (isIdentityHashFieldOptional()) { header |= (IDHASH_STATE_IN_FIELD.rawValue() << IDHASH_STATE_SHIFT); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index 7841fc6e0b34..afcdc50fa19e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -24,8 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import jdk.graal.compiler.api.replacements.Fold; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -41,8 +39,13 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectVisitor; +import com.oracle.svm.core.os.CommittedMemoryProvider; import com.oracle.svm.core.util.UnsignedUtils; +import jdk.graal.compiler.api.directives.GraalDirectives; +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.word.Word; + /** * An UnalignedHeapChunk holds exactly one Object. *

@@ -131,8 +134,11 @@ public static UnalignedHeader getEnclosingChunk(Object obj) { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - static UnalignedHeader getEnclosingChunkFromObjectPointer(Pointer objPointer) { - Pointer chunkPointer = objPointer.subtract(getObjectStartOffset()); + static UnalignedHeader getEnclosingChunkFromObjectPointer(Pointer ptr) { + if (!GraalDirectives.inIntrinsic()) { + assert !HeapImpl.getHeapImpl().isInImageHeap(ptr) || CommittedMemoryProvider.get().guaranteesHeapPreferredAddressSpaceAlignment() : "can't be used because the image heap is unaligned"; + } + Pointer chunkPointer = ptr.subtract(getObjectStartOffset()); return (UnalignedHeader) chunkPointer; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java index 6727381fad42..a5a1cc983c51 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java @@ -43,7 +43,6 @@ import com.oracle.svm.core.genscavenge.HeapImplMemoryMXBean; import com.oracle.svm.core.genscavenge.ImageHeapInfo; import com.oracle.svm.core.genscavenge.IncrementalGarbageCollectorMXBean; -import com.oracle.svm.core.genscavenge.LinearImageHeapLayouter; import com.oracle.svm.core.genscavenge.jvmstat.EpsilonGCPerfData; import com.oracle.svm.core.genscavenge.jvmstat.SerialGCPerfData; import com.oracle.svm.core.genscavenge.remset.CardTableBasedRememberedSet; @@ -143,13 +142,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { @Override public void afterAnalysis(AfterAnalysisAccess access) { - ImageHeapLayouter heapLayouter; - int imageHeapNullRegionSize = Heap.getHeap().getImageHeapNullRegionSize(); - if (HeapImpl.usesImageHeapChunks()) { // needs CommittedMemoryProvider: registered late - heapLayouter = new ChunkedImageHeapLayouter(HeapImpl.getImageHeapInfo(), 0, imageHeapNullRegionSize); - } else { - heapLayouter = new LinearImageHeapLayouter(HeapImpl.getImageHeapInfo(), 0, imageHeapNullRegionSize); - } + ImageHeapLayouter heapLayouter = new ChunkedImageHeapLayouter(HeapImpl.getImageHeapInfo(), 0); ImageSingletons.add(ImageHeapLayouter.class, heapLayouter); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java index ada7186fb4b5..d9fb3d7e1d3d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java @@ -26,9 +26,6 @@ import java.lang.ref.Reference; -import jdk.graal.compiler.core.common.SuppressFBWarnings; -import jdk.graal.compiler.nodes.extended.BranchProbabilityNode; -import jdk.graal.compiler.word.Word; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -48,6 +45,10 @@ import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.util.UnsignedUtils; +import jdk.graal.compiler.core.common.SuppressFBWarnings; +import jdk.graal.compiler.nodes.extended.BranchProbabilityNode; +import jdk.graal.compiler.word.Word; + /** * A card table is a remembered set that summarizes pointer stores into a region. A card is "dirty" * if a pointer has been stored recently into the memory summarized by the card, or "clean" @@ -173,23 +174,26 @@ private static boolean verifyReferent(Reference ref, Pointer cardTableStart, } private static boolean verifyReference(Object parentObject, Pointer cardTableStart, Pointer objectsStart, Pointer reference, Pointer referencedObject) { - if (referencedObject.isNonNull() && !HeapImpl.getHeapImpl().isInImageHeap(referencedObject)) { - Object obj = referencedObject.toObject(); - HeapChunk.Header objChunk = HeapChunk.getEnclosingHeapChunk(obj); - // Fail if we find a reference from the image heap to the runtime heap, or from the - // old generation (which is the only one with remembered sets) to the young generation. - boolean fromImageHeap = HeapImpl.usesImageHeapCardMarking() && HeapImpl.getHeapImpl().isInImageHeap(parentObject); - if (fromImageHeap || HeapChunk.getSpace(objChunk).isYoungSpace()) { - UnsignedWord cardTableIndex = memoryOffsetToIndex(Word.objectToUntrackedPointer(parentObject).subtract(objectsStart)); - Pointer cardTableAddress = cardTableStart.add(cardTableIndex); - Log.log().string("Object ").zhex(Word.objectToUntrackedPointer(parentObject)).string(" (").string(parentObject.getClass().getName()).character(')') - .string(fromImageHeap ? ", which is in the image heap, " : " ") - .string("has an object reference at ") - .zhex(reference).string(" that points to ").zhex(referencedObject).string(" (").string(obj.getClass().getName()).string("), ") - .string("which is in the ").string(fromImageHeap ? "runtime heap" : "young generation").string(". ") - .string("However, the card table at ").zhex(cardTableAddress).string(" is clean.").newline(); - return false; - } + if (referencedObject.isNull() || + HeapImpl.getHeapImpl().isInImageHeap(referencedObject) || + HeapImpl.getHeapImpl().isInImageHeap(parentObject) && !HeapImpl.usesImageHeapCardMarking()) { + return true; + } + + Object obj = referencedObject.toObject(); + HeapChunk.Header objChunk = HeapChunk.getEnclosingHeapChunk(obj); + /* Fail if we find a reference to the young generation. */ + boolean fromImageHeap = HeapImpl.getHeapImpl().isInImageHeap(parentObject); + if (fromImageHeap || HeapChunk.getSpace(objChunk).isYoungSpace()) { + UnsignedWord cardTableIndex = memoryOffsetToIndex(Word.objectToUntrackedPointer(parentObject).subtract(objectsStart)); + Pointer cardTableAddress = cardTableStart.add(cardTableIndex); + Log.log().string("Object ").zhex(Word.objectToUntrackedPointer(parentObject)).string(" (").string(parentObject.getClass().getName()).character(')') + .string(fromImageHeap ? ", which is in the image heap, " : " ") + .string("has an object reference at ") + .zhex(reference).string(" that points to ").zhex(referencedObject).string(" (").string(obj.getClass().getName()).string("), ") + .string("which is in the ").string(fromImageHeap ? "runtime heap" : "young generation").string(". ") + .string("However, the card table at ").zhex(cardTableAddress).string(" is clean.").newline(); + return false; } return true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java index b0544608c316..16b1d7f0b4bd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java @@ -26,7 +26,6 @@ import java.util.List; -import jdk.graal.compiler.nodes.gc.BarrierSet; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.struct.SizeOf; @@ -46,6 +45,7 @@ import com.oracle.svm.core.util.UnsignedUtils; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.nodes.gc.BarrierSet; import jdk.vm.ci.meta.MetaAccessProvider; /** diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java index 555a381ff406..163428a33c3f 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java @@ -38,7 +38,6 @@ import java.util.concurrent.ThreadLocalRandom; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.WordPointer; @@ -67,6 +66,8 @@ import com.oracle.svm.core.posix.headers.Unistd; import com.oracle.svm.core.util.PointerUtils; +import jdk.graal.compiler.word.Word; + /** * An optimal image heap provider for Linux which creates isolate image heaps that retain the * copy-on-write, lazy loading and reclamation semantics provided by the original heap's backing @@ -171,14 +172,6 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W return CEntryPointErrors.PROTECT_HEAP_FAILED; } - // Protect the null region. - int nullRegionSize = Heap.getHeap().getImageHeapNullRegionSize(); - if (nullRegionSize > 0) { - if (VirtualMemoryProvider.get().protect(imageHeapBegin, WordFactory.unsigned(nullRegionSize), Access.NONE) != 0) { - return CEntryPointErrors.PROTECT_HEAP_FAILED; - } - } - basePointer.write(imageHeapBegin); if (endPointer.isNonNull()) { endPointer.write(IMAGE_HEAP_END.get()); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java index 0e4873ad4a35..103ee02581c7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java @@ -29,7 +29,6 @@ import java.util.List; import java.util.function.Consumer; -import jdk.graal.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; @@ -45,7 +44,8 @@ import com.oracle.svm.core.log.Log; import com.oracle.svm.core.option.RuntimeOptionKey; import com.oracle.svm.core.os.CommittedMemoryProvider; -import com.oracle.svm.core.os.ImageHeapProvider; + +import jdk.graal.compiler.api.replacements.Fold; public abstract class Heap { @Fold @@ -157,15 +157,6 @@ public void visitLoadedClasses(Consumer> visitor) { @Fold public abstract int getImageHeapOffsetInAddressSpace(); - /** - * Returns the number of null bytes that should be prepended to the image heap during the image - * build. This value must be a multiple of the page size. When the image heap is mapped at - * runtime, this extra memory gets mapped as well but is marked as inaccessible (see - * {@link ImageHeapProvider} for more details). - */ - @Fold - public abstract int getImageHeapNullRegionSize(); - /** * Returns whether the runtime page size doesn't have to match the page size set at image * creation ({@link SubstrateOptions#getPageSize()}). If there is a mismatch, then the page size diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java index 55c4add3af9e..fcbdbdf643d5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java @@ -32,7 +32,6 @@ import java.util.EnumSet; -import jdk.graal.compiler.api.replacements.Fold; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; @@ -47,6 +46,8 @@ import com.oracle.svm.core.util.UnsignedUtils; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.api.replacements.Fold; + public abstract class AbstractCommittedMemoryProvider implements CommittedMemoryProvider { @Fold @Override @@ -57,7 +58,6 @@ public boolean guaranteesHeapPreferredAddressSpaceAlignment() { @Uninterruptible(reason = "Still being initialized.") protected static int protectSingleIsolateImageHeap() { assert !SubstrateOptions.SpawnIsolates.getValue() : "Must be handled by ImageHeapProvider when SpawnIsolates is enabled"; - assert Heap.getHeap().getImageHeapNullRegionSize() == 0 : "A null region only makes sense with a heap base."; Pointer heapBegin = IMAGE_HEAP_BEGIN.get(); if (Heap.getHeap().getImageHeapOffsetInAddressSpace() != 0) { return CEntryPointErrors.MAP_HEAP_FAILED; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java index afe8fde670ff..1bca6c7217ef 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java @@ -29,8 +29,6 @@ import static com.oracle.svm.core.Isolates.IMAGE_HEAP_WRITABLE_END; import static com.oracle.svm.core.util.PointerUtils.roundUp; -import com.oracle.svm.core.code.DynamicMethodAddressResolutionHeapSupport; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; @@ -39,10 +37,13 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.function.CEntryPointErrors; +import com.oracle.svm.core.code.DynamicMethodAddressResolutionHeapSupport; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.os.VirtualMemoryProvider.Access; import com.oracle.svm.core.util.UnsignedUtils; +import jdk.graal.compiler.word.Word; + public abstract class AbstractCopyingImageHeapProvider extends AbstractImageHeapProvider { @Override public boolean guaranteesHeapPreferredAddressSpaceAlignment() { @@ -108,9 +109,8 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W // Protect the read-only parts at the start of the image heap. UnsignedWord pageSize = VirtualMemoryProvider.get().getGranularity(); - int nullRegionSize = Heap.getHeap().getImageHeapNullRegionSize(); - Pointer firstPartOfReadOnlyImageHeap = imageHeap.add(nullRegionSize); - UnsignedWord writableBeginPageOffset = UnsignedUtils.roundDown(IMAGE_HEAP_WRITABLE_BEGIN.get().subtract(imageHeapBegin.add(nullRegionSize)), pageSize); + Pointer firstPartOfReadOnlyImageHeap = imageHeap; + UnsignedWord writableBeginPageOffset = UnsignedUtils.roundDown(IMAGE_HEAP_WRITABLE_BEGIN.get().subtract(imageHeapBegin), pageSize); if (writableBeginPageOffset.aboveThan(0)) { if (VirtualMemoryProvider.get().protect(firstPartOfReadOnlyImageHeap, writableBeginPageOffset, Access.READ) != 0) { freeImageHeap(allocatedMemory); @@ -129,13 +129,6 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W } } - // Protect the null region. - if (nullRegionSize > 0) { - if (VirtualMemoryProvider.get().protect(imageHeapBegin, WordFactory.unsigned(nullRegionSize), Access.NONE) != 0) { - return CEntryPointErrors.PROTECT_HEAP_FAILED; - } - } - // Update the heap base and end pointers. basePointer.write(heapBase); if (endPointer.isNonNull()) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java index b277db95d154..a20f821b0ec0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.os; -import jdk.graal.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.word.Pointer; @@ -34,15 +33,16 @@ import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.heap.Heap; +import jdk.graal.compiler.api.replacements.Fold; + /** * Provides new instances of the image heap for creating isolates. The same image heap provider * implementation can be shared by different garbage collectors. *

* When a heap base is used, then the image heap is always mapped in a way that the memory at the - * heap base is protected and marked as inaccessible. Depending on the specific scenario, that - * memory may or may not be part of the Native Image file (see - * {@link Heap#getImageHeapNullRegionSize()} and {@link Heap#getImageHeapOffsetInAddressSpace()} for - * more details). This is done regardless of the used GC, platform, or CPU architecture: + * heap base is protected and marked as inaccessible (see + * {@link Heap#getImageHeapOffsetInAddressSpace()} for more details). This is done regardless of the + * used GC, platform, or CPU architecture: * * 

  * | protected memory | image heap |