Skip to content

Commit f3c2685

Browse files
peter-hoferpeter-hofer
committed
[GR-32940] [GR-31997] Use image heap remembered set during complete collections.
PullRequest: graal/9507
2 parents b7c4725 + b632da1 commit f3c2685

File tree

11 files changed

+101
-56
lines changed

11 files changed

+101
-56
lines changed

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -810,12 +810,12 @@ private void blackenDirtyImageHeapRoots() {
810810
Timer blackenImageHeapRootsTimer = timers.blackenImageHeapRoots.open();
811811
try {
812812
ImageHeapInfo info = HeapImpl.getImageHeapInfo();
813-
blackenDirtyImageHeapChunkRoots(info.getFirstAlignedImageHeapChunk(), info.getFirstUnalignedImageHeapChunk());
813+
blackenDirtyImageHeapChunkRoots(info.getFirstWritableAlignedChunk(), info.getFirstWritableUnalignedChunk());
814814

815815
if (AuxiliaryImageHeap.isPresent()) {
816816
ImageHeapInfo auxInfo = AuxiliaryImageHeap.singleton().getImageHeapInfo();
817817
if (auxInfo != null) {
818-
blackenDirtyImageHeapChunkRoots(auxInfo.getFirstAlignedImageHeapChunk(), auxInfo.getFirstUnalignedImageHeapChunk());
818+
blackenDirtyImageHeapChunkRoots(auxInfo.getFirstWritableAlignedChunk(), auxInfo.getFirstWritableUnalignedChunk());
819819
}
820820
}
821821
} finally {
@@ -824,20 +824,34 @@ private void blackenDirtyImageHeapRoots() {
824824
}
825825

826826
private void blackenDirtyImageHeapChunkRoots(AlignedHeader firstAligned, UnalignedHeader firstUnaligned) {
827+
/*
828+
* We clean and remark cards of the image heap only during complete collections when we also
829+
* collect the old generation and can easily remark references into it. It also only makes a
830+
* difference after references to the runtime heap were nulled, which is assumed to be rare.
831+
*/
832+
boolean clean = completeCollection;
833+
827834
AlignedHeader aligned = firstAligned;
828835
while (aligned.isNonNull()) {
829-
RememberedSet.get().walkDirtyObjects(aligned, greyToBlackObjectVisitor);
836+
RememberedSet.get().walkDirtyObjects(aligned, greyToBlackObjectVisitor, clean);
830837
aligned = HeapChunk.getNext(aligned);
831838
}
832839

833840
UnalignedHeader unaligned = firstUnaligned;
834841
while (unaligned.isNonNull()) {
835-
RememberedSet.get().walkDirtyObjects(unaligned, greyToBlackObjectVisitor);
842+
RememberedSet.get().walkDirtyObjects(unaligned, greyToBlackObjectVisitor, clean);
836843
unaligned = HeapChunk.getNext(unaligned);
837844
}
838845
}
839846

840847
private void blackenImageHeapRoots() {
848+
if (HeapImpl.usesImageHeapCardMarking()) {
849+
// Avoid scanning the entire image heap even for complete collections: its remembered
850+
// set contains references into both the runtime heap's old and young generations.
851+
blackenDirtyImageHeapRoots();
852+
return;
853+
}
854+
841855
Timer blackenImageHeapRootsTimer = timers.blackenImageHeapRoots.open();
842856
try {
843857
HeapImpl.getHeapImpl().walkNativeImageHeapRegions(blackenImageHeapRootsVisitor);
@@ -864,7 +878,7 @@ private void blackenDirtyCardRoots() {
864878
* Promote any referenced young objects.
865879
*/
866880
Space oldGenToSpace = HeapImpl.getHeapImpl().getOldGeneration().getToSpace();
867-
RememberedSet.get().walkDirtyObjects(oldGenToSpace, greyToBlackObjectVisitor);
881+
RememberedSet.get().walkDirtyObjects(oldGenToSpace, greyToBlackObjectVisitor, true);
868882
} finally {
869883
blackenDirtyCardRootsTimer.close();
870884
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapPolicy.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public static UnsignedWord m(long bytes) {
8686
return WordFactory.unsigned(bytes).multiply(1024).multiply(1024);
8787
}
8888

89-
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
89+
@Fold
9090
public static int getMaxSurvivorSpaces() {
9191
return HeapPolicyOptions.MaxSurvivorSpaces.getValue();
9292
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,8 @@ private static boolean verifyImageHeapObjects() {
7777
private static boolean verifyChunkedImageHeap() {
7878
boolean success = true;
7979
ImageHeapInfo info = HeapImpl.getImageHeapInfo();
80-
success &= verifyAlignedChunks(null, info.getFirstAlignedImageHeapChunk());
81-
success &= verifyUnalignedChunks(null, info.getFirstUnalignedImageHeapChunk());
80+
success &= verifyAlignedChunks(null, info.getFirstWritableAlignedChunk());
81+
success &= verifyUnalignedChunks(null, info.getFirstWritableUnalignedChunk());
8282
return success;
8383
}
8484

@@ -161,8 +161,8 @@ private static boolean verifyRememberedSets() {
161161
* GC itself may result in dirty cards.
162162
*/
163163
ImageHeapInfo info = HeapImpl.getImageHeapInfo();
164-
success &= rememberedSet.verify(info.getFirstAlignedImageHeapChunk());
165-
success &= rememberedSet.verify(info.getFirstUnalignedImageHeapChunk());
164+
success &= rememberedSet.verify(info.getFirstWritableAlignedChunk());
165+
success &= rememberedSet.verify(info.getFirstWritableUnalignedChunk());
166166
}
167167

168168
OldGeneration oldGeneration = HeapImpl.getHeapImpl().getOldGeneration();

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,8 @@ public final class ImageHeapInfo {
6868
@UnknownObjectField(types = Object.class) public Object firstObject;
6969
@UnknownObjectField(types = Object.class) public Object lastObject;
7070

71-
@UnknownPrimitiveField public long offsetOfFirstAlignedChunkWithRememberedSet;
72-
@UnknownPrimitiveField public long offsetOfFirstUnalignedChunkWithRememberedSet;
71+
@UnknownPrimitiveField public long offsetOfFirstWritableAlignedChunk;
72+
@UnknownPrimitiveField public long offsetOfFirstWritableUnalignedChunk;
7373

7474
@UnknownPrimitiveField public int dynamicHubCount;
7575

@@ -80,10 +80,10 @@ public ImageHeapInfo() {
8080
public void initialize(Object firstReadOnlyPrimitiveObject, Object lastReadOnlyPrimitiveObject, Object firstReadOnlyReferenceObject, Object lastReadOnlyReferenceObject,
8181
Object firstReadOnlyRelocatableObject, Object lastReadOnlyRelocatableObject, Object firstWritablePrimitiveObject, Object lastWritablePrimitiveObject,
8282
Object firstWritableReferenceObject, Object lastWritableReferenceObject, Object firstWritableHugeObject, Object lastWritableHugeObject,
83-
Object firstReadOnlyHugeObject, Object lastReadOnlyHugeObject, long offsetOfFirstAlignedChunkWithRememberedSet, long offsetOfFirstUnalignedChunkWithRememberedSet,
83+
Object firstReadOnlyHugeObject, Object lastReadOnlyHugeObject, long offsetOfFirstWritableAlignedChunk, long offsetOfFirstWritableUnalignedChunk,
8484
int dynamicHubCount) {
85-
assert offsetOfFirstAlignedChunkWithRememberedSet == NO_CHUNK || offsetOfFirstAlignedChunkWithRememberedSet >= 0;
86-
assert offsetOfFirstUnalignedChunkWithRememberedSet == NO_CHUNK || offsetOfFirstUnalignedChunkWithRememberedSet >= 0;
85+
assert offsetOfFirstWritableAlignedChunk == NO_CHUNK || offsetOfFirstWritableAlignedChunk >= 0;
86+
assert offsetOfFirstWritableUnalignedChunk == NO_CHUNK || offsetOfFirstWritableUnalignedChunk >= 0;
8787

8888
this.firstReadOnlyPrimitiveObject = firstReadOnlyPrimitiveObject;
8989
this.lastReadOnlyPrimitiveObject = lastReadOnlyPrimitiveObject;
@@ -99,8 +99,8 @@ public void initialize(Object firstReadOnlyPrimitiveObject, Object lastReadOnlyP
9999
this.lastWritableHugeObject = lastWritableHugeObject;
100100
this.firstReadOnlyHugeObject = firstReadOnlyHugeObject;
101101
this.lastReadOnlyHugeObject = lastReadOnlyHugeObject;
102-
this.offsetOfFirstAlignedChunkWithRememberedSet = offsetOfFirstAlignedChunkWithRememberedSet;
103-
this.offsetOfFirstUnalignedChunkWithRememberedSet = offsetOfFirstUnalignedChunkWithRememberedSet;
102+
this.offsetOfFirstWritableAlignedChunk = offsetOfFirstWritableAlignedChunk;
103+
this.offsetOfFirstWritableUnalignedChunk = offsetOfFirstWritableUnalignedChunk;
104104
this.dynamicHubCount = dynamicHubCount;
105105

106106
// Compute boundaries for checks considering partitions can be empty (first == last == null)
@@ -184,12 +184,12 @@ public boolean isInImageHeap(Pointer objectPointer) {
184184
return result;
185185
}
186186

187-
public AlignedHeader getFirstAlignedImageHeapChunk() {
188-
return asImageHeapChunk(offsetOfFirstAlignedChunkWithRememberedSet);
187+
public AlignedHeader getFirstWritableAlignedChunk() {
188+
return asImageHeapChunk(offsetOfFirstWritableAlignedChunk);
189189
}
190190

191-
public UnalignedHeader getFirstUnalignedImageHeapChunk() {
192-
return asImageHeapChunk(offsetOfFirstUnalignedChunkWithRememberedSet);
191+
public UnalignedHeader getFirstWritableUnalignedChunk() {
192+
return asImageHeapChunk(offsetOfFirstWritableUnalignedChunk);
193193
}
194194

195195
@SuppressWarnings("unchecked")

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -244,9 +244,17 @@ private UnsignedWord computeSurvivorObjectBytes() {
244244
return usedObjectBytes;
245245
}
246246

247+
@AlwaysInline("GC performance")
247248
@SuppressWarnings("static-method")
248249
public boolean contains(Object object) {
249-
return HeapChunk.getSpace(HeapChunk.getEnclosingHeapChunk(object)).isYoungSpace();
250+
if (!HeapImpl.usesImageHeapCardMarking()) {
251+
return HeapChunk.getSpace(HeapChunk.getEnclosingHeapChunk(object)).isYoungSpace();
252+
}
253+
// Only objects in the young generation have no remembered set
254+
UnsignedWord header = ObjectHeaderImpl.readHeaderFromObject(object);
255+
boolean young = !ObjectHeaderImpl.hasRememberedSet(header);
256+
assert young == HeapChunk.getSpace(HeapChunk.getEnclosingHeapChunk(object)).isYoungSpace();
257+
return young;
250258
}
251259

252260
@AlwaysInline("GC performance")

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ public static void dirtyCardForObject(Object object, boolean verifyOnly) {
125125
}
126126
}
127127

128-
public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor) {
128+
public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) {
129129
Pointer cardTableStart = getCardTableStart(chunk);
130130
Pointer fotStart = getFirstObjectTableStart(chunk);
131131
Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk);
@@ -135,7 +135,9 @@ public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisito
135135

136136
for (UnsignedWord index = WordFactory.zero(); index.belowThan(indexLimit); index = index.add(1)) {
137137
if (CardTable.isDirty(cardTableStart, index)) {
138-
CardTable.setClean(cardTableStart, index);
138+
if (clean) {
139+
CardTable.setClean(cardTableStart, index);
140+
}
139141

140142
Pointer ptr = FirstObjectTable.getFirstObjectImprecise(fotStart, objectsStart, objectsLimit, index);
141143
Pointer cardLimit = CardTable.indexToMemoryPointer(objectsStart, index.add(1));

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
import com.oracle.svm.core.UnmanagedMemoryUtil;
3636
import com.oracle.svm.core.genscavenge.HeapChunk;
3737
import com.oracle.svm.core.genscavenge.HeapImpl;
38-
import com.oracle.svm.core.genscavenge.Space;
3938
import com.oracle.svm.core.genscavenge.graal.BarrierSnippets;
4039
import com.oracle.svm.core.heap.ObjectReferenceVisitor;
4140
import com.oracle.svm.core.heap.ReferenceAccess;
@@ -167,14 +166,18 @@ private static boolean verifyReference(Object parentObject, Pointer cardTableSta
167166
if (referencedObject.isNonNull() && !HeapImpl.getHeapImpl().isInImageHeap(referencedObject)) {
168167
Object obj = referencedObject.toObject();
169168
HeapChunk.Header<?> objChunk = HeapChunk.getEnclosingHeapChunk(obj);
170-
Space chunkSpace = HeapChunk.getSpace(objChunk);
171-
// Fail if we find a reference to the young generation.
172-
if (chunkSpace.isYoungSpace()) {
169+
// Fail if we find a reference from the image heap to the runtime heap, or from the
170+
// old generation (which is the only one with remembered sets) to the young generation.
171+
boolean fromImageHeap = HeapImpl.usesImageHeapCardMarking() && HeapImpl.getHeapImpl().isInImageHeap(parentObject);
172+
if (fromImageHeap || HeapChunk.getSpace(objChunk).isYoungSpace()) {
173173
UnsignedWord cardTableIndex = memoryOffsetToIndex(Word.objectToUntrackedPointer(parentObject).subtract(objectsStart));
174174
Pointer cardTableAddress = cardTableStart.add(indexToTableOffset(cardTableIndex));
175-
Log.log().string("Object ").hex(Word.objectToUntrackedPointer(parentObject)).string(" (").string(parentObject.getClass().getName()).string(") has an object reference at ")
176-
.hex(reference).string(" that points to ").hex(referencedObject).string(" (").string(obj.getClass().getName())
177-
.string("), which is in the young generation. However, the card table at ").hex(cardTableAddress).string(" is clean.").newline();
175+
Log.log().string("Object ").hex(Word.objectToUntrackedPointer(parentObject)).string(" (").string(parentObject.getClass().getName()).character(')')
176+
.string(fromImageHeap ? ", which is in the image heap, " : " ")
177+
.string("has an object reference at ")
178+
.hex(reference).string(" that points to ").hex(referencedObject).string(" (").string(obj.getClass().getName()).string("), ")
179+
.string("which is in the ").string(fromImageHeap ? "runtime heap" : "young generation").string(". ")
180+
.string("However, the card table at ").hex(cardTableAddress).string(" is clean.").newline();
178181
return false;
179182
}
180183
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -131,8 +131,22 @@ public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) {
131131
@Override
132132
@AlwaysInline("GC performance")
133133
public void dirtyCardIfNecessary(Object holderObject, Object object) {
134-
if (HeapPolicy.getMaxSurvivorSpaces() == 0 || holderObject == null || object == null || GCImpl.getGCImpl().isCompleteCollection() ||
135-
!HeapImpl.getHeapImpl().getYoungGeneration().contains(object)) {
134+
if (holderObject == null || object == null) {
135+
return;
136+
}
137+
// We dirty the cards of ...
138+
if (HeapPolicy.getMaxSurvivorSpaces() != 0 && !GCImpl.getGCImpl().isCompleteCollection() && HeapImpl.getHeapImpl().getYoungGeneration().contains(object)) {
139+
/*
140+
* ...references from the old generation to the young generation, unless there cannot be
141+
* any such references if we do not use survivor spaces, or if we do but are doing a
142+
* complete collection: in both cases, all objects are promoted to the old generation.
143+
* (We avoid an extra old generation check and might remark a few image heap cards, too)
144+
*/
145+
} else if (HeapImpl.usesImageHeapCardMarking() && GCImpl.getGCImpl().isCompleteCollection() && HeapImpl.getHeapImpl().isInImageHeap(holderObject)) {
146+
// ...references from the image heap to the runtime heap, but we clean and remark those
147+
// only during complete collections.
148+
assert !HeapImpl.getHeapImpl().isInImageHeap(object) : "should never be called for references to image heap objects";
149+
} else {
136150
return;
137151
}
138152

@@ -148,26 +162,26 @@ public void dirtyCardIfNecessary(Object holderObject, Object object) {
148162
}
149163

150164
@Override
151-
public void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor) {
152-
AlignedChunkRememberedSet.walkDirtyObjects(chunk, visitor);
165+
public void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) {
166+
AlignedChunkRememberedSet.walkDirtyObjects(chunk, visitor, clean);
153167
}
154168

155169
@Override
156-
public void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor) {
157-
UnalignedChunkRememberedSet.walkDirtyObjects(chunk, visitor);
170+
public void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) {
171+
UnalignedChunkRememberedSet.walkDirtyObjects(chunk, visitor, clean);
158172
}
159173

160174
@Override
161-
public void walkDirtyObjects(Space space, GreyToBlackObjectVisitor visitor) {
175+
public void walkDirtyObjects(Space space, GreyToBlackObjectVisitor visitor, boolean clean) {
162176
AlignedHeader aChunk = space.getFirstAlignedHeapChunk();
163177
while (aChunk.isNonNull()) {
164-
walkDirtyObjects(aChunk, visitor);
178+
walkDirtyObjects(aChunk, visitor, clean);
165179
aChunk = HeapChunk.getNext(aChunk);
166180
}
167181

168182
UnalignedHeader uChunk = space.getFirstUnalignedHeapChunk();
169183
while (uChunk.isNonNull()) {
170-
walkDirtyObjects(uChunk, visitor);
184+
walkDirtyObjects(uChunk, visitor, clean);
171185
uChunk = HeapChunk.getNext(uChunk);
172186
}
173187
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -132,17 +132,17 @@ public void dirtyCardIfNecessary(Object holderObject, Object object) {
132132
}
133133

134134
@Override
135-
public void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor) {
135+
public void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) {
136136
throw VMError.shouldNotReachHere();
137137
}
138138

139139
@Override
140-
public void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor) {
140+
public void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) {
141141
throw VMError.shouldNotReachHere();
142142
}
143143

144144
@Override
145-
public void walkDirtyObjects(Space space, GreyToBlackObjectVisitor visitor) {
145+
public void walkDirtyObjects(Space space, GreyToBlackObjectVisitor visitor, boolean clean) {
146146
throw VMError.shouldNotReachHere();
147147
}
148148

0 commit comments

Comments
 (0)