[vm, gc] Apply incremental marking back-pressure in proportion to internal allocation size.

In particular, do not apply incremental marking back-pressure for external allocations. External allocations do not cause additional marking work. If a mutator thread performs a storm of external allocations, this should not cause it to effectively become a marking thread.

TEST=ci
Bug: #47492
Bug: flutter/flutter#106305
Change-Id: Icace57e73d695143f7e295924b61b9b1c04be121
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/249403
Commit-Queue: Ryan Macnak <[email protected]>
Reviewed-by: Martin Kustermann <[email protected]>

Author: rmacnak-google

runtime/vm/heap/heap.cc

@@ -193,7 +193,7 @@ void Heap::CheckExternalGC(Thread* thread) {
     }
     CollectGarbage(thread, GCType::kMarkSweep, GCReason::kExternal);
   } else {
-    CheckConcurrentMarking(thread, GCReason::kExternal);
+    CheckConcurrentMarking(thread, GCReason::kExternal, 0);
   }
 }
 
@@ -459,7 +459,7 @@ void Heap::CollectNewSpaceGarbage(Thread* thread,
         CollectOldSpaceGarbage(thread, GCType::kMarkSweep,
                                GCReason::kPromotion);
       } else {
-        CheckConcurrentMarking(thread, GCReason::kPromotion);
+        CheckConcurrentMarking(thread, GCReason::kPromotion, 0);
       }
     }
   }
@@ -561,7 +561,9 @@ void Heap::CollectAllGarbage(GCReason reason, bool compact) {
   WaitForSweeperTasks(thread);
 }
 
-void Heap::CheckConcurrentMarking(Thread* thread, GCReason reason) {
+void Heap::CheckConcurrentMarking(Thread* thread,
+                                  GCReason reason,
+                                  intptr_t size) {
   PageSpace::Phase phase;
   {
     MonitorLocker ml(old_space_.tasks_lock());
@@ -570,11 +572,9 @@ void Heap::CheckConcurrentMarking(Thread* thread, GCReason reason) {
 
   switch (phase) {
     case PageSpace::kMarking:
-      // TODO(rmacnak): Make the allocator of a large page mark size equals to
-      // the large page?
-      COMPILE_ASSERT(kOldPageSize == 512 * KB);
-      COMPILE_ASSERT(kNewPageSize == 512 * KB);
-      old_space_.IncrementalMarkWithSizeBudget(512 * KB);
+      if (size != 0) {
+        old_space_.IncrementalMarkWithSizeBudget(size);
+      }
       return;
     case PageSpace::kSweepingLarge:
     case PageSpace::kSweepingRegular:

runtime/vm/heap/heap.h

@@ -125,7 +125,7 @@ class Heap {
   void CollectAllGarbage(GCReason reason = GCReason::kFull,
                          bool compact = false);
 
-  void CheckConcurrentMarking(Thread* thread, GCReason reason);
+  void CheckConcurrentMarking(Thread* thread, GCReason reason, intptr_t size);
   void StartConcurrentMarking(Thread* thread, GCReason reason);
   void WaitForMarkerTasks(Thread* thread);
   void WaitForSweeperTasks(Thread* thread);

runtime/vm/heap/pages.cc

@@ -473,7 +473,8 @@ uword PageSpace::TryAllocateInFreshPage(intptr_t size,
   if (growth_policy != kForceGrowth) {
     ASSERT(GrowthControlState());
     if (heap_ != nullptr) {  // Some unit tests.
-      heap_->CheckConcurrentMarking(Thread::Current(), GCReason::kOldSpace);
+      heap_->CheckConcurrentMarking(Thread::Current(), GCReason::kOldSpace,
+                                    kOldPageSize);
     }
   }
 
@@ -514,7 +515,8 @@ uword PageSpace::TryAllocateInFreshLargePage(intptr_t size,
   if (growth_policy != kForceGrowth) {
     ASSERT(GrowthControlState());
     if (heap_ != nullptr) {  // Some unit tests.
-      heap_->CheckConcurrentMarking(Thread::Current(), GCReason::kOldSpace);
+      heap_->CheckConcurrentMarking(Thread::Current(), GCReason::kOldSpace,
+                                    size);
     }
   }
 

runtime/vm/heap/safepoint.cc

@@ -65,7 +65,7 @@ ForceGrowthSafepointOperationScope::~ForceGrowthSafepointOperationScope() {
     if (heap->old_space()->ReachedHardThreshold()) {
       heap->CollectGarbage(T, GCType::kMarkSweep, GCReason::kOldSpace);
     } else {
-      heap->CheckConcurrentMarking(T, GCReason::kOldSpace);
+      heap->CheckConcurrentMarking(T, GCReason::kOldSpace, 0);
     }
   }
 }

runtime/vm/heap/scavenger.cc

@@ -1680,7 +1680,7 @@ void Scavenger::TryAllocateNewTLAB(Thread* thread,
 
   if (can_safepoint) {
     ASSERT(thread->no_safepoint_scope_depth() == 0);
-    heap_->CheckConcurrentMarking(thread, GCReason::kNewSpace);
+    heap_->CheckConcurrentMarking(thread, GCReason::kNewSpace, kNewPageSize);
   }
 
   MutexLocker ml(&space_lock_);