[VM] Optimize generation of type testing stubs in JIT mode.

This CL introduces, similar to RawClass::direct_subclasses_, a
RawClass::direct_implementors field which is kept up-to-date.

The generation of type testing stubs uses this cached
reverse-relationship information for faster cid-range calculation (time
spent in type testing stub generation is decreased by an order of
magnitude).

This CL also enable type testing stubs for any instantiated type.

This seems to improve

  * analysis-server-cold-analysis by 6%
  * analysis-server-warm-analysis by 11%

as well as other non-analysis benchmarks in JIT mode.

Issue #33257

Change-Id: If01ee06ac08251b2ed27f79d6b82ad7354c8336e
Reviewed-on: https://dart-review.googlesource.com/66576
Commit-Queue: Martin Kustermann <[email protected]>
Reviewed-by: Alexander Markov <[email protected]>
Reviewed-by: Vyacheslav Egorov <[email protected]>

Author: mkustermann

runtime/vm/class_finalizer.cc

@@ -95,6 +95,60 @@ static void CollectFinalizedSuperClasses(
   }
 }
 
+class InterfaceFinder {
+ public:
+  InterfaceFinder(Zone* zone,
+                  ClassTable* class_table,
+                  GrowableArray<intptr_t>* cids)
+      : class_table_(class_table),
+        array_handles_(zone),
+        class_handles_(zone),
+        type_handles_(zone),
+        cids_(cids) {}
+
+  void FindAllInterfaces(const Class& klass) {
+    // The class is implementing it's own interface.
+    cids_->Add(klass.id());
+
+    ScopedHandle<Array> array(&array_handles_);
+    ScopedHandle<Class> interface_class(&class_handles_);
+    ScopedHandle<Class> current_class(&class_handles_);
+    ScopedHandle<AbstractType> type(&type_handles_);
+
+    *current_class = klass.raw();
+    while (true) {
+      // We don't care about top types.
+      const intptr_t cid = current_class->id();
+      if (cid == kObjectCid || cid == kDynamicCid || cid == kVoidCid) {
+        break;
+      }
+
+      // The class is implementing it's directly declared implemented
+      // interfaces.
+      *array = klass.interfaces();
+      if (!array->IsNull()) {
+        for (intptr_t i = 0; i < array->Length(); ++i) {
+          *type ^= array->At(i);
+          *interface_class ^= class_table_->At(type->type_class_id());
+          FindAllInterfaces(*interface_class);
+        }
+      }
+
+      // The class is implementing it's super type's interfaces.
+      *type = current_class->super_type();
+      if (type->IsNull()) break;
+      *current_class = class_table_->At(type->type_class_id());
+    }
+  }
+
+ private:
+  ClassTable* class_table_;
+  ReusableHandleStack<Array> array_handles_;
+  ReusableHandleStack<Class> class_handles_;
+  ReusableHandleStack<AbstractType> type_handles_;
+  GrowableArray<intptr_t>* cids_;
+};
+
 static void CollectImmediateSuperInterfaces(const Class& cls,
                                             GrowableArray<intptr_t>* cids) {
   const Array& interfaces = Array::Handle(cls.interfaces());
@@ -2609,12 +2663,37 @@ void ClassFinalizer::FinalizeTypesInClass(const Class& cls) {
   // Mark as type finalized before resolving type parameter upper bounds
   // in order to break cycles.
   cls.set_is_type_finalized();
+
   // Add this class to the direct subclasses of the superclass, unless the
   // superclass is Object.
   if (!super_type.IsNull() && !super_type.IsObjectType()) {
     ASSERT(!super_class.IsNull());
     super_class.AddDirectSubclass(cls);
   }
+
+  // Add this class as an implementor to the implemented interface's type
+  // classes.
+  Zone* zone = thread->zone();
+  auto& interface_class = Class::Handle(zone);
+  for (intptr_t i = 0; i < interface_types.Length(); ++i) {
+    interface_type ^= interface_types.At(i);
+    interface_class = interface_type.type_class();
+    interface_class.AddDirectImplementor(cls);
+  }
+
+  if (FLAG_use_cha_deopt) {
+    // Invalidate all CHA code which depends on knowing the implementors of any
+    // of the interfaces implemented by this new class.
+    ClassTable* class_table = thread->isolate()->class_table();
+    GrowableArray<intptr_t> cids;
+    InterfaceFinder finder(zone, class_table, &cids);
+    finder.FindAllInterfaces(cls);
+    for (intptr_t j = 0; j < cids.length(); ++j) {
+      interface_class = class_table->At(cids[j]);
+      interface_class.DisableCHAImplementorUsers();
+    }
+  }
+
   // A top level class is parsed eagerly so just finalize it.
   if (cls.IsTopLevel()) {
     FinalizeClass(cls);

runtime/vm/compiler/aot/precompiler.cc

@@ -2102,8 +2102,9 @@ void Precompiler::TraceTypesFromRetainedClasses() {
         continue;  // class 'dynamic' is in the read-only VM isolate.
       }
 
-      // The subclasses array is only needed for CHA.
+      // The subclasses/implementors array is only needed for CHA.
       cls.ClearDirectSubclasses();
+      cls.ClearDirectImplementors();
 
       bool retain = false;
       members = cls.fields();

runtime/vm/compiler/backend/flow_graph_compiler.cc

@@ -1986,7 +1986,9 @@ void FlowGraphCompiler::GenerateCidRangesCheck(Assembler* assembler,
 
 bool FlowGraphCompiler::ShouldUseTypeTestingStubFor(bool optimizing,
                                                     const AbstractType& type) {
-  return FLAG_precompiled_mode || (optimizing && type.IsTypeParameter());
+  return FLAG_precompiled_mode ||
+         (optimizing &&
+          (type.IsTypeParameter() || (type.IsType() && type.IsInstantiated())));
 }
 
 void FlowGraphCompiler::GenerateAssertAssignableViaTypeTestingStub(
@@ -2061,8 +2063,11 @@ void FlowGraphCompiler::GenerateAssertAssignableViaTypeTestingStub(
       // call-site, we want an optimized type testing stub and therefore record
       // it in the [TypeUsageInfo].
       if (!check_handled_at_callsite) {
-        ASSERT(type_usage_info != NULL);
-        type_usage_info->UseTypeInAssertAssignable(dst_type);
+        if (type_usage_info != NULL) {
+          type_usage_info->UseTypeInAssertAssignable(dst_type);
+        } else {
+          ASSERT(!FLAG_precompiled_mode);
+        }
       }
     }
     __ LoadObject(dst_type_reg, dst_type);

runtime/vm/compiler/backend/il.cc

@@ -47,6 +47,66 @@ DEFINE_FLAG(bool,
             "Support unboxed double and float32x4 fields.");
 DECLARE_FLAG(bool, eliminate_type_checks);
 
+class SubclassFinder {
+ public:
+  SubclassFinder(Zone* zone,
+                 GrowableArray<intptr_t>* cids,
+                 bool include_abstract)
+      : array_handles_(zone),
+        class_handles_(zone),
+        cids_(cids),
+        include_abstract_(include_abstract) {}
+
+  void ScanSubClasses(const Class& klass) {
+    if (include_abstract_ || !klass.is_abstract()) {
+      cids_->Add(klass.id());
+    }
+    ScopedHandle<GrowableObjectArray> array(&array_handles_);
+    ScopedHandle<Class> subclass(&class_handles_);
+    *array = klass.direct_subclasses();
+    if (!array->IsNull()) {
+      for (intptr_t i = 0; i < array->Length(); ++i) {
+        *subclass ^= array->At(i);
+        ScanSubClasses(*subclass);
+      }
+    }
+  }
+
+  void ScanImplementorClasses(const Class& klass) {
+    // An implementor of [klass] is
+    //    * the [klass] itself.
+    //    * all implementors of the direct subclasses of [klass].
+    //    * all implementors of the direct implementors of [klass].
+    if (include_abstract_ || !klass.is_abstract()) {
+      cids_->Add(klass.id());
+    }
+
+    ScopedHandle<GrowableObjectArray> array(&array_handles_);
+    ScopedHandle<Class> subclass_or_implementor(&class_handles_);
+
+    *array = klass.direct_subclasses();
+    if (!array->IsNull()) {
+      for (intptr_t i = 0; i < array->Length(); ++i) {
+        *subclass_or_implementor ^= (*array).At(i);
+        ScanImplementorClasses(*subclass_or_implementor);
+      }
+    }
+    *array = klass.direct_implementors();
+    if (!array->IsNull()) {
+      for (intptr_t i = 0; i < array->Length(); ++i) {
+        *subclass_or_implementor ^= (*array).At(i);
+        ScanImplementorClasses(*subclass_or_implementor);
+      }
+    }
+  }
+
+ private:
+  ReusableHandleStack<GrowableObjectArray> array_handles_;
+  ReusableHandleStack<Class> class_handles_;
+  GrowableArray<intptr_t>* cids_;
+  const bool include_abstract_;
+};
+
 const CidRangeVector& HierarchyInfo::SubtypeRangesForClass(
     const Class& klass,
     bool include_abstract) {
@@ -60,8 +120,13 @@ const CidRangeVector& HierarchyInfo::SubtypeRangesForClass(
 
   CidRangeVector& ranges = (*cid_ranges)[klass.id()];
   if (ranges.length() == 0) {
-    BuildRangesFor(table, &ranges, klass, /*use_subtype_test=*/true,
-                   include_abstract);
+    if (!FLAG_precompiled_mode) {
+      BuildRangesForJIT(table, &ranges, klass, /*use_subtype_test=*/true,
+                        include_abstract);
+    } else {
+      BuildRangesFor(table, &ranges, klass, /*use_subtype_test=*/true,
+                     include_abstract);
+    }
   }
   return ranges;
 }
@@ -76,7 +141,11 @@ const CidRangeVector& HierarchyInfo::SubclassRangesForClass(
 
   CidRangeVector& ranges = cid_subclass_ranges_[klass.id()];
   if (ranges.length() == 0) {
-    BuildRangesFor(table, &ranges, klass, /*use_subtype_test=*/false);
+    if (!FLAG_precompiled_mode) {
+      BuildRangesForJIT(table, &ranges, klass, /*use_subtype_test=*/true);
+    } else {
+      BuildRangesFor(table, &ranges, klass, /*use_subtype_test=*/false);
+    }
   }
   return ranges;
 }
@@ -87,12 +156,14 @@ void HierarchyInfo::BuildRangesFor(ClassTable* table,
                                    bool use_subtype_test,
                                    bool include_abstract) {
   Zone* zone = thread()->zone();
-  Class& cls = Class::Handle(zone);
+  ClassTable* class_table = thread()->isolate()->class_table();
 
   // Only really used if `use_subtype_test == true`.
   const Type& dst_type = Type::Handle(zone, Type::RawCast(klass.RareType()));
   AbstractType& cls_type = AbstractType::Handle(zone);
 
+  Class& cls = Class::Handle(zone);
+  AbstractType& super_type = AbstractType::Handle(zone);
   const intptr_t cid_count = table->NumCids();
 
   intptr_t start = -1;
@@ -123,7 +194,10 @@ void HierarchyInfo::BuildRangesFor(ClassTable* table,
           test_succeded = true;
           break;
         }
-        cls = cls.SuperClass();
+
+        super_type = cls.super_type();
+        const intptr_t type_class_id = super_type.type_class_id();
+        cls = class_table->At(type_class_id);
       }
     }
 
@@ -148,6 +222,94 @@ void HierarchyInfo::BuildRangesFor(ClassTable* table,
   }
 }
 
+void HierarchyInfo::BuildRangesForJIT(ClassTable* table,
+                                      CidRangeVector* ranges,
+                                      const Class& dst_klass,
+                                      bool use_subtype_test,
+                                      bool include_abstract) {
+  if (dst_klass.InVMHeap()) {
+    BuildRangesFor(table, ranges, dst_klass, use_subtype_test,
+                   include_abstract);
+    return;
+  }
+
+  Zone* zone = thread()->zone();
+  GrowableArray<intptr_t> cids;
+  SubclassFinder finder(zone, &cids, include_abstract);
+  if (use_subtype_test) {
+    finder.ScanImplementorClasses(dst_klass);
+  } else {
+    finder.ScanSubClasses(dst_klass);
+  }
+
+  // Sort all collected cids.
+  intptr_t* cids_array = cids.data();
+
+  qsort(cids_array, cids.length(), sizeof(intptr_t),
+        [](const void* a, const void* b) {
+          return static_cast<int>(*static_cast<const intptr_t*>(a) -
+                                  *static_cast<const intptr_t*>(b));
+        });
+
+  // Build ranges of all the cids.
+  Class& klass = Class::Handle();
+  intptr_t left_cid = -1;
+  intptr_t last_cid = -1;
+  for (intptr_t i = 0; i < cids.length(); ++i) {
+    if (left_cid == -1) {
+      left_cid = last_cid = cids[i];
+    } else {
+      const intptr_t current_cid = cids[i];
+
+      // Skip duplicates.
+      if (current_cid == last_cid) continue;
+
+      // Consecutive numbers cids are ok.
+      if (current_cid == (last_cid + 1)) {
+        last_cid = current_cid;
+      } else {
+        // We sorted, after all!
+        RELEASE_ASSERT(last_cid < current_cid);
+
+        intptr_t j = last_cid + 1;
+        for (; j < current_cid; ++j) {
+          if (table->HasValidClassAt(j)) {
+            klass = table->At(j);
+            if (!klass.is_patch() && !klass.IsTopLevel()) {
+              // If we care about abstract classes also, we cannot skip over any
+              // arbitrary abstract class, only those which are subtypes.
+              if (include_abstract) {
+                break;
+              }
+
+              // If the class is concrete we cannot skip over it.
+              if (!klass.is_abstract()) {
+                break;
+              }
+            }
+          }
+        }
+
+        if (current_cid == j) {
+          // If there's only abstract cids between [last_cid] and the
+          // [current_cid] then we connect them.
+          last_cid = current_cid;
+        } else {
+          // Finish the current open cid range and start a new one.
+          ranges->Add(CidRange{left_cid, last_cid});
+          left_cid = last_cid = current_cid;
+        }
+      }
+    }
+  }
+
+  // If there is an open cid-range which we haven't finished yet, we'll
+  // complete it.
+  if (left_cid != -1) {
+    ranges->Add(CidRange{left_cid, last_cid});
+  }
+}
+
 bool HierarchyInfo::CanUseSubtypeRangeCheckFor(const AbstractType& type) {
   ASSERT(type.IsFinalized() && !type.IsMalformedOrMalbounded());
 

runtime/vm/compiler/backend/il.h

@@ -397,12 +397,22 @@ class HierarchyInfo : public StackResource {
   bool CanUseGenericSubtypeRangeCheckFor(const AbstractType& type);
 
  private:
+  // Does not use any hierarchy information available in the system but computes
+  // it via O(n) class table traversal.
   void BuildRangesFor(ClassTable* table,
                       CidRangeVector* ranges,
                       const Class& klass,
                       bool use_subtype_test,
                       bool include_abstract = false);
 
+  // In JIT mode we use hierarchy information stored in the [RawClass]s
+  // direct_subclasses_/direct_implementors_ arrays.
+  void BuildRangesForJIT(ClassTable* table,
+                         CidRangeVector* ranges,
+                         const Class& klass,
+                         bool use_subtype_test,
+                         bool include_abstract = false);
+
   CidRangeVector* cid_subtype_ranges_;
   CidRangeVector* cid_subtype_ranges_abstract_;
   CidRangeVector* cid_subclass_ranges_;