[VM] Introduction of type testing stubs - Part 3

The changes include:

  * Make AssertAssignableInstr no longer have a call-summary, which
    helps methods with several parameter checks by not having to
    re-load/re-initialize type arguments registers

  * Lazily create SubtypeTestCaches: We already go to runtime to warm up
    the caches, so we now also create the caches on the first runtime
    call and patch the pool entries.

  * No longer load the destination name into a register: We only need
    the name when we throw an exception, so it is not on the hot path.
    Instead we let the runtime look at the call site, decoding a pool
    index from the instructions stream.  The destination name will be
    available in the pool, at a consecutive index to the subtype cache.

  * Remove the fall-through to N=1 case for probing subtypeing tests,
    since those will always be handled by the optimized stubs.

  * Do not generate optimized stubs for FutureOr<T> (so far it just
    falled-through to TTS).  We can make optimzed version of that later,
    but it requires special subtyping rules.

  * Local code quality improvement in the type-testing-stubs: Avoid
    extra jump at last case of cid-class-range checks.

There are still a number of optimization opportunities we can do in
future changes.

Issue #31798

Change-Id: I4dc5a8a49f939178fe74d44736ef69e4b9088e46
Reviewed-on: https://dart-review.googlesource.com/46984
Reviewed-by: Vyacheslav Egorov <[email protected]>
Reviewed-by: Régis Crelier <[email protected]>

Author: mkustermann

runtime/vm/code_patcher.cc

@@ -28,4 +28,15 @@ WritableInstructionsScope::~WritableInstructionsScope() {
   }
 }
 
+bool MatchesPattern(uword addr, int16_t* pattern, intptr_t size) {
+  uint8_t* bytes = reinterpret_cast<uint8_t*>(addr);
+  for (intptr_t i = 0; i < size; i++) {
+    int16_t val = pattern[i];
+    if ((val >= 0) && (val != bytes[i])) {
+      return false;
+    }
+  }
+  return true;
+}
+
 }  // namespace dart

runtime/vm/code_patcher.h

@@ -102,8 +102,16 @@ class CodePatcher : public AllStatic {
                                 NativeFunction target,
                                 const Code& trampoline);
 #endif
+
+  static intptr_t GetSubtypeTestCachePoolIndex(uword return_address);
 };
 
+// Beginning from [addr] we compare [size] bytes with [pattern].  All [0..255]
+// values in [pattern] have to match, negative values are skipped.
+//
+// Example pattern: `[0x3d, 0x8b, -1, -1]`.
+bool MatchesPattern(uword addr, int16_t* pattern, intptr_t size);
+
 }  // namespace dart
 
 #endif  // RUNTIME_VM_CODE_PATCHER_H_

runtime/vm/code_patcher_x64.cc

@@ -16,17 +16,6 @@
 
 namespace dart {
 
-static bool MatchesPattern(uword addr, int16_t* pattern, intptr_t size) {
-  uint8_t* bytes = reinterpret_cast<uint8_t*>(addr);
-  for (intptr_t i = 0; i < size; i++) {
-    int16_t val = pattern[i];
-    if ((val >= 0) && (val != bytes[i])) {
-      return false;
-    }
-  }
-  return true;
-}
-
 intptr_t IndexFromPPLoad(uword start) {
   int32_t offset = *reinterpret_cast<int32_t*>(start);
   return ObjectPool::IndexFromOffset(offset);

runtime/vm/compiler/backend/flow_graph_compiler.cc

@@ -1214,7 +1214,7 @@ bool FlowGraphCompiler::NeedsEdgeCounter(TargetEntryInstr* block) {
            (block == flow_graph().graph_entry()->normal_entry())));
 }
 
-// Allocate a register that is not explicitly blocked.
+// Allocate a register that is not explictly blocked.
 static Register AllocateFreeRegister(bool* blocked_registers) {
   for (intptr_t regno = 0; regno < kNumberOfCpuRegisters; regno++) {
     if (!blocked_registers[regno]) {
@@ -1879,18 +1879,32 @@ bool FlowGraphCompiler::GenerateSubtypeRangeCheck(Register class_id_reg,
 void FlowGraphCompiler::GenerateCidRangesCheck(Assembler* assembler,
                                                Register class_id_reg,
                                                const CidRangeVector& cid_ranges,
-                                               Label* is_subtype) {
-  Label fail;
+                                               Label* inside_range_lbl,
+                                               Label* outside_range_lbl,
+                                               bool fall_through_if_inside) {
+  // If there are no valid class ranges, the check will fail.  If we are
+  // supposed to fall-through in the positive case, we'll explicitly jump to
+  // the [outside_range_lbl].
+  if (cid_ranges.length() == 1 && cid_ranges[0].IsIllegalRange()) {
+    if (fall_through_if_inside) {
+      assembler->Jump(outside_range_lbl);
+    }
+    return;
+  }
+
   int bias = 0;
   for (intptr_t i = 0; i < cid_ranges.length(); ++i) {
     const CidRange& range = cid_ranges[i];
-    if (!range.IsIllegalRange()) {
-      bias = EmitTestAndCallCheckCid(assembler, is_subtype, class_id_reg, range,
-                                     bias,
-                                     /*jump_on_miss=*/false);
-    }
+    RELEASE_ASSERT(!range.IsIllegalRange());
+    const bool last_round = i == (cid_ranges.length() - 1);
+
+    Label* jump_label = last_round && fall_through_if_inside ? outside_range_lbl
+                                                             : inside_range_lbl;
+    const bool jump_on_miss = last_round && fall_through_if_inside;
+
+    bias = EmitTestAndCallCheckCid(assembler, jump_label, class_id_reg, range,
+                                   bias, jump_on_miss);
   }
-  assembler->Bind(&fail);
 }
 
 void FlowGraphCompiler::GenerateAssertAssignableAOT(
@@ -1901,7 +1915,7 @@ void FlowGraphCompiler::GenerateAssertAssignableAOT(
     const Register function_type_args_reg,
     const Register subtype_cache_reg,
     const Register dst_type_reg,
-    const Register dst_name_reg,
+    const Register scratch_reg,
     Label* done) {
   // If the int type is assignable to [dst_type] we special case it on the
   // caller side!
@@ -1929,9 +1943,6 @@ void FlowGraphCompiler::GenerateAssertAssignableAOT(
     if (hi != NULL) {
       const Class& type_class = Class::Handle(zone(), dst_type.type_class());
 
-      // We can use [dst_name_reg], there is no overlap of use.
-      const Register scratch_reg = dst_name_reg;
-
       bool used_cid_range_check = false;
       const bool can_use_simple_cid_range_test =
           hi->CanUseSubtypeRangeCheckFor(dst_type);
@@ -1953,10 +1964,6 @@ void FlowGraphCompiler::GenerateAssertAssignableAOT(
     }
     __ LoadObject(dst_type_reg, dst_type);
   }
-
-  __ LoadObject(dst_name_reg, dst_name);
-  __ LoadObject(subtype_cache_reg,
-                SubtypeTestCache::ZoneHandle(zone(), SubtypeTestCache::New()));
 }
 
 #undef __

runtime/vm/compiler/backend/flow_graph_compiler.h

@@ -345,7 +345,7 @@ class FlowGraphCompiler : public ValueObject {
                                    const Register function_type_args_reg,
                                    const Register subtype_cache_reg,
                                    const Register dst_type_reg,
-                                   const Register dst_name_reg,
+                                   const Register scratch_reg,
                                    Label* done);
 
 // DBC emits calls very differently from all other architectures due to its
@@ -424,11 +424,15 @@ class FlowGraphCompiler : public ValueObject {
   // of going into the subtyping cache)
   static const intptr_t kMaxNumberOfCidRangesToTest = 4;
 
-  // Falls through to false.
+  // If [fall_through_if_inside] is `true`, then [outside_range_lbl] must be
+  // supplied, since it will be jumped to in the last case if the cid is outside
+  // the range.
   static void GenerateCidRangesCheck(Assembler* assembler,
                                      Register class_id_reg,
                                      const CidRangeVector& cid_ranges,
-                                     Label* is_subtype_lbl);
+                                     Label* inside_range_lbl,
+                                     Label* outside_range_lbl = NULL,
+                                     bool fall_through_if_inside = false);
 
   void EmitOptimizedInstanceCall(const StubEntry& stub_entry,
                                  const ICData& ic_data,

runtime/vm/compiler/backend/flow_graph_compiler_arm.cc

@@ -717,17 +717,33 @@ void FlowGraphCompiler::GenerateAssertAssignableAOT(
 
   const Register kSubtypeTestCacheReg = R3;
   const Register kDstTypeReg = R8;
-  const Register kDstNameReg = R4;
+  const Register kScratchReg = R4;
 
   Label done;
 
   GenerateAssertAssignableAOT(dst_type, dst_name, kInstanceReg,
                               kInstantiatorTypeArgumentsReg,
                               kFunctionTypeArgumentsReg, kSubtypeTestCacheReg,
-                              kDstTypeReg, kDstNameReg, &done);
+                              kDstTypeReg, kScratchReg, &done);
+  // We use 2 consecutive entries in the pool for the subtype cache and the
+  // destination name.  The second entry, namely [dst_name] seems to be unused,
+  // but it will be used by the code throwing a TypeError if the type test fails
+  // (see runtime/vm/runtime_entry.cc:TypeCheck).  It will use pattern matching
+  // on the call site to find out at which pool index the destination name is
+  // located.
+  const intptr_t sub_type_cache_index = __ object_pool_wrapper().AddObject(
+      Object::null_object(), Patchability::kPatchable);
+  const intptr_t sub_type_cache_offset =
+      ObjectPool::element_offset(sub_type_cache_index) - kHeapObjectTag;
+  const intptr_t dst_name_index =
+      __ object_pool_wrapper().AddObject(dst_name, Patchability::kPatchable);
+  ASSERT((sub_type_cache_index + 1) == dst_name_index);
+  ASSERT(__ constant_pool_allowed());
+
   __ LoadField(R9,
                FieldAddress(kDstTypeReg,
                             AbstractType::type_test_stub_entry_point_offset()));
+  __ ldr(kSubtypeTestCacheReg, Address(PP, sub_type_cache_offset));
   __ blx(R9);
   EmitCallsiteMetadata(token_pos, deopt_id, RawPcDescriptors::kOther, locs);
   __ Bind(&done);
@@ -1233,13 +1249,14 @@ int FlowGraphCompiler::EmitTestAndCallCheckCid(Assembler* assembler,
                                                bool jump_on_miss) {
   intptr_t cid_start = range.cid_start;
   if (range.IsSingleCid()) {
-    __ CompareImmediate(class_id_reg, cid_start - bias);
-    __ b(label, jump_on_miss ? NE : EQ);
+    __ AddImmediateSetFlags(class_id_reg, class_id_reg, bias - cid_start);
+    __ BranchIf(jump_on_miss ? NOT_ZERO : ZERO, label);
+    bias = cid_start;
   } else {
     __ AddImmediate(class_id_reg, class_id_reg, bias - cid_start);
-    bias = cid_start;
     __ CompareImmediate(class_id_reg, range.Extent());
-    __ b(label, jump_on_miss ? HI : LS);  // Unsigned higher.
+    __ BranchIf(jump_on_miss ? UNSIGNED_GREATER : UNSIGNED_LESS_EQUAL, label);
+    bias = cid_start;
   }
   return bias;
 }

runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc

@@ -694,18 +694,34 @@ void FlowGraphCompiler::GenerateAssertAssignableAOT(
 
   const Register kSubtypeTestCacheReg = R3;
   const Register kDstTypeReg = R8;
-  const Register kDstNameReg = R4;
+  const Register kScratchReg = R4;
 
   Label done;
 
   GenerateAssertAssignableAOT(dst_type, dst_name, kInstanceReg,
                               kInstantiatorTypeArgumentsReg,
                               kFunctionTypeArgumentsReg, kSubtypeTestCacheReg,
-                              kDstTypeReg, kDstNameReg, &done);
+                              kDstTypeReg, kScratchReg, &done);
+
+  // We use 2 consecutive entries in the pool for the subtype cache and the
+  // destination name.  The second entry, namely [dst_name] seems to be unused,
+  // but it will be used by the code throwing a TypeError if the type test fails
+  // (see runtime/vm/runtime_entry.cc:TypeCheck).  It will use pattern matching
+  // on the call site to find out at which pool index the destination name is
+  // located.
+  const intptr_t sub_type_cache_index = __ object_pool_wrapper().AddObject(
+      Object::null_object(), Patchability::kPatchable);
+  const intptr_t sub_type_cache_offset =
+      ObjectPool::element_offset(sub_type_cache_index);
+  const intptr_t dst_name_index =
+      __ object_pool_wrapper().AddObject(dst_name, Patchability::kPatchable);
+  ASSERT((sub_type_cache_index + 1) == dst_name_index);
+  ASSERT(__ constant_pool_allowed());
 
   __ LoadField(R9,
                FieldAddress(kDstTypeReg,
                             AbstractType::type_test_stub_entry_point_offset()));
+  __ ldr(kSubtypeTestCacheReg, Address(PP, sub_type_cache_offset));
   __ blr(R9);
   EmitCallsiteMetadata(token_pos, deopt_id, RawPcDescriptors::kOther, locs);
   __ Bind(&done);
@@ -1190,13 +1206,14 @@ int FlowGraphCompiler::EmitTestAndCallCheckCid(Assembler* assembler,
                                                bool jump_on_miss) {
   intptr_t cid_start = range.cid_start;
   if (range.IsSingleCid()) {
-    __ CompareImmediate(class_id_reg, cid_start - bias);
-    __ b(label, jump_on_miss ? NE : EQ);
+    __ AddImmediateSetFlags(class_id_reg, class_id_reg, bias - cid_start);
+    __ BranchIf(jump_on_miss ? NOT_EQUAL : EQUAL, label);
+    bias = cid_start;
   } else {
     __ AddImmediate(class_id_reg, bias - cid_start);
     bias = cid_start;
     __ CompareImmediate(class_id_reg, range.Extent());
-    __ b(label, jump_on_miss ? HI : LS);  // Unsigned higher / less-or-equal.
+    __ BranchIf(jump_on_miss ? UNSIGNED_GREATER : UNSIGNED_LESS_EQUAL, label);
   }
   return bias;
 }

runtime/vm/compiler/backend/flow_graph_compiler_x64.cc

@@ -702,14 +702,30 @@ void FlowGraphCompiler::GenerateAssertAssignableAOT(
   Label done;
 
   const Register subtype_cache_reg = R9;
-  const Register dst_type_reg = RBX;
-  const Register dst_name_reg = R10;
+  const Register kScratchReg = RBX;
 
   GenerateAssertAssignableAOT(dst_type, dst_name, kInstanceReg,
                               kInstantiatorTypeArgumentsReg,
                               kFunctionTypeArgumentsReg, subtype_cache_reg,
-                              dst_type_reg, dst_name_reg, &done);
-
+                              kScratchReg, kScratchReg, &done);
+
+  // We use 2 consecutive entries in the pool for the subtype cache and the
+  // destination name.  The second entry, namely [dst_name] seems to be unused,
+  // but it will be used by the code throwing a TypeError if the type test fails
+  // (see runtime/vm/runtime_entry.cc:TypeCheck).  It will use pattern matching
+  // on the call site to find out at which pool index the destination name is
+  // located.
+  const intptr_t sub_type_cache_index = __ object_pool_wrapper().AddObject(
+      Object::null_object(), Patchability::kPatchable);
+  const intptr_t sub_type_cache_offset =
+      ObjectPool::element_offset(sub_type_cache_index) - kHeapObjectTag;
+  const intptr_t dst_name_index =
+      __ object_pool_wrapper().AddObject(dst_name, Patchability::kPatchable);
+  ASSERT((sub_type_cache_index + 1) == dst_name_index);
+  ASSERT(__ constant_pool_allowed());
+
+  __ movq(subtype_cache_reg,
+          Address::AddressBaseImm32(PP, sub_type_cache_offset));
   __ call(FieldAddress(RBX, AbstractType::type_test_stub_entry_point_offset()));
   EmitCallsiteMetadata(token_pos, deopt_id, RawPcDescriptors::kOther, locs);
   __ Bind(&done);
@@ -1151,14 +1167,13 @@ int FlowGraphCompiler::EmitTestAndCallCheckCid(Assembler* assembler,
                                                bool jump_on_miss) {
   intptr_t cid_start = range.cid_start;
   if (range.IsSingleCid()) {
-    __ cmpl(class_id_reg, Immediate(cid_start - bias));
-    __ j(jump_on_miss ? NOT_EQUAL : EQUAL, label);
+    __ CompareImmediate(class_id_reg, cid_start - bias);
+    __ BranchIf(jump_on_miss ? NOT_EQUAL : EQUAL, label);
   } else {
     __ addl(class_id_reg, Immediate(bias - cid_start));
     bias = cid_start;
     __ cmpl(class_id_reg, Immediate(range.Extent()));
-    __ j(jump_on_miss ? ABOVE : BELOW_EQUAL,
-         label);  // Unsigned higher / lower-or-equal.
+    __ BranchIf(jump_on_miss ? UNSIGNED_GREATER : UNSIGNED_LESS_EQUAL, label);
   }
   return bias;
 }

runtime/vm/compiler/backend/il.cc

@@ -151,6 +151,18 @@ bool HierarchyInfo::CanUseSubtypeRangeCheckFor(const AbstractType& type) {
   Zone* zone = thread_->zone();
   const Class& type_class = Class::Handle(zone, type.type_class());
 
+  // The FutureOr<T> type cannot be handled by checking whether the instance is
+  // a subtype of FutureOr and then checking whether the type argument `T`
+  // matches.
+  //
+  // Instead we would need to perform multiple checks:
+  //
+  //    instance is Null || instance is T || instance is Future<T>
+  //
+  if (type_class.IsFutureOrClass()) {
+    return false;
+  }
+
   // We can use class id range checks only if we don't have to test type
   // arguments.
   //
@@ -187,6 +199,18 @@ bool HierarchyInfo::CanUseGenericSubtypeRangeCheckFor(
   const intptr_t num_type_parameters = type_class.NumTypeParameters();
   const intptr_t num_type_arguments = type_class.NumTypeArguments();
 
+  // The FutureOr<T> type cannot be handled by checking whether the instance is
+  // a subtype of FutureOr and then checking whether the type argument `T`
+  // matches.
+  //
+  // Instead we would need to perform multiple checks:
+  //
+  //    instance is Null || instance is T || instance is Future<T>
+  //
+  if (type_class.IsFutureOrClass()) {
+    return false;
+  }
+
   // This function should only be called for generic classes.
   ASSERT(type_class.NumTypeParameters() > 0 &&
          type.arguments() != TypeArguments::null());
@@ -212,7 +236,9 @@ bool HierarchyInfo::CanUseGenericSubtypeRangeCheckFor(
   AbstractType& type_arg = AbstractType::Handle(zone);
   for (intptr_t i = 0; i < num_type_parameters; ++i) {
     type_arg = ta.TypeAt(num_type_arguments - num_type_parameters + i);
-    if (!CanUseSubtypeRangeCheckFor(type_arg) && !type_arg.IsTypeParameter()) {
+    // NOTE: We can handle type parameters but it increases size by a
+    // significant amount.
+    if (!CanUseSubtypeRangeCheckFor(type_arg)) {
       return false;
     }
   }