[vm/compiler] Use a block size of 16 bytes for ARM64 MemoryCopy.

sstrickl · Commit Queue · commit 959f21ab408d · 2023-09-11T09:24:19.000Z
TEST=vm/cc/IRTest_Memory, co19{,_2}/LibTest/typed_data, lib{,_2}/typed_data, corelib{,_2}/list_test Issue: #42072 Cq-Include-Trybots: luci.dart.try:vm-mac-debug-arm64-try,vm-aot-linux-release-simarm64-try,vm-aot-mac-release-arm64-try,vm-linux-release-simarm64-try,vm-mac-release-arm64-try,vm-aot-android-release-arm64c-try,vm-ffi-android-debug-arm64c-try,vm-ffi-qemu-linux-release-arm-try,vm-ffi-qemu-linux-release-riscv64-try,vm-linux-debug-simriscv64-try,vm-linux-release-simarm-try Change-Id: Ife645a1d09be862d74e198162b124e657878280a Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/324683 Commit-Queue: Tess Strickland <sstrickl@google.com> Reviewed-by: Alexander Markov <alexmarkov@google.com>
diff --git a/runtime/vm/compiler/backend/il_arm.cc b/runtime/vm/compiler/backend/il_arm.cc
@@ -271,21 +271,22 @@ static void CopyUpToWordMultiple(FlowGraphCompiler* compiler,
   ASSERT(Utils::IsPowerOfTwo(element_size));
   if (element_size >= compiler::target::kWordSize) return;
 
-  const intptr_t base_shift = (unboxed_inputs ? 0 : kSmiTagShift) -
-                              Utils::ShiftForPowerOfTwo(element_size);
+  const intptr_t element_shift = Utils::ShiftForPowerOfTwo(element_size);
+  const intptr_t base_shift =
+      (unboxed_inputs ? 0 : kSmiTagShift) - element_shift;
   auto const mode =
       reversed ? compiler::Address::NegPreIndex : compiler::Address::PostIndex;
   intptr_t tested_bits = 0;
 
   __ Comment("Copying until region is a multiple of word size");
 
-  for (intptr_t bit = compiler::target::kWordSizeLog2 - 1; bit >= 0; bit--) {
+  for (intptr_t bit = compiler::target::kWordSizeLog2 - 1; bit >= element_shift;
+       bit--) {
     const intptr_t bytes = 1 << bit;
-    if (element_size > bytes) continue;
-    auto const sz = OperandSizeFor(bytes);
     const intptr_t tested_bit = bit + base_shift;
     tested_bits |= (1 << tested_bit);
     __ tst(length_reg, compiler::Operand(1 << tested_bit));
+    auto const sz = OperandSizeFor(bytes);
     __ LoadFromOffset(TMP, compiler::Address(src_reg, bytes, mode), sz,
                       NOT_ZERO);
     __ StoreToOffset(TMP, compiler::Address(dest_reg, bytes, mode), sz,
diff --git a/runtime/vm/compiler/backend/il_arm64.cc b/runtime/vm/compiler/backend/il_arm64.cc
@@ -192,35 +192,37 @@ static compiler::OperandSize OperandSizeFor(intptr_t bytes) {
   }
 }
 
-static void CopyUpToWordMultiple(FlowGraphCompiler* compiler,
-                                 Register dest_reg,
-                                 Register src_reg,
-                                 Register length_reg,
-                                 intptr_t element_size,
-                                 bool unboxed_inputs,
-                                 bool reversed,
-                                 compiler::Label* done) {
+static void CopyUpToMultipleOfChunkSize(FlowGraphCompiler* compiler,
+                                        Register dest_reg,
+                                        Register src_reg,
+                                        Register length_reg,
+                                        intptr_t element_size,
+                                        bool unboxed_inputs,
+                                        bool reversed,
+                                        intptr_t chunk_size,
+                                        compiler::Label* done) {
   ASSERT(Utils::IsPowerOfTwo(element_size));
-  if (element_size >= compiler::target::kWordSize) return;
+  if (element_size >= chunk_size) return;
 
-  const intptr_t base_shift = (unboxed_inputs ? 0 : kSmiTagShift) -
-                              Utils::ShiftForPowerOfTwo(element_size);
+  const intptr_t element_shift = Utils::ShiftForPowerOfTwo(element_size);
+  const intptr_t base_shift =
+      (unboxed_inputs ? 0 : kSmiTagShift) - element_shift;
   const intptr_t offset_sign = reversed ? -1 : 1;
   auto const mode =
       reversed ? compiler::Address::PreIndex : compiler::Address::PostIndex;
   intptr_t tested_bits = 0;
 
-  __ Comment("Copying until region is a multiple of word size");
+  __ Comment("Copying until region size is a multiple of chunk size");
 
-  for (intptr_t bit = compiler::target::kWordSizeLog2 - 1; bit >= 0; bit--) {
+  for (intptr_t bit = Utils::ShiftForPowerOfTwo(chunk_size) - 1;
+       bit >= element_shift; bit--) {
     const intptr_t bytes = 1 << bit;
-    if (element_size > bytes) continue;
-    auto const sz = OperandSizeFor(bytes);
     const intptr_t tested_bit = bit + base_shift;
     tested_bits |= (1 << tested_bit);
     const intptr_t offset = offset_sign * bytes;
     compiler::Label skip_copy;
     __ tbz(&skip_copy, length_reg, tested_bit);
+    auto const sz = OperandSizeFor(bytes);
     __ ldr(TMP, compiler::Address(src_reg, offset, mode), sz);
     __ str(TMP, compiler::Address(dest_reg, offset, mode), sz);
     __ Bind(&skip_copy);
@@ -262,45 +264,24 @@ void MemoryCopyInstr::EmitLoopCopy(FlowGraphCompiler* compiler,
       __ add(dest_reg, dest_reg, compiler::Operand(length_reg, LSL, shift));
     }
   }
-  CopyUpToWordMultiple(compiler, dest_reg, src_reg, length_reg, element_size_,
-                       unboxed_inputs_, reversed, done);
+  const intptr_t kChunkSize = 16;
+  ASSERT(kChunkSize >= element_size_);
+  CopyUpToMultipleOfChunkSize(compiler, dest_reg, src_reg, length_reg,
+                              element_size_, unboxed_inputs_, reversed,
+                              kChunkSize, done);
+  // The size of the uncopied region is now a multiple of the chunk size.
+  const intptr_t loop_subtract = (kChunkSize / element_size_)
+                                 << (unboxed_inputs_ ? 0 : kSmiTagShift);
   // When reversed, the src and dest registers are adjusted to start with the
   // end addresses, so apply the negated offset prior to indexing.
-  const intptr_t offset =
-      (reversed ? -1 : 1) *
-      Utils::Maximum<intptr_t>(compiler::target::kWordSize, element_size_);
-  const auto mode = element_size_ == 16
-                        ? (reversed ? compiler::Address::PairPreIndex
-                                    : compiler::Address::PairPostIndex)
-                        : (reversed ? compiler::Address::PreIndex
-                                    : compiler::Address::PostIndex);
-  // The size of the uncopied region is a multiple of the word size, so now we
-  // copy the rest by word (unless the element size is larger).
-  const intptr_t loop_subtract =
-      Utils::Maximum<intptr_t>(1, compiler::target::kWordSize / element_size_)
-      << (unboxed_inputs_ ? 0 : kSmiTagShift);
-  const auto src_address = compiler::Address(src_reg, offset, mode);
-  const auto dest_address = compiler::Address(dest_reg, offset, mode);
-  __ Comment("Copying by multiples of word size");
+  const intptr_t offset = (reversed ? -1 : 1) * kChunkSize;
+  const auto mode = reversed ? compiler::Address::PairPreIndex
+                             : compiler::Address::PairPostIndex;
+  __ Comment("Copying chunks at a time");
   compiler::Label loop;
   __ Bind(&loop);
-  switch (element_size_) {
-    // Fall through for the sizes smaller than compiler::target::kWordSize.
-    case 1:
-    case 2:
-    case 4:
-    case 8:
-      __ ldr(TMP, src_address, compiler::kEightBytes);
-      __ str(TMP, dest_address, compiler::kEightBytes);
-      break;
-    case 16:
-      __ ldp(TMP, TMP2, src_address, compiler::kEightBytes);
-      __ stp(TMP, TMP2, dest_address, compiler::kEightBytes);
-      break;
-    default:
-      UNREACHABLE();
-      break;
-  }
+  __ ldp(TMP, TMP2, compiler::Address(src_reg, offset, mode));
+  __ stp(TMP, TMP2, compiler::Address(dest_reg, offset, mode));
   __ subs(length_reg, length_reg, compiler::Operand(loop_subtract),
           compiler::kObjectBytes);
   __ b(&loop, NOT_ZERO);
diff --git a/runtime/vm/compiler/backend/il_riscv.cc b/runtime/vm/compiler/backend/il_riscv.cc
@@ -292,17 +292,18 @@ static void CopyUpToWordMultiple(FlowGraphCompiler* compiler,
   ASSERT(Utils::IsPowerOfTwo(element_size));
   if (element_size >= compiler::target::kWordSize) return;
 
-  const intptr_t base_shift = (unboxed_inputs ? 0 : kSmiTagShift) -
-                              Utils::ShiftForPowerOfTwo(element_size);
+  const intptr_t element_shift = Utils::ShiftForPowerOfTwo(element_size);
+  const intptr_t base_shift =
+      (unboxed_inputs ? 0 : kSmiTagShift) - element_shift;
   intptr_t tested_bits = 0;
 
   __ Comment("Copying until region is a multiple of word size");
 
   COMPILE_ASSERT(XLEN <= 128);
 
-  for (intptr_t bit = compiler::target::kWordSizeLog2 - 1; bit >= 0; bit--) {
+  for (intptr_t bit = compiler::target::kWordSizeLog2 - 1; bit >= element_shift;
+       bit--) {
     const intptr_t bytes = 1 << bit;
-    if (element_size > bytes) continue;
     const intptr_t tested_bit = bit + base_shift;
     tested_bits |= 1 << tested_bit;
     compiler::Label skip_copy;
