[CodeGen] Renumber slot indexes before register allocation (#66334)

RegAllocGreedy uses SlotIndexes::getApproxInstrDistance to approximate
the length of a live range for its heuristics. Renumbering all slot
indexes with the default instruction distance ensures that this estimate
will be as accurate as possible, and will not depend on the history of
how instructions have been added to and removed from SlotIndexes's maps.

This also means that enabling -early-live-intervals, which runs the
SlotIndexes analysis earlier, will not cause large amounts of churn due
to different register allocator decisions.

Author: jayfoad

llvm/include/llvm/CodeGen/SlotIndexes.h

@@ -640,6 +640,9 @@ class raw_ostream;
       renumberIndexes(newItr);
       llvm::sort(idx2MBBMap, less_first());
     }
+
+    /// Renumber all indexes using the default instruction distance.
+    void packIndexes();
   };
 
   // Specialize IntervalMapInfo for half-open slot index intervals.

llvm/lib/CodeGen/RegAllocGreedy.cpp

@@ -2692,6 +2692,9 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
     return false;
 
   Indexes = &getAnalysis<SlotIndexes>();
+  // Renumber to get accurate and consistent results from
+  // SlotIndexes::getApproxInstrDistance.
+  Indexes->packIndexes();
   MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
   DomTree = &getAnalysis<MachineDominatorTree>();
   ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE();

llvm/lib/CodeGen/SlotIndexes.cpp

@@ -237,6 +237,11 @@ void SlotIndexes::repairIndexesInRange(MachineBasicBlock *MBB,
   }
 }
 
+void SlotIndexes::packIndexes() {
+  for (auto [Index, Entry] : enumerate(indexList))
+    Entry.setIndex(Index * SlotIndex::InstrDist);
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 LLVM_DUMP_METHOD void SlotIndexes::dump() const {
   for (const IndexListEntry &ILE : indexList) {

llvm/test/CodeGen/AArch64/active_lane_mask.ll

@@ -221,69 +221,69 @@ define <vscale x 32 x i1> @lane_mask_nxv32i1_i64(i64 %index, i64 %TC) {
 ; CHECK-NEXT:    index z1.d, #0, #1
 ; CHECK-NEXT:    mov z0.d, x0
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov z7.d, x1
+; CHECK-NEXT:    mov z3.d, x1
 ; CHECK-NEXT:    mov z2.d, z1.d
-; CHECK-NEXT:    mov z3.d, z1.d
+; CHECK-NEXT:    mov z4.d, z1.d
 ; CHECK-NEXT:    mov z6.d, z1.d
-; CHECK-NEXT:    uqadd z5.d, z1.d, z0.d
+; CHECK-NEXT:    uqadd z25.d, z1.d, z0.d
 ; CHECK-NEXT:    incd z1.d, all, mul #8
 ; CHECK-NEXT:    incd z2.d
-; CHECK-NEXT:    incd z3.d, all, mul #2
+; CHECK-NEXT:    incd z4.d, all, mul #2
 ; CHECK-NEXT:    incd z6.d, all, mul #4
-; CHECK-NEXT:    cmphi p1.d, p0/z, z7.d, z5.d
+; CHECK-NEXT:    cmphi p1.d, p0/z, z3.d, z25.d
 ; CHECK-NEXT:    uqadd z1.d, z1.d, z0.d
-; CHECK-NEXT:    mov z4.d, z2.d
-; CHECK-NEXT:    uqadd z24.d, z2.d, z0.d
-; CHECK-NEXT:    mov z25.d, z2.d
-; CHECK-NEXT:    mov z27.d, z3.d
-; CHECK-NEXT:    uqadd z26.d, z3.d, z0.d
+; CHECK-NEXT:    mov z5.d, z2.d
+; CHECK-NEXT:    uqadd z26.d, z2.d, z0.d
+; CHECK-NEXT:    mov z7.d, z2.d
+; CHECK-NEXT:    mov z24.d, z4.d
+; CHECK-NEXT:    uqadd z27.d, z4.d, z0.d
 ; CHECK-NEXT:    uqadd z28.d, z6.d, z0.d
 ; CHECK-NEXT:    incd z2.d, all, mul #8
-; CHECK-NEXT:    incd z3.d, all, mul #8
-; CHECK-NEXT:    incd z6.d, all, mul #8
-; CHECK-NEXT:    incd z4.d, all, mul #2
-; CHECK-NEXT:    incd z25.d, all, mul #4
-; CHECK-NEXT:    cmphi p2.d, p0/z, z7.d, z24.d
-; CHECK-NEXT:    incd z27.d, all, mul #4
-; CHECK-NEXT:    cmphi p3.d, p0/z, z7.d, z26.d
-; CHECK-NEXT:    cmphi p5.d, p0/z, z7.d, z28.d
-; CHECK-NEXT:    uqadd z2.d, z2.d, z0.d
-; CHECK-NEXT:    uqadd z3.d, z3.d, z0.d
-; CHECK-NEXT:    mov z24.d, z4.d
-; CHECK-NEXT:    uqadd z5.d, z4.d, z0.d
-; CHECK-NEXT:    uqadd z26.d, z25.d, z0.d
 ; CHECK-NEXT:    incd z4.d, all, mul #8
-; CHECK-NEXT:    incd z25.d, all, mul #8
-; CHECK-NEXT:    uzp1 p1.s, p1.s, p2.s
+; CHECK-NEXT:    incd z6.d, all, mul #8
+; CHECK-NEXT:    incd z5.d, all, mul #2
+; CHECK-NEXT:    incd z7.d, all, mul #4
+; CHECK-NEXT:    cmphi p2.d, p0/z, z3.d, z26.d
 ; CHECK-NEXT:    incd z24.d, all, mul #4
-; CHECK-NEXT:    cmphi p8.d, p0/z, z7.d, z2.d
-; CHECK-NEXT:    cmphi p4.d, p0/z, z7.d, z5.d
-; CHECK-NEXT:    uqadd z5.d, z27.d, z0.d
-; CHECK-NEXT:    incd z27.d, all, mul #8
+; CHECK-NEXT:    cmphi p3.d, p0/z, z3.d, z27.d
+; CHECK-NEXT:    cmphi p5.d, p0/z, z3.d, z28.d
+; CHECK-NEXT:    uqadd z2.d, z2.d, z0.d
 ; CHECK-NEXT:    uqadd z4.d, z4.d, z0.d
-; CHECK-NEXT:    cmphi p6.d, p0/z, z7.d, z26.d
-; CHECK-NEXT:    uqadd z28.d, z24.d, z0.d
+; CHECK-NEXT:    uqadd z6.d, z6.d, z0.d
+; CHECK-NEXT:    mov z26.d, z5.d
+; CHECK-NEXT:    uqadd z25.d, z5.d, z0.d
+; CHECK-NEXT:    uqadd z27.d, z7.d, z0.d
+; CHECK-NEXT:    incd z5.d, all, mul #8
+; CHECK-NEXT:    incd z7.d, all, mul #8
+; CHECK-NEXT:    uzp1 p1.s, p1.s, p2.s
+; CHECK-NEXT:    incd z26.d, all, mul #4
+; CHECK-NEXT:    cmphi p8.d, p0/z, z3.d, z2.d
+; CHECK-NEXT:    cmphi p4.d, p0/z, z3.d, z25.d
+; CHECK-NEXT:    uqadd z25.d, z24.d, z0.d
 ; CHECK-NEXT:    incd z24.d, all, mul #8
+; CHECK-NEXT:    uqadd z5.d, z5.d, z0.d
+; CHECK-NEXT:    uqadd z7.d, z7.d, z0.d
+; CHECK-NEXT:    cmphi p6.d, p0/z, z3.d, z27.d
+; CHECK-NEXT:    uqadd z28.d, z26.d, z0.d
+; CHECK-NEXT:    incd z26.d, all, mul #8
 ; CHECK-NEXT:    uzp1 p3.s, p3.s, p4.s
-; CHECK-NEXT:    cmphi p7.d, p0/z, z7.d, z5.d
-; CHECK-NEXT:    uqadd z5.d, z6.d, z0.d
-; CHECK-NEXT:    uqadd z6.d, z25.d, z0.d
-; CHECK-NEXT:    uqadd z25.d, z27.d, z0.d
-; CHECK-NEXT:    cmphi p4.d, p0/z, z7.d, z1.d
+; CHECK-NEXT:    uqadd z24.d, z24.d, z0.d
+; CHECK-NEXT:    cmphi p7.d, p0/z, z3.d, z25.d
+; CHECK-NEXT:    cmphi p4.d, p0/z, z3.d, z1.d
 ; CHECK-NEXT:    uzp1 p5.s, p5.s, p6.s
-; CHECK-NEXT:    cmphi p6.d, p0/z, z7.d, z3.d
-; CHECK-NEXT:    cmphi p9.d, p0/z, z7.d, z4.d
-; CHECK-NEXT:    uqadd z0.d, z24.d, z0.d
-; CHECK-NEXT:    cmphi p2.d, p0/z, z7.d, z28.d
-; CHECK-NEXT:    cmphi p10.d, p0/z, z7.d, z6.d
+; CHECK-NEXT:    cmphi p6.d, p0/z, z3.d, z4.d
+; CHECK-NEXT:    cmphi p9.d, p0/z, z3.d, z5.d
+; CHECK-NEXT:    cmphi p10.d, p0/z, z3.d, z7.d
+; CHECK-NEXT:    uqadd z0.d, z26.d, z0.d
+; CHECK-NEXT:    cmphi p2.d, p0/z, z3.d, z28.d
 ; CHECK-NEXT:    uzp1 p4.s, p4.s, p8.s
-; CHECK-NEXT:    cmphi p8.d, p0/z, z7.d, z25.d
+; CHECK-NEXT:    cmphi p8.d, p0/z, z3.d, z24.d
 ; CHECK-NEXT:    uzp1 p6.s, p6.s, p9.s
 ; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    uzp1 p2.s, p7.s, p2.s
-; CHECK-NEXT:    cmphi p7.d, p0/z, z7.d, z5.d
-; CHECK-NEXT:    cmphi p0.d, p0/z, z7.d, z0.d
 ; CHECK-NEXT:    uzp1 p1.h, p1.h, p3.h
+; CHECK-NEXT:    uzp1 p2.s, p7.s, p2.s
+; CHECK-NEXT:    cmphi p7.d, p0/z, z3.d, z6.d
+; CHECK-NEXT:    cmphi p0.d, p0/z, z3.d, z0.d
 ; CHECK-NEXT:    uzp1 p7.s, p7.s, p10.s
 ; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    uzp1 p0.s, p8.s, p0.s

llvm/test/CodeGen/AArch64/arm64-addr-type-promotion.ll

@@ -10,28 +10,28 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 define zeroext i8 @fullGtU(i32 %i1, i32 %i2) {
 ; CHECK-LABEL: fullGtU:
 ; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:    adrp x8, _block@GOTPAGE
+; CHECK-NEXT:    adrp x9, _block@GOTPAGE
 ; CHECK-NEXT:    ; kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    ; kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    sxtw x9, w0
+; CHECK-NEXT:    sxtw x8, w0
 ; CHECK-NEXT:    sxtw x10, w1
-; CHECK-NEXT:    ldr x8, [x8, _block@GOTPAGEOFF]
-; CHECK-NEXT:    ldr x8, [x8]
-; CHECK-NEXT:    ldrb w11, [x8, x9]
-; CHECK-NEXT:    ldrb w12, [x8, x10]
+; CHECK-NEXT:    ldr x9, [x9, _block@GOTPAGEOFF]
+; CHECK-NEXT:    ldr x9, [x9]
+; CHECK-NEXT:    ldrb w11, [x9, x8]
+; CHECK-NEXT:    ldrb w12, [x9, x10]
 ; CHECK-NEXT:    cmp w11, w12
 ; CHECK-NEXT:    b.ne LBB0_3
 ; CHECK-NEXT:  ; %bb.1: ; %if.end
-; CHECK-NEXT:    add x9, x9, x8
-; CHECK-NEXT:    add x8, x10, x8
-; CHECK-NEXT:    ldrb w10, [x9, #1]
-; CHECK-NEXT:    ldrb w11, [x8, #1]
+; CHECK-NEXT:    add x8, x8, x9
+; CHECK-NEXT:    add x9, x10, x9
+; CHECK-NEXT:    ldrb w10, [x8, #1]
+; CHECK-NEXT:    ldrb w11, [x9, #1]
 ; CHECK-NEXT:    cmp w10, w11
 ; CHECK-NEXT:    b.ne LBB0_3
 ; CHECK-NEXT:  ; %bb.2: ; %if.end25
-; CHECK-NEXT:    ldrb w9, [x9, #2]
 ; CHECK-NEXT:    ldrb w8, [x8, #2]
-; CHECK-NEXT:    cmp w9, w8
+; CHECK-NEXT:    ldrb w9, [x9, #2]
+; CHECK-NEXT:    cmp w8, w9
 ; CHECK-NEXT:    cset w8, hi
 ; CHECK-NEXT:    csel w0, wzr, w8, eq
 ; CHECK-NEXT:    ret

llvm/test/CodeGen/AArch64/arm64-cse.ll

@@ -8,16 +8,16 @@ target triple = "arm64-apple-ios"
 define ptr @t1(ptr %base, ptr nocapture %offset, i32 %size) nounwind {
 ; CHECK-LABEL: t1:
 ; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:    ldr w9, [x1]
-; CHECK-NEXT:    subs w8, w9, w2
+; CHECK-NEXT:    ldr w8, [x1]
+; CHECK-NEXT:    subs w9, w8, w2
 ; CHECK-NEXT:    b.ge LBB0_2
 ; CHECK-NEXT:  ; %bb.1:
 ; CHECK-NEXT:    mov x0, xzr
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  LBB0_2: ; %if.end
-; CHECK-NEXT:    add x0, x0, w8, sxtw
-; CHECK-NEXT:    sub w9, w9, w8
-; CHECK-NEXT:    str w9, [x1]
+; CHECK-NEXT:    add x0, x0, w9, sxtw
+; CHECK-NEXT:    sub w8, w8, w9
+; CHECK-NEXT:    str w8, [x1]
 ; CHECK-NEXT:    ret
 entry:
  %0 = load i32, ptr %offset, align 4

llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll

@@ -1059,30 +1059,30 @@ define void @stack_realign2(i32 %a, i32 %b, ptr %ptr1, ptr %ptr2, ptr %ptr3, ptr
 ; ENABLE-NEXT:    .cfi_offset w27, -88
 ; ENABLE-NEXT:    .cfi_offset w28, -96
 ; ENABLE-NEXT:    lsl w8, w1, w0
-; ENABLE-NEXT:    lsr w10, w0, w1
-; ENABLE-NEXT:    lsl w16, w0, w1
+; ENABLE-NEXT:    lsr w9, w0, w1
+; ENABLE-NEXT:    lsl w14, w0, w1
 ; ENABLE-NEXT:    lsr w11, w1, w0
-; ENABLE-NEXT:    add w14, w1, w0
-; ENABLE-NEXT:    sub w9, w8, w10
+; ENABLE-NEXT:    add w15, w1, w0
+; ENABLE-NEXT:    sub w10, w8, w9
 ; ENABLE-NEXT:    subs w17, w1, w0
-; ENABLE-NEXT:    add w15, w16, w8
-; ENABLE-NEXT:    add w12, w10, w11
-; ENABLE-NEXT:    add w13, w11, w14
+; ENABLE-NEXT:    add w16, w14, w8
+; ENABLE-NEXT:    add w12, w9, w11
+; ENABLE-NEXT:    add w13, w11, w15
 ; ENABLE-NEXT:    b.le LBB14_2
 ; ENABLE-NEXT:  ; %bb.1: ; %true
 ; ENABLE-NEXT:    str w0, [sp]
 ; ENABLE-NEXT:    ; InlineAsm Start
 ; ENABLE-NEXT:    nop
 ; ENABLE-NEXT:    ; InlineAsm End
 ; ENABLE-NEXT:  LBB14_2: ; %false
-; ENABLE-NEXT:    str w16, [x2]
+; ENABLE-NEXT:    str w14, [x2]
 ; ENABLE-NEXT:    str w8, [x3]
-; ENABLE-NEXT:    str w10, [x4]
+; ENABLE-NEXT:    str w9, [x4]
 ; ENABLE-NEXT:    str w11, [x5]
-; ENABLE-NEXT:    str w14, [x6]
+; ENABLE-NEXT:    str w15, [x6]
 ; ENABLE-NEXT:    str w17, [x7]
 ; ENABLE-NEXT:    stp w0, w1, [x2, #4]
-; ENABLE-NEXT:    stp w15, w9, [x2, #12]
+; ENABLE-NEXT:    stp w16, w10, [x2, #12]
 ; ENABLE-NEXT:    stp w12, w13, [x2, #20]
 ; ENABLE-NEXT:    sub sp, x29, #80
 ; ENABLE-NEXT:    ldp x29, x30, [sp, #80] ; 16-byte Folded Reload
@@ -1118,30 +1118,30 @@ define void @stack_realign2(i32 %a, i32 %b, ptr %ptr1, ptr %ptr2, ptr %ptr3, ptr
 ; DISABLE-NEXT:    .cfi_offset w27, -88
 ; DISABLE-NEXT:    .cfi_offset w28, -96
 ; DISABLE-NEXT:    lsl w8, w1, w0
-; DISABLE-NEXT:    lsr w10, w0, w1
-; DISABLE-NEXT:    lsl w16, w0, w1
+; DISABLE-NEXT:    lsr w9, w0, w1
+; DISABLE-NEXT:    lsl w14, w0, w1
 ; DISABLE-NEXT:    lsr w11, w1, w0
-; DISABLE-NEXT:    add w14, w1, w0
-; DISABLE-NEXT:    sub w9, w8, w10
+; DISABLE-NEXT:    add w15, w1, w0
+; DISABLE-NEXT:    sub w10, w8, w9
 ; DISABLE-NEXT:    subs w17, w1, w0
-; DISABLE-NEXT:    add w15, w16, w8
-; DISABLE-NEXT:    add w12, w10, w11
-; DISABLE-NEXT:    add w13, w11, w14
+; DISABLE-NEXT:    add w16, w14, w8
+; DISABLE-NEXT:    add w12, w9, w11
+; DISABLE-NEXT:    add w13, w11, w15
 ; DISABLE-NEXT:    b.le LBB14_2
 ; DISABLE-NEXT:  ; %bb.1: ; %true
 ; DISABLE-NEXT:    str w0, [sp]
 ; DISABLE-NEXT:    ; InlineAsm Start
 ; DISABLE-NEXT:    nop
 ; DISABLE-NEXT:    ; InlineAsm End
 ; DISABLE-NEXT:  LBB14_2: ; %false
-; DISABLE-NEXT:    str w16, [x2]
+; DISABLE-NEXT:    str w14, [x2]
 ; DISABLE-NEXT:    str w8, [x3]
-; DISABLE-NEXT:    str w10, [x4]
+; DISABLE-NEXT:    str w9, [x4]
 ; DISABLE-NEXT:    str w11, [x5]
-; DISABLE-NEXT:    str w14, [x6]
+; DISABLE-NEXT:    str w15, [x6]
 ; DISABLE-NEXT:    str w17, [x7]
 ; DISABLE-NEXT:    stp w0, w1, [x2, #4]
-; DISABLE-NEXT:    stp w15, w9, [x2, #12]
+; DISABLE-NEXT:    stp w16, w10, [x2, #12]
 ; DISABLE-NEXT:    stp w12, w13, [x2, #20]
 ; DISABLE-NEXT:    sub sp, x29, #80
 ; DISABLE-NEXT:    ldp x29, x30, [sp, #80] ; 16-byte Folded Reload

llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll

@@ -26,25 +26,25 @@ define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    zip1 z1.d, z1.d, z1.d
 ; CHECK-NEXT:  .LBB0_1: // %vector.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    zip2 p2.d, p1.d, p1.d
+; CHECK-NEXT:    zip2 p3.d, p1.d, p1.d
 ; CHECK-NEXT:    add x13, x0, x8
 ; CHECK-NEXT:    add x14, x1, x8
-; CHECK-NEXT:    zip1 p3.d, p1.d, p1.d
+; CHECK-NEXT:    zip1 p2.d, p1.d, p1.d
 ; CHECK-NEXT:    mov z6.d, z1.d
 ; CHECK-NEXT:    mov z7.d, z0.d
 ; CHECK-NEXT:    whilelo p1.d, x12, x9
 ; CHECK-NEXT:    add x8, x8, x11
 ; CHECK-NEXT:    add x12, x12, x10
-; CHECK-NEXT:    ld1d { z2.d }, p2/z, [x13, #1, mul vl]
-; CHECK-NEXT:    ld1d { z3.d }, p3/z, [x13]
-; CHECK-NEXT:    ld1d { z4.d }, p2/z, [x14, #1, mul vl]
-; CHECK-NEXT:    ld1d { z5.d }, p3/z, [x14]
+; CHECK-NEXT:    ld1d { z2.d }, p3/z, [x13, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p2/z, [x13]
+; CHECK-NEXT:    ld1d { z4.d }, p3/z, [x14, #1, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p2/z, [x14]
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #0
 ; CHECK-NEXT:    fcmla z7.d, p0/m, z4.d, z2.d, #0
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #90
 ; CHECK-NEXT:    fcmla z7.d, p0/m, z4.d, z2.d, #90
-; CHECK-NEXT:    mov z0.d, p2/m, z7.d
-; CHECK-NEXT:    mov z1.d, p3/m, z6.d
+; CHECK-NEXT:    mov z0.d, p3/m, z7.d
+; CHECK-NEXT:    mov z1.d, p2/m, z6.d
 ; CHECK-NEXT:    b.mi .LBB0_1
 ; CHECK-NEXT:  // %bb.2: // %exit.block
 ; CHECK-NEXT:    uzp1 z2.d, z1.d, z0.d
@@ -237,19 +237,19 @@ define %"class.std::complex" @complex_mul_predicated_x2_v2f64(ptr %a, ptr %b, pt
 ; CHECK-NEXT:    add x9, x9, x11
 ; CHECK-NEXT:    add x8, x8, x12
 ; CHECK-NEXT:    cmpne p1.d, p1/z, z2.d, #0
-; CHECK-NEXT:    zip2 p2.d, p1.d, p1.d
-; CHECK-NEXT:    zip1 p3.d, p1.d, p1.d
+; CHECK-NEXT:    zip2 p3.d, p1.d, p1.d
+; CHECK-NEXT:    zip1 p2.d, p1.d, p1.d
 ; CHECK-NEXT:    whilelo p1.d, x9, x10
-; CHECK-NEXT:    ld1d { z2.d }, p2/z, [x13, #1, mul vl]
-; CHECK-NEXT:    ld1d { z3.d }, p3/z, [x13]
-; CHECK-NEXT:    ld1d { z4.d }, p2/z, [x14, #1, mul vl]
-; CHECK-NEXT:    ld1d { z5.d }, p3/z, [x14]
+; CHECK-NEXT:    ld1d { z2.d }, p3/z, [x13, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p2/z, [x13]
+; CHECK-NEXT:    ld1d { z4.d }, p3/z, [x14, #1, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p2/z, [x14]
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #0
 ; CHECK-NEXT:    fcmla z7.d, p0/m, z4.d, z2.d, #0
 ; CHECK-NEXT:    fcmla z6.d, p0/m, z5.d, z3.d, #90
 ; CHECK-NEXT:    fcmla z7.d, p0/m, z4.d, z2.d, #90
-; CHECK-NEXT:    mov z0.d, p2/m, z7.d
-; CHECK-NEXT:    mov z1.d, p3/m, z6.d
+; CHECK-NEXT:    mov z0.d, p3/m, z7.d
+; CHECK-NEXT:    mov z1.d, p2/m, z6.d
 ; CHECK-NEXT:    b.mi .LBB2_1
 ; CHECK-NEXT:  // %bb.2: // %exit.block
 ; CHECK-NEXT:    uzp1 z2.d, z1.d, z0.d