Skip to content

Commit d209084

Browse files
fhahnfhahn
committed
[VPlan] Replace versioned stride with constant during VPlan opts.
After constructing the initial VPlan, replace VPValues for versioned strides with their constant counterparts. Differential Revision: https://reviews.llvm.org/D147783
1 parent 4acbd44 commit d209084

File tree

4 files changed

+40
-31
lines changed

4 files changed

+40
-31
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9116,6 +9116,19 @@ std::optional<VPlanPtr> LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
91169116
Plan->addVF(VF);
91179117
Plan->setName("Initial VPlan");
91189118

9119+
// Replace VPValues for known constant strides guaranteed by predicate scalar
9120+
// evolution.
9121+
for (auto [_, Stride] : Legal->getLAI()->getSymbolicStrides()) {
9122+
auto *StrideV = cast<SCEVUnknown>(Stride)->getValue();
9123+
auto *ScevStride = cast<SCEVConstant>(PSE.getSCEV(StrideV));
9124+
Constant *CI = ConstantInt::get(Stride->getType(), ScevStride->getAPInt());
9125+
9126+
auto *ConstVPV = Plan->getVPValueOrAddLiveIn(CI);
9127+
// The versioned value may not be used in the loop directly, so just add a
9128+
// new live-in in those cases.
9129+
Plan->getVPValueOrAddLiveIn(StrideV)->replaceAllUsesWith(ConstVPV);
9130+
}
9131+
91199132
// From this point onwards, VPlan-to-VPlan transformations may change the plan
91209133
// in ways that accessing values using original IR values is incorrect.
91219134
Plan->disableValue2VPValue();

‎llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -341,7 +341,7 @@ define void @test_stride_loopinvar_4i32(ptr readonly %data, ptr noalias nocaptur
341341
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
342342
; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0
343343
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[TMP0]], i32 [[N]])
344-
; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw i32 [[TMP0]], [[STRIDE]]
344+
; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw i32 [[TMP0]], 1
345345
; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 2
346346
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DATA:%.*]], i32 [[TMP2]]
347347
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i32 0

‎llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll

Lines changed: 24 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,7 @@ define void @single_stride_int_scaled(ptr %p, i64 %stride) {
233233
; NOSTRIDED: vector.body:
234234
; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
235235
; NOSTRIDED-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0
236-
; NOSTRIDED-NEXT: [[TMP6:%.*]] = mul nuw nsw i64 [[TMP5]], [[STRIDE]]
236+
; NOSTRIDED-NEXT: [[TMP6:%.*]] = mul nuw nsw i64 [[TMP5]], 1
237237
; NOSTRIDED-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
238238
; NOSTRIDED-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i32 0
239239
; NOSTRIDED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
@@ -317,19 +317,17 @@ define void @single_stride_int_iv(ptr %p, i64 %stride) {
317317
; NOSTRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
318318
; NOSTRIDED: vector.body:
319319
; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
320-
; NOSTRIDED-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[STRIDE]]
321-
; NOSTRIDED-NEXT: [[TMP5:%.*]] = mul i64 0, [[STRIDE]]
322-
; NOSTRIDED-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], [[TMP5]]
323-
; NOSTRIDED-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
324-
; NOSTRIDED-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i32 0
325-
; NOSTRIDED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
326-
; NOSTRIDED-NEXT: [[TMP9:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
327-
; NOSTRIDED-NEXT: store <vscale x 4 x i32> [[TMP9]], ptr [[TMP8]], align 4
328-
; NOSTRIDED-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
329-
; NOSTRIDED-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4
330-
; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
331-
; NOSTRIDED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
332-
; NOSTRIDED-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
320+
; NOSTRIDED-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0
321+
; NOSTRIDED-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP5]]
322+
; NOSTRIDED-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i32 0
323+
; NOSTRIDED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
324+
; NOSTRIDED-NEXT: [[TMP8:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
325+
; NOSTRIDED-NEXT: store <vscale x 4 x i32> [[TMP8]], ptr [[TMP7]], align 4
326+
; NOSTRIDED-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
327+
; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4
328+
; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
329+
; NOSTRIDED-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
330+
; NOSTRIDED-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
333331
; NOSTRIDED: middle.block:
334332
; NOSTRIDED-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
335333
; NOSTRIDED-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -452,7 +450,7 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
452450
; NOSTRIDED: vector.body:
453451
; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
454452
; NOSTRIDED-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 0
455-
; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul nuw nsw i64 [[TMP9]], [[STRIDE]]
453+
; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul nuw nsw i64 [[TMP9]], 1
456454
; NOSTRIDED-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP10]]
457455
; NOSTRIDED-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[TMP11]], i32 0
458456
; NOSTRIDED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP12]], align 4
@@ -601,19 +599,17 @@ define void @double_stride_int_iv(ptr %p, ptr %p2, i64 %stride) {
601599
; NOSTRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
602600
; NOSTRIDED: vector.body:
603601
; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
604-
; NOSTRIDED-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[STRIDE]]
605-
; NOSTRIDED-NEXT: [[TMP5:%.*]] = mul i64 0, [[STRIDE]]
606-
; NOSTRIDED-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], [[TMP5]]
607-
; NOSTRIDED-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
608-
; NOSTRIDED-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i32 0
609-
; NOSTRIDED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
610-
; NOSTRIDED-NEXT: [[TMP9:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
611-
; NOSTRIDED-NEXT: store <vscale x 4 x i32> [[TMP9]], ptr [[TMP8]], align 4
612-
; NOSTRIDED-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
613-
; NOSTRIDED-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4
614-
; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
615-
; NOSTRIDED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
616-
; NOSTRIDED-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
602+
; NOSTRIDED-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0
603+
; NOSTRIDED-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP5]]
604+
; NOSTRIDED-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i32 0
605+
; NOSTRIDED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
606+
; NOSTRIDED-NEXT: [[TMP8:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
607+
; NOSTRIDED-NEXT: store <vscale x 4 x i32> [[TMP8]], ptr [[TMP7]], align 4
608+
; NOSTRIDED-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
609+
; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4
610+
; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
611+
; NOSTRIDED-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
612+
; NOSTRIDED-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
617613
; NOSTRIDED: middle.block:
618614
; NOSTRIDED-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
619615
; NOSTRIDED-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]

‎llvm/test/Transforms/LoopVectorize/runtime-check-needed-but-empty.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,12 @@ define void @test(ptr %A, i32 %x) {
1717
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 0
1818
; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
1919
; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
20-
; CHECK-NEXT: [[TMP5:%.*]] = mul i32 [[TMP4]], [[X]]
20+
; CHECK-NEXT: [[TMP5:%.*]] = mul i32 [[TMP4]], 1
2121
; CHECK-NEXT: [[TMP6:%.*]] = zext i32 [[TMP5]] to i64
2222
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[TMP6]]
2323
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 0
2424
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP8]], align 4
25-
; CHECK-NEXT: [[TMP9:%.*]] = mul i32 [[TMP1]], [[X]]
25+
; CHECK-NEXT: [[TMP9:%.*]] = mul i32 [[TMP1]], 1
2626
; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP9]] to i64
2727
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP10]]
2828
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i32 0

0 commit comments

Comments
 (0)