Skip to content

Commit eb31272

Browse files
fhahnfhahn
committed
[VPlan] Add VPValue for VF, use it for VPWidenIntOrFpInductionRecipe.
Similar to VFxUF, also add a VF VPValue to VPlan and use it to get the runtime VF in VPWidenIntOrFpInductionRecipe. Code for VF is only generated if there are users of VF, to avoid unnecessary test changes. Note: some tests still need updating, will do once we converge on a final version of the patch.
1 parent 98174fb commit eb31272

12 files changed

+63
-69
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8069,10 +8069,12 @@ createWidenInductionRecipes(PHINode *Phi, Instruction *PhiOrTrunc,
80698069
VPValue *Step =
80708070
vputils::getOrCreateVPValueForSCEVExpr(Plan, IndDesc.getStep(), SE);
80718071
if (auto *TruncI = dyn_cast<TruncInst>(PhiOrTrunc)) {
8072-
return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, IndDesc, TruncI);
8072+
return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, Plan.getVF(),
8073+
IndDesc, TruncI);
80738074
}
80748075
assert(isa<PHINode>(PhiOrTrunc) && "must be a phi node here");
8075-
return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, IndDesc);
8076+
return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, Plan.getVF(),
8077+
IndDesc);
80768078
}
80778079

80788080
VPHeaderPHIRecipe *VPRecipeBuilder::tryToOptimizeInductionPHI(
@@ -8487,6 +8489,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW,
84878489
VPBasicBlock *Header = TopRegion->getEntryBasicBlock();
84888490
Header->insert(CanonicalIVPHI, Header->begin());
84898491

8492+
VPBuilder PhBuilder(cast<VPBasicBlock>(TopRegion->getSinglePredecessor()));
84908493
VPBuilder Builder(TopRegion->getExitingBasicBlock());
84918494
// Add a VPInstruction to increment the scalar canonical IV by VF * UF.
84928495
auto *CanonicalIVIncrement = Builder.createOverflowingOp(

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -814,8 +814,21 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
814814

815815
IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());
816816
// FIXME: Model VF * UF computation completely in VPlan.
817-
VFxUF.setUnderlyingValue(
818-
createStepForVF(Builder, TripCountV->getType(), State.VF, State.UF));
817+
Value *RuntimeVF = nullptr;
818+
if (VF.getNumUsers()) {
819+
RuntimeVF = createStepForVF(Builder, TripCountV->getType(), State.VF, 1);
820+
VF.setUnderlyingValue(RuntimeVF);
821+
}
822+
if (RuntimeVF) {
823+
VFxUF.setUnderlyingValue(
824+
State.UF > 1 ? Builder.CreateMul(
825+
VF.getLiveInIRValue(),
826+
ConstantInt::get(TripCountV->getType(), State.UF))
827+
: VF.getLiveInIRValue());
828+
} else {
829+
VFxUF.setUnderlyingValue(
830+
createStepForVF(Builder, TripCountV->getType(), State.VF, State.UF));
831+
}
819832

820833
// When vectorizing the epilogue loop, the canonical induction start value
821834
// needs to be changed from zero to the value after the main vector loop.
@@ -1067,6 +1080,7 @@ VPlan *VPlan::duplicate() {
10671080
}
10681081
Old2NewVPValues[&VectorTripCount] = &NewPlan->VectorTripCount;
10691082
Old2NewVPValues[&VFxUF] = &NewPlan->VFxUF;
1083+
Old2NewVPValues[&VF] = &NewPlan->VF;
10701084
if (BackedgeTakenCount) {
10711085
NewPlan->BackedgeTakenCount = new VPValue();
10721086
Old2NewVPValues[BackedgeTakenCount] = NewPlan->BackedgeTakenCount;

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1705,25 +1705,27 @@ class VPWidenIntOrFpInductionRecipe : public VPHeaderPHIRecipe {
17051705

17061706
public:
17071707
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step,
1708-
const InductionDescriptor &IndDesc)
1708+
VPValue *VF, const InductionDescriptor &IndDesc)
17091709
: VPHeaderPHIRecipe(VPDef::VPWidenIntOrFpInductionSC, IV, Start), IV(IV),
17101710
Trunc(nullptr), IndDesc(IndDesc) {
17111711
addOperand(Step);
1712+
addOperand(VF);
17121713
}
17131714

17141715
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step,
1715-
const InductionDescriptor &IndDesc,
1716+
VPValue *VF, const InductionDescriptor &IndDesc,
17161717
TruncInst *Trunc)
17171718
: VPHeaderPHIRecipe(VPDef::VPWidenIntOrFpInductionSC, Trunc, Start),
17181719
IV(IV), Trunc(Trunc), IndDesc(IndDesc) {
17191720
addOperand(Step);
1721+
addOperand(VF);
17201722
}
17211723

17221724
~VPWidenIntOrFpInductionRecipe() override = default;
17231725

17241726
VPWidenIntOrFpInductionRecipe *clone() override {
1725-
return new VPWidenIntOrFpInductionRecipe(IV, getStartValue(),
1726-
getStepValue(), IndDesc, Trunc);
1727+
return new VPWidenIntOrFpInductionRecipe(
1728+
IV, getStartValue(), getStepValue(), getOperand(2), IndDesc, Trunc);
17271729
}
17281730

17291731
VP_CLASSOF_IMPL(VPDef::VPWidenIntOrFpInductionSC)
@@ -3142,6 +3144,8 @@ class VPlan {
31423144
/// Represents the vector trip count.
31433145
VPValue VectorTripCount;
31443146

3147+
VPValue VF;
3148+
31453149
/// Represents the loop-invariant VF * UF of the vector loop region.
31463150
VPValue VFxUF;
31473151

@@ -3233,6 +3237,8 @@ class VPlan {
32333237
/// Returns VF * UF of the vector loop region.
32343238
VPValue &getVFxUF() { return VFxUF; }
32353239

3240+
VPValue *getVF() { return &VF; };
3241+
32363242
void addVF(ElementCount VF) { VFs.insert(VF); }
32373243

32383244
void setVF(ElementCount VF) {

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1217,11 +1217,11 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
12171217
// Multiply the vectorization factor by the step using integer or
12181218
// floating-point arithmetic as appropriate.
12191219
Type *StepType = Step->getType();
1220-
Value *RuntimeVF;
1220+
Value *RuntimeVF = State.get(getOperand(2), {0, 0});
12211221
if (Step->getType()->isFloatingPointTy())
12221222
RuntimeVF = getRuntimeVFAsFloat(Builder, StepType, State.VF);
12231223
else
1224-
RuntimeVF = getRuntimeVF(Builder, StepType, State.VF);
1224+
RuntimeVF = Builder.CreateZExtOrTrunc(RuntimeVF, StepType);
12251225
Value *Mul = Builder.CreateBinOp(MulOp, Step, RuntimeVF);
12261226

12271227
// Create a vector splat to use in the induction update.

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
5555
VPValue *Start = Plan->getOrAddLiveIn(II->getStartValue());
5656
VPValue *Step =
5757
vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE);
58-
NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II);
58+
NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, Step,
59+
Plan->getVF(), *II);
5960
} else {
6061
assert(isa<VPInstruction>(&Ingredient) &&
6162
"only VPInstructions expected here");

‎llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,7 @@ define void @foo() {
2424
; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 4 x i64> [[TMP4]], zeroinitializer
2525
; CHECK-NEXT: [[TMP6:%.*]] = mul <vscale x 4 x i64> [[TMP5]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
2626
; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP6]]
27-
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
28-
; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4
29-
; CHECK-NEXT: [[TMP9:%.*]] = mul i64 1, [[TMP8]]
27+
; CHECK-NEXT: [[TMP9:%.*]] = mul i64 1, [[TMP19]]
3028
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
3129
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
3230
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ define void @induction_i7(ptr %dst) #0 {
1313
; CHECK: vector.ph:
1414
; CHECK: %ind.end = trunc i64 %n.vec to i7
1515
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
16-
; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
16+
; CHECK-NEXT: [[TMP40:%.*]] = mul i64 [[TMP4]], 2
17+
; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP40]], 2
1718
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i8> @llvm.experimental.stepvector.nxv2i8()
1819
; CHECK-NEXT: [[TMP7:%.*]] = trunc <vscale x 2 x i8> [[TMP6]] to <vscale x 2 x i7>
1920
; CHECK-NEXT: [[TMP8:%.*]] = add <vscale x 2 x i7> [[TMP7]], zeroinitializer
@@ -73,7 +74,8 @@ define void @induction_i3_zext(ptr %dst) #0 {
7374
; CHECK: vector.ph:
7475
; CHECK: %ind.end = trunc i64 %n.vec to i3
7576
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
76-
; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
77+
; CHECK-NEXT: [[TMP40:%.*]] = mul i64 [[TMP4]], 2
78+
; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP40]], 2
7779
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i8> @llvm.experimental.stepvector.nxv2i8()
7880
; CHECK-NEXT: [[TMP7:%.*]] = trunc <vscale x 2 x i8> [[TMP6]] to <vscale x 2 x i3>
7981
; CHECK-NEXT: [[TMP8:%.*]] = add <vscale x 2 x i3> [[TMP7]], zeroinitializer

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll

Lines changed: 13 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,7 @@ define void @test_array_load2_i16_store2(i32 %C, i32 %D) #1 {
110110
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
111111
; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
112112
; CHECK-NEXT: [[TMP3:%.*]] = shl <vscale x 4 x i64> [[TMP2]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
113-
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
114-
; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3
113+
; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP0]], 3
115114
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP5]], i64 0
116115
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
117116
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[C:%.*]], i64 0
@@ -201,8 +200,7 @@ define void @test_array_load2_store2_i16(i32 noundef %C, i32 noundef %D) #1 {
201200
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
202201
; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
203202
; CHECK-NEXT: [[TMP3:%.*]] = shl <vscale x 4 x i64> [[TMP2]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
204-
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
205-
; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3
203+
; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP0]], 3
206204
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP5]], i64 0
207205
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
208206
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[C:%.*]], i64 0
@@ -280,9 +278,7 @@ define i32 @test_struct_load6(ptr %S) #1 {
280278
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
281279
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
282280
; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
283-
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
284-
; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
285-
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP4]], i64 0
281+
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP1]], i64 0
286282
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
287283
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
288284
; CHECK: vector.body:
@@ -385,8 +381,8 @@ define void @test_reversed_load2_store2(ptr noalias nocapture readonly %A, ptr n
385381
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
386382
; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.experimental.stepvector.nxv4i32()
387383
; CHECK-NEXT: [[INDUCTION:%.*]] = sub <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1023, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer), [[TMP2]]
388-
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
389-
; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i32 [[TMP3]], -4
384+
; CHECK-NEXT: [[TMP3:%.*]] = trunc nuw nsw i64 [[TMP1]] to i32
385+
; CHECK-NEXT: [[DOTNEG:%.*]] = sub nsw i32 0, [[TMP3]]
390386
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTNEG]], i64 0
391387
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
392388
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -644,8 +640,7 @@ define void @load_gap_reverse(ptr noalias nocapture readonly %P1, ptr noalias no
644640
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
645641
; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
646642
; CHECK-NEXT: [[INDUCTION:%.*]] = sub <vscale x 4 x i64> shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1023, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer), [[TMP2]]
647-
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
648-
; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP3]], -4
643+
; CHECK-NEXT: [[DOTNEG:%.*]] = sub nsw i64 0, [[TMP1]]
649644
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[DOTNEG]], i64 0
650645
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
651646
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[X:%.*]], i64 0
@@ -895,9 +890,7 @@ define void @PR27626_0(ptr %p, i32 %z, i64 %n) #1 {
895890
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
896891
; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
897892
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
898-
; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
899-
; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 2
900-
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP11]], i64 0
893+
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
901894
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
902895
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z:%.*]], i64 0
903896
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
@@ -982,9 +975,7 @@ define i32 @PR27626_1(ptr %p, i64 %n) #1 {
982975
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
983976
; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
984977
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
985-
; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
986-
; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 2
987-
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP11]], i64 0
978+
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
988979
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
989980
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
990981
; CHECK: vector.body:
@@ -1077,9 +1068,7 @@ define void @PR27626_2(ptr %p, i64 %n, i32 %z) #1 {
10771068
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
10781069
; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
10791070
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
1080-
; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
1081-
; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 2
1082-
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP11]], i64 0
1071+
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
10831072
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
10841073
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z:%.*]], i64 0
10851074
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
@@ -1167,9 +1156,7 @@ define i32 @PR27626_3(ptr %p, i64 %n, i32 %z) #1 {
11671156
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
11681157
; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
11691158
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
1170-
; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
1171-
; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 2
1172-
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP11]], i64 0
1159+
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
11731160
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
11741161
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
11751162
; CHECK: vector.body:
@@ -1271,8 +1258,7 @@ define void @PR27626_4(ptr %a, i32 %x, i32 %y, i32 %z, i64 %n) #1 {
12711258
; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2
12721259
; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
12731260
; CHECK-NEXT: [[TMP9:%.*]] = shl <vscale x 4 x i64> [[TMP8]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
1274-
; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
1275-
; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 3
1261+
; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP6]], 3
12761262
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP11]], i64 0
12771263
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
12781264
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X:%.*]], i64 0
@@ -1368,8 +1354,7 @@ define void @PR27626_5(ptr %a, i32 %x, i32 %y, i32 %z, i64 %n) #1 {
13681354
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
13691355
; CHECK-NEXT: [[TMP10:%.*]] = shl <vscale x 4 x i64> [[TMP9]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
13701356
; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[TMP10]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 3, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
1371-
; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
1372-
; CHECK-NEXT: [[TMP12:%.*]] = shl nuw nsw i64 [[TMP11]], 3
1357+
; CHECK-NEXT: [[TMP12:%.*]] = shl nuw nsw i64 [[TMP7]], 3
13731358
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP12]], i64 0
13741359
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
13751360
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X:%.*]], i64 0
@@ -1481,8 +1466,7 @@ define void @PR34743(ptr %a, ptr %b, i64 %n) #1 {
14811466
; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[DOTPRE]], i32 [[TMP13]]
14821467
; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
14831468
; CHECK-NEXT: [[TMP15:%.*]] = shl <vscale x 4 x i64> [[TMP14]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
1484-
; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
1485-
; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i64 [[TMP16]], 3
1469+
; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i64 [[TMP9]], 3
14861470
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP17]], i64 0
14871471
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
14881472
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]

0 commit comments

Comments
 (0)