diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 660a6ef574576..27d67066f61df 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8933,14 +8933,15 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF, for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) { VFRange SubRange = {VF, MaxVFTimes2}; if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange)) { + bool HasScalarVF = Plan->hasVF(ElementCount::getFixed(1)); // Now optimize the initial VPlan. - if (!Plan->hasVF(ElementCount::getFixed(1))) + if (!HasScalarVF) VPlanTransforms::runPass(VPlanTransforms::truncateToMinimalBitwidths, *Plan, CM.getMinimalBitwidths()); VPlanTransforms::optimize(*Plan); // TODO: try to put it close to addActiveLaneMask(). // Discard the plan if it is not EVL-compatible - if (CM.foldTailWithEVL() && + if (CM.foldTailWithEVL() && !HasScalarVF && !VPlanTransforms::runPass(VPlanTransforms::tryAddExplicitVectorLength, *Plan, CM.getMaxSafeElements())) break; diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll index a360339556d28..81a2675dc6583 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll @@ -6,8 +6,8 @@ ; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -62,8 +62,8 @@ exit: } define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -118,8 +118,8 @@ exit: } define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -174,8 +174,8 @@ exit: } define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -230,8 +230,8 @@ exit: } define void @vp_ctlz(ptr %a, ptr %b, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -281,8 +281,8 @@ exit: } define void @vp_cttz(ptr %a, ptr %b, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -332,8 +332,8 @@ exit: } define void @vp_lrint(ptr %a, ptr %b, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -387,8 +387,8 @@ exit: } define void @vp_llrint(ptr %a, ptr %b, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -442,8 +442,8 @@ exit: } define void @vp_abs(ptr %a, ptr %b, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll index d64a24bd29b89..40cf02c4663fe 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll @@ -5,8 +5,8 @@ ; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s define void @vp_sext(ptr %a, ptr %b, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -58,8 +58,8 @@ exit: } define void @vp_zext(ptr %a, ptr %b, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -109,8 +109,8 @@ exit: } define void @vp_trunc(ptr %a, ptr %b, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -160,8 +160,8 @@ exit: } define void @vp_fpext(ptr %a, ptr %b, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -211,8 +211,8 @@ exit: } define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -262,8 +262,8 @@ exit: } define void @vp_sitofp(ptr %a, ptr %b, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -313,8 +313,8 @@ exit: } define void @vp_uitofp(ptr %a, ptr %b, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -364,8 +364,8 @@ exit: } define void @vp_fptosi(ptr %a, ptr %b, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -415,8 +415,8 @@ exit: } define void @vp_fptoui(ptr %a, ptr %b, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF @@ -466,8 +466,8 @@ exit: } define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll index aacf6327add7d..0daf15d04af35 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll @@ -24,8 +24,8 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; IF-EVL-OUTLOOP: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-OUTLOOP-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll index 5824a96169a4b..e64ea387d4582 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll @@ -11,8 +11,8 @@ ; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefixes=NO-VP,CHECK %s define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) { -; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' -; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI +; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' +; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll index 74e250394ce12..90de5b6bfd1d3 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll @@ -6,8 +6,8 @@ ; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s define void @vp_select(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) { - ; IF-EVL: VPlan 'Initial VPlan for VF={1},UF={1}' - ; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI + ; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' + ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' ; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ;