From 28e4d5c8b4505719a4cfd773c57ba0292934d1bc Mon Sep 17 00:00:00 2001 From: LiqinWeng Date: Thu, 12 Sep 2024 17:24:35 +0800 Subject: [PATCH 01/11] [LV][EVL] Support cast instruction with EVL-vectorization --- .../Transforms/Vectorize/LoopVectorize.cpp | 1 + llvm/lib/Transforms/Vectorize/VPlan.h | 77 +++++- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 47 ++++ .../Transforms/Vectorize/VPlanTransforms.cpp | 9 + llvm/lib/Transforms/Vectorize/VPlanValue.h | 1 + .../LoopVectorize/RISCV/inloop-reduction.ll | 18 +- .../RISCV/vplan-vp-cast-intrinsics.ll | 227 ++++++++++++++++++ 7 files changed, 366 insertions(+), 14 deletions(-) create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index d68a26251ac9d..6b6d338cdf8eb 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4516,6 +4516,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF, case VPDef::VPWidenCallSC: case VPDef::VPWidenCanonicalIVSC: case VPDef::VPWidenCastSC: + case VPDef::VPWidenCastEVLSC: case VPDef::VPWidenGEPSC: case VPDef::VPWidenIntrinsicSC: case VPDef::VPWidenSC: diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 1b1630ebc6c23..e5035a319f344 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -866,6 +866,7 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue { case VPRecipeBase::VPWidenCallSC: case VPRecipeBase::VPWidenCanonicalIVSC: case VPRecipeBase::VPWidenCastSC: + case VPRecipeBase::VPWidenCastEVLSC: case VPRecipeBase::VPWidenGEPSC: case VPRecipeBase::VPWidenIntrinsicSC: case VPRecipeBase::VPWidenSC: @@ -1063,6 +1064,7 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe { R->getVPDefID() == VPRecipeBase::VPWidenEVLSC || R->getVPDefID() == VPRecipeBase::VPWidenGEPSC || R->getVPDefID() == VPRecipeBase::VPWidenCastSC || + R->getVPDefID() == VPRecipeBase::VPWidenCastEVLSC || R->getVPDefID() == VPRecipeBase::VPReplicateSC || R->getVPDefID() == VPRecipeBase::VPReverseVectorPointerSC || R->getVPDefID() == VPRecipeBase::VPVectorPointerSC; @@ -1542,19 +1544,28 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags { /// Result type for the cast. Type *ResultTy; -public: - VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, - CastInst &UI) +protected: + VPWidenCastRecipe(unsigned VPDefOpcode, Instruction::CastOps Opcode, + VPValue *Op, Type *ResultTy, CastInst &UI) : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), Opcode(Opcode), ResultTy(ResultTy) { assert(UI.getOpcode() == Opcode && "opcode of underlying cast doesn't match"); } - VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy) + VPWidenCastRecipe(unsigned VPDefOpcode, Instruction::CastOps Opcode, + VPValue *Op, Type *ResultTy) : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op), Opcode(Opcode), ResultTy(ResultTy) {} +public: + VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, + CastInst &UI) + : VPWidenCastRecipe(VPDef::VPWidenCastSC, Opcode, Op, ResultTy, UI) {} + + VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy) + : VPWidenCastRecipe(VPDef::VPWidenCastSC, Opcode, Op, ResultTy) {} + ~VPWidenCastRecipe() override = default; VPWidenCastRecipe *clone() override { @@ -1565,7 +1576,15 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags { return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy); } - VP_CLASSOF_IMPL(VPDef::VPWidenCastSC) + static inline bool classof(const VPRecipeBase *R) { + return R->getVPDefID() == VPRecipeBase::VPWidenCastSC || + R->getVPDefID() == VPRecipeBase::VPWidenCastEVLSC; + } + + static inline bool classof(const VPUser *U) { + auto *R = dyn_cast(U); + return R && classof(R); + } /// Produce widened copies of the cast. void execute(VPTransformState &State) override; @@ -1586,6 +1605,54 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags { Type *getResultType() const { return ResultTy; } }; +// A recipe for widening cast operation with vector-predication intrinsics with +/// explicit vector length (EVL). +class VPWidenCastEVLRecipe : public VPWidenCastRecipe { + using VPRecipeWithIRFlags::transferFlags; + +public: + VPWidenCastEVLRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, + VPValue &EVL) + : VPWidenCastRecipe(VPDef::VPWidenCastEVLSC, Opcode, Op, ResultTy) { + addOperand(&EVL); + } + VPWidenCastEVLRecipe(VPWidenCastRecipe &W, VPValue &EVL) + : VPWidenCastEVLRecipe(W.getOpcode(), W.getOperand(0), W.getResultType(), + EVL) { + transferFlags(W); + } + + ~VPWidenCastEVLRecipe() override = default; + + VPWidenCastEVLRecipe *clone() final { + llvm_unreachable("VPWidenEVLRecipe cannot be cloned"); + return nullptr; + } + + VP_CLASSOF_IMPL(VPDef::VPWidenCastEVLSC) + + VPValue *getEVL() { return getOperand(getNumOperands() - 1); } + const VPValue *getEVL() const { return getOperand(getNumOperands() - 1); } + + /// Produce a vp-intrinsic copies of the cast. + void execute(VPTransformState &State) final; + + /// Returns true if the recipe only uses the first lane of operand \p Op. + bool onlyFirstLaneUsed(const VPValue *Op) const override { + assert(is_contained(operands(), Op) && + "Op must be an operand of the recipe"); + // EVL in that recipe is always the last operand, thus any use before means + // the VPValue should be vectorized. + return getEVL() == Op; + } + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + /// Print the recipe. + void print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const final; +#endif +}; + /// VPScalarCastRecipe is a recipe to create scalar cast instructions. class VPScalarCastRecipe : public VPSingleDefRecipe { Instruction::CastOps Opcode; diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 3b11ec77813f3..f8303a64ca14e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -92,6 +92,7 @@ bool VPRecipeBase::mayWriteToMemory() const { case VPVectorPointerSC: case VPWidenCanonicalIVSC: case VPWidenCastSC: + case VPWidenCastEVLSC: case VPWidenGEPSC: case VPWidenIntOrFpInductionSC: case VPWidenLoadEVLSC: @@ -138,6 +139,7 @@ bool VPRecipeBase::mayReadFromMemory() const { case VPVectorPointerSC: case VPWidenCanonicalIVSC: case VPWidenCastSC: + case VPWidenCastEVLSC: case VPWidenGEPSC: case VPWidenIntOrFpInductionSC: case VPWidenPHISC: @@ -178,6 +180,7 @@ bool VPRecipeBase::mayHaveSideEffects() const { case VPVectorPointerSC: case VPWidenCanonicalIVSC: case VPWidenCastSC: + case VPWidenCastEVLSC: case VPWidenGEPSC: case VPWidenIntOrFpInductionSC: case VPWidenPHISC: @@ -1554,6 +1557,40 @@ InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF, dyn_cast_if_present(getUnderlyingValue())); } +void VPWidenCastEVLRecipe::execute(VPTransformState &State) { + unsigned Opcode = getOpcode(); + State.setDebugLocFrom(getDebugLoc()); + assert(State.UF == 1 && "Expected only UF == 1 when vectorizing with " + "explicit vector length."); + + assert(State.get(getOperand(0), 0)->getType()->isVectorTy() && + "VPWidenCastEVLRecipe should not be used for scalars"); + + // TODO: add more cast instruction, eg: fptoint/inttofp/inttoptr/fptofp + if (Opcode == Instruction::SExt || Opcode == Instruction::ZExt || + Opcode == Instruction::Trunc) { + Value *SrcVal = State.get(getOperand(0), 0); + VectorType *SrcTy = cast(SrcVal->getType()); + VectorType *DsType = + VectorType::get(getResultType(), SrcTy->getElementCount()); + + IRBuilderBase &BuilderIR = State.Builder; + VectorBuilder Builder(BuilderIR); + Value *Mask = BuilderIR.CreateVectorSplat(State.VF, BuilderIR.getTrue()); + + Builder.setMask(Mask).setEVL(State.get(getEVL(), 0, /*NeedsScalar=*/true)); + Value *VPInst = + Builder.createVectorInstruction(Opcode, DsType, {SrcVal}, "vp.cast"); + if (VPInst) { + if (auto *VecOp = dyn_cast(VPInst)) + VecOp->copyIRFlags(getUnderlyingInstr()); + } + State.set(this, VPInst, 0); + State.addMetadata(VPInst, + dyn_cast_or_null(getUnderlyingValue())); + } +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPWidenCastRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { @@ -1564,6 +1601,16 @@ void VPWidenCastRecipe::print(raw_ostream &O, const Twine &Indent, printOperands(O, SlotTracker); O << " to " << *getResultType(); } + +void VPWidenCastEVLRecipe::print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const { + O << Indent << "WIDEN-VP "; + printAsOperand(O, SlotTracker); + O << " = vp." << Instruction::getOpcodeName(getOpcode()) << " "; + printFlags(O); + printOperands(O, SlotTracker); + O << " to " << *getResultType(); +} #endif InstructionCost VPHeaderPHIRecipe::computeCost(ElementCount VF, diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index ba791c679b4ae..57801b90739a3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1477,6 +1477,15 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { return nullptr; return new VPWidenEVLRecipe(*W, EVL); }) + .Case( + [&](VPWidenCastRecipe *W) -> VPRecipeBase * { + unsigned Opcode = W->getOpcode(); + if (Opcode != Instruction::SExt && + Opcode != Instruction::ZExt && + Opcode != Instruction::Trunc) + return nullptr; + return new VPWidenCastEVLRecipe(*W, EVL); + }) .Case([&](VPReductionRecipe *Red) { VPValue *NewMask = GetNewMask(Red->getCondOp()); return new VPReductionEVLRecipe(*Red, EVL, NewMask); diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index 691b0d40823cf..97fc2e88f412d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -337,6 +337,7 @@ class VPDef { VPWidenCallSC, VPWidenCanonicalIVSC, VPWidenCastSC, + VPWidenCastEVLSC, VPWidenGEPSC, VPWidenIntrinsicSC, VPWidenLoadEVLSC, diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll index 8368022bceed9..9ee1708f175ab 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll @@ -194,16 +194,16 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) { ; IF-EVL-INLOOP-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; IF-EVL-INLOOP-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ] ; IF-EVL-INLOOP-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ] -; IF-EVL-INLOOP-NEXT: [[AVL:%.*]] = sub i32 [[N]], [[EVL_BASED_IV]] -; IF-EVL-INLOOP-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true) -; IF-EVL-INLOOP-NEXT: [[TMP6:%.*]] = add i32 [[EVL_BASED_IV]], 0 -; IF-EVL-INLOOP-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP6]] -; IF-EVL-INLOOP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[TMP7]], i32 0 -; IF-EVL-INLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv8i16.p0(ptr align 2 [[TMP8]], splat (i1 true), i32 [[TMP5]]) -; IF-EVL-INLOOP-NEXT: [[TMP9:%.*]] = sext [[VP_OP_LOAD]] to -; IF-EVL-INLOOP-NEXT: [[TMP10:%.*]] = call i32 @llvm.vp.reduce.add.nxv8i32(i32 0, [[TMP9]], splat (i1 true), i32 [[TMP5]]) +; IF-EVL-INLOOP-NEXT: [[TMP5:%.*]] = sub i32 [[N]], [[EVL_BASED_IV]] +; IF-EVL-INLOOP-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[TMP5]], i32 8, i1 true) +; IF-EVL-INLOOP-NEXT: [[TMP7:%.*]] = add i32 [[EVL_BASED_IV]], 0 +; IF-EVL-INLOOP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP7]] +; IF-EVL-INLOOP-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[TMP8]], i32 0 +; IF-EVL-INLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv8i16.p0(ptr align 2 [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP6]]) +; IF-EVL-INLOOP-NEXT: [[VP_CAST:%.*]] = call @llvm.vp.sext.nxv8i32.nxv8i16( [[VP_OP_LOAD]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP6]]) +; IF-EVL-INLOOP-NEXT: [[TMP10:%.*]] = call i32 @llvm.vp.reduce.add.nxv8i32(i32 0, [[VP_CAST]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP6]]) ; IF-EVL-INLOOP-NEXT: [[TMP11]] = add i32 [[TMP10]], [[VEC_PHI]] -; IF-EVL-INLOOP-NEXT: [[INDEX_EVL_NEXT]] = add i32 [[TMP5]], [[EVL_BASED_IV]] +; IF-EVL-INLOOP-NEXT: [[INDEX_EVL_NEXT]] = add i32 [[TMP6]], [[EVL_BASED_IV]] ; IF-EVL-INLOOP-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP4]] ; IF-EVL-INLOOP-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; IF-EVL-INLOOP-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll new file mode 100644 index 0000000000000..f1f3d2c88301e --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll @@ -0,0 +1,227 @@ +; REQUIRES: asserts + +; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \ +; RUN: -force-tail-folding-style=data-with-evl \ +; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \ +; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s + +; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \ +; RUN: -force-tail-folding-style=none \ +; RUN: -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \ +; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=NO-VP %s + +define void @vp_sext(ptr noalias %a, ptr noalias %b, i64 %N) { +; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF={1}' { +; IF-EVL-NEXT: Live-in vp<%0> = VF * UF +; IF-EVL-NEXT: Live-in vp<%1> = vector-trip-count +; IF-EVL-NEXT: Live-in ir<%N> = original trip-count + +; IF-EVL: vector.ph: +; IF-EVL-NEXT: Successor(s): vector loop + +; IF-EVL: vector loop: { +; IF-EVL-NEXT: vector.body: +; IF-EVL-NEXT: EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%12> +; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<%4> = phi ir<0>, vp<%11> +; IF-EVL-NEXT: EMIT vp<%5> = EXPLICIT-VECTOR-LENGTH vp<%4>, ir<%N> +; IF-EVL-NEXT: vp<%6> = SCALAR-STEPS vp<%4>, ir<1> +; IF-EVL-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%6> +; IF-EVL-NEXT: vp<%7> = vector-pointer ir<%arrayidx> +; IF-EVL-NEXT: WIDEN ir<%0> = vp.load vp<%7>, vp<%5> +; IF-EVL-NEXT: WIDEN-VP vp<%8> = vp.sext ir<%0>, vp<%5> to i64 +; IF-EVL-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%6> +; IF-EVL-NEXT: vp<%9> = vector-pointer ir<%arrayidx4> +; IF-EVL-NEXT: WIDEN vp.store vp<%9>, vp<%8>, vp<%5> +; IF-EVL-NEXT: SCALAR-CAST vp<%10> = zext vp<%5> to i64 +; IF-EVL-NEXT: EMIT vp<%11> = add vp<%10>, vp<%4> +; IF-EVL-NEXT: EMIT vp<%12> = add vp<%3>, vp<%0> +; IF-EVL-NEXT: EMIT branch-on-count vp<%12>, vp<%1> +; IF-EVL-NEXT: No successors +; IF-EVL-NEXT: } + +; NO-VP: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF>=1' { +; NO-VP-NEXT: Live-in vp<%0> = VF * UF +; NO-VP-NEXT: Live-in vp<%1> = vector-trip-count +; NO-VP-NEXT: Live-in ir<%N> = original trip-count + +; NO-VP: vector.ph: +; NO-VP-NEXT: Successor(s): vector loop + +; NO-VP: vector loop: { +; NO-VP-NEXT: vector.body: +; NO-VP-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%6> +; NO-VP-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1> +; NO-VP-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%3> +; NO-VP-NEXT: vp<%4> = vector-pointer ir<%arrayidx> +; NO-VP-NEXT: WIDEN ir<%0> = load vp<%4> +; NO-VP-NEXT: WIDEN-CAST ir<%conv2> = sext ir<%0> to i64 +; NO-VP-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%3> +; NO-VP-NEXT: vp<%5> = vector-pointer ir<%arrayidx4> +; NO-VP-NEXT: WIDEN store vp<%5>, ir<%conv2> +; NO-VP-NEXT: EMIT vp<%6> = add nuw vp<%2>, vp<%0> +; NO-VP-NEXT: EMIT branch-on-count vp<%6>, vp<%1> +; NO-VP-NEXT: No successors +; NO-VP-NEXT: } + +entry: + %cmp8 = icmp sgt i64 %N, 0 + br i1 %cmp8, label %for.body, label %for.cond.cleanup + +for.cond.cleanup: + ret void + +for.body: + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32, ptr %b, i64 %indvars.iv + %0 = load i32, ptr %arrayidx, align 4 + %conv2 = sext i32 %0 to i64 + %arrayidx4 = getelementptr inbounds i64, ptr %a, i64 %indvars.iv + store i64 %conv2, ptr %arrayidx4, align 8 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond.not = icmp eq i64 %indvars.iv.next, %N + br i1 %exitcond.not, label %for.cond.cleanup, label %for.body +} + +define void @vp_zext(ptr noalias %a, ptr noalias %b, i64 %N) { +; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF={1}' { +; IF-EVL-NEXT: Live-in vp<%0> = VF * UF +; IF-EVL-NEXT: Live-in vp<%1> = vector-trip-count +; IF-EVL-NEXT: Live-in ir<%N> = original trip-count + +; IF-EVL: vector.ph: +; IF-EVL-NEXT: Successor(s): vector loop + +; IF-EVL: vector loop: { +; IF-EVL-NEXT: vector.body: +; IF-EVL-NEXT: EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%12> +; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<%4> = phi ir<0>, vp<%11> +; IF-EVL-NEXT: EMIT vp<%5> = EXPLICIT-VECTOR-LENGTH vp<%4>, ir<%N> +; IF-EVL-NEXT: vp<%6> = SCALAR-STEPS vp<%4>, ir<1> +; IF-EVL-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%6> +; IF-EVL-NEXT: vp<%7> = vector-pointer ir<%arrayidx> +; IF-EVL-NEXT: WIDEN ir<%0> = vp.load vp<%7>, vp<%5> +; IF-EVL-NEXT: WIDEN-VP vp<%8> = vp.zext ir<%0>, vp<%5> to i64 +; IF-EVL-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%6> +; IF-EVL-NEXT: vp<%9> = vector-pointer ir<%arrayidx4> +; IF-EVL-NEXT: WIDEN vp.store vp<%9>, vp<%8>, vp<%5> +; IF-EVL-NEXT: SCALAR-CAST vp<%10> = zext vp<%5> to i64 +; IF-EVL-NEXT: EMIT vp<%11> = add vp<%10>, vp<%4> +; IF-EVL-NEXT: EMIT vp<%12> = add vp<%3>, vp<%0> +; IF-EVL-NEXT: EMIT branch-on-count vp<%12>, vp<%1> +; IF-EVL-NEXT: No successors +; IF-EVL-NEXT: } + +; NO-VP: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF>=1' { +; NO-VP-NEXT: Live-in vp<%0> = VF * UF +; NO-VP-NEXT: Live-in vp<%1> = vector-trip-count +; NO-VP-NEXT: Live-in ir<%N> = original trip-count + +; NO-VP: vector.ph: +; NO-VP-NEXT: Successor(s): vector loop + +; NO-VP: vector loop: { +; NO-VP-NEXT: vector.body: +; NO-VP-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%6> +; NO-VP-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1> +; NO-VP-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%3> +; NO-VP-NEXT: vp<%4> = vector-pointer ir<%arrayidx> +; NO-VP-NEXT: WIDEN ir<%0> = load vp<%4> +; NO-VP-NEXT: WIDEN-CAST ir<%conv2> = zext ir<%0> to i64 +; NO-VP-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%3> +; NO-VP-NEXT: vp<%5> = vector-pointer ir<%arrayidx4> +; NO-VP-NEXT: WIDEN store vp<%5>, ir<%conv2> +; NO-VP-NEXT: EMIT vp<%6> = add nuw vp<%2>, vp<%0> +; NO-VP-NEXT: EMIT branch-on-count vp<%6>, vp<%1> +; NO-VP-NEXT: No successors +; NO-VP-NEXT: } + +entry: + %cmp8 = icmp sgt i64 %N, 0 + br i1 %cmp8, label %for.body, label %for.cond.cleanup + +for.cond.cleanup: + ret void + +for.body: + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32, ptr %b, i64 %indvars.iv + %0 = load i32, ptr %arrayidx, align 4 + %conv2 = zext i32 %0 to i64 + %arrayidx4 = getelementptr inbounds i64, ptr %a, i64 %indvars.iv + store i64 %conv2, ptr %arrayidx4, align 8 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond.not = icmp eq i64 %indvars.iv.next, %N + br i1 %exitcond.not, label %for.cond.cleanup, label %for.body +} + +define void @vp_truncate(ptr noalias %a, ptr noalias %b, i64 %N) { +; IF-EVL : VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { +; IF-EVL-NEXT : Live-in vp<%0> = VF * UF +; IF-EVL-NEXT : Live-in vp<%1> = vector-trip-count +; IF-EVL-NEXT : Live-in ir<%N> = original trip-count + +; IF-EVL : vector.ph: +; IF-EVL-NEXT : Successor(s): vector loop + +; IF-EVL : vector loop: { +; IF-EVL-NEXT : vector.body: +; IF-EVL-NEXT : EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%12> +; IF-EVL-NEXT : EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<%4> = phi ir<0>, vp<%11> +; IF-EVL-NEXT : EMIT vp<%5> = EXPLICIT-VECTOR-LENGTH vp<%4>, ir<%N> +; IF-EVL-NEXT : vp<%6> = SCALAR-STEPS vp<%4>, ir<1> +; IF-EVL-NEXT : CLONE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%6> +; IF-EVL-NEXT : vp<%7> = vector-pointer ir<%arrayidx> +; IF-EVL-NEXT : WIDEN ir<%0> = vp.load vp<%7>, vp<%5> +; IF-EVL-NEXT : WIDEN-VP vp<%8> = vp.trunc ir<%0>, vp<%5> to i16 +; IF-EVL-NEXT : CLONE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%6> +; IF-EVL-NEXT : vp<%9> = vector-pointer ir<%arrayidx4> +; IF-EVL-NEXT : WIDEN vp.store vp<%9>, vp<%8>, vp<%5> +; IF-EVL-NEXT : SCALAR-CAST vp<%10> = zext vp<%5> to i64 +; IF-EVL-NEXT : EMIT vp<%11> = add vp<%10>, vp<%4> +; IF-EVL-NEXT : EMIT vp<%12> = add vp<%3>, vp<%0> +; IF-EVL-NEXT : EMIT branch-on-count vp<%12>, vp<%1> +; IF-EVL-NEXT : No successors +; IF-EVL-NEXT : } + +; NO-VP: Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF>=1' { +; NO-VP-NEXT: Live-in vp<%0> = VF * UF +; NO-VP-NEXT: Live-in vp<%1> = vector-trip-count +; NO-VP-NEXT: Live-in ir<%N> = original trip-count + +; NO-VP: vector.ph: +; NO-VP-NEXT: Successor(s): vector loop + +; NO-VP: vector loop: { +; NO-VP: vector.body: +; NO-VP-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%6> +; NO-VP-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1> +; NO-VP-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%3> +; NO-VP-NEXT: vp<%4> = vector-pointer ir<%arrayidx> +; NO-VP-NEXT: WIDEN ir<%0> = load vp<%4> +; NO-VP-NEXT: WIDEN-CAST ir<%conv2> = trunc ir<%0> to i16 +; NO-VP-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%3> +; NO-VP-NEXT: vp<%5> = vector-pointer ir<%arrayidx4> +; NO-VP-NEXT: WIDEN store vp<%5>, ir<%conv2> +; NO-VP-NEXT: EMIT vp<%6> = add nuw vp<%2>, vp<%0> +; NO-VP-NEXT: EMIT branch-on-count vp<%6>, vp<%1> +; NO-VP-NEXT: No successors +; NO-VP-NEXT: } + +entry: + %cmp8 = icmp sgt i64 %N, 0 + br i1 %cmp8, label %for.body, label %for.cond.cleanup + +for.cond.cleanup: + ret void + +for.body: + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32, ptr %b, i64 %indvars.iv + %0 = load i32, ptr %arrayidx, align 4 + %conv2 = trunc i32 %0 to i16 + %arrayidx4 = getelementptr inbounds i16, ptr %a, i64 %indvars.iv + store i16 %conv2, ptr %arrayidx4, align 2 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond.not = icmp eq i64 %indvars.iv.next, %N + br i1 %exitcond.not, label %for.cond.cleanup, label %for.body +} From 4d3be159fabe574c31902dd077e6eb9316d27053 Mon Sep 17 00:00:00 2001 From: "Liqin.Weng" Date: Fri, 18 Oct 2024 14:02:54 +0800 Subject: [PATCH 02/11] Rebase and Revert the implement of castEVLRecipe --- .../Transforms/Vectorize/LoopVectorize.cpp | 1 - llvm/lib/Transforms/Vectorize/VPlan.h | 77 +-- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 47 -- .../Transforms/Vectorize/VPlanTransforms.cpp | 9 - llvm/lib/Transforms/Vectorize/VPlanValue.h | 1 - .../LoopVectorize/RISCV/inloop-reduction.ll | 2 +- .../RISCV/vplan-vp-cast-intrinsics.ll | 526 ++++++++++++------ 7 files changed, 351 insertions(+), 312 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 6b6d338cdf8eb..d68a26251ac9d 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4516,7 +4516,6 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF, case VPDef::VPWidenCallSC: case VPDef::VPWidenCanonicalIVSC: case VPDef::VPWidenCastSC: - case VPDef::VPWidenCastEVLSC: case VPDef::VPWidenGEPSC: case VPDef::VPWidenIntrinsicSC: case VPDef::VPWidenSC: diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index e5035a319f344..1b1630ebc6c23 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -866,7 +866,6 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue { case VPRecipeBase::VPWidenCallSC: case VPRecipeBase::VPWidenCanonicalIVSC: case VPRecipeBase::VPWidenCastSC: - case VPRecipeBase::VPWidenCastEVLSC: case VPRecipeBase::VPWidenGEPSC: case VPRecipeBase::VPWidenIntrinsicSC: case VPRecipeBase::VPWidenSC: @@ -1064,7 +1063,6 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe { R->getVPDefID() == VPRecipeBase::VPWidenEVLSC || R->getVPDefID() == VPRecipeBase::VPWidenGEPSC || R->getVPDefID() == VPRecipeBase::VPWidenCastSC || - R->getVPDefID() == VPRecipeBase::VPWidenCastEVLSC || R->getVPDefID() == VPRecipeBase::VPReplicateSC || R->getVPDefID() == VPRecipeBase::VPReverseVectorPointerSC || R->getVPDefID() == VPRecipeBase::VPVectorPointerSC; @@ -1544,28 +1542,19 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags { /// Result type for the cast. Type *ResultTy; -protected: - VPWidenCastRecipe(unsigned VPDefOpcode, Instruction::CastOps Opcode, - VPValue *Op, Type *ResultTy, CastInst &UI) +public: + VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, + CastInst &UI) : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), Opcode(Opcode), ResultTy(ResultTy) { assert(UI.getOpcode() == Opcode && "opcode of underlying cast doesn't match"); } - VPWidenCastRecipe(unsigned VPDefOpcode, Instruction::CastOps Opcode, - VPValue *Op, Type *ResultTy) + VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy) : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op), Opcode(Opcode), ResultTy(ResultTy) {} -public: - VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, - CastInst &UI) - : VPWidenCastRecipe(VPDef::VPWidenCastSC, Opcode, Op, ResultTy, UI) {} - - VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy) - : VPWidenCastRecipe(VPDef::VPWidenCastSC, Opcode, Op, ResultTy) {} - ~VPWidenCastRecipe() override = default; VPWidenCastRecipe *clone() override { @@ -1576,15 +1565,7 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags { return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy); } - static inline bool classof(const VPRecipeBase *R) { - return R->getVPDefID() == VPRecipeBase::VPWidenCastSC || - R->getVPDefID() == VPRecipeBase::VPWidenCastEVLSC; - } - - static inline bool classof(const VPUser *U) { - auto *R = dyn_cast(U); - return R && classof(R); - } + VP_CLASSOF_IMPL(VPDef::VPWidenCastSC) /// Produce widened copies of the cast. void execute(VPTransformState &State) override; @@ -1605,54 +1586,6 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags { Type *getResultType() const { return ResultTy; } }; -// A recipe for widening cast operation with vector-predication intrinsics with -/// explicit vector length (EVL). -class VPWidenCastEVLRecipe : public VPWidenCastRecipe { - using VPRecipeWithIRFlags::transferFlags; - -public: - VPWidenCastEVLRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, - VPValue &EVL) - : VPWidenCastRecipe(VPDef::VPWidenCastEVLSC, Opcode, Op, ResultTy) { - addOperand(&EVL); - } - VPWidenCastEVLRecipe(VPWidenCastRecipe &W, VPValue &EVL) - : VPWidenCastEVLRecipe(W.getOpcode(), W.getOperand(0), W.getResultType(), - EVL) { - transferFlags(W); - } - - ~VPWidenCastEVLRecipe() override = default; - - VPWidenCastEVLRecipe *clone() final { - llvm_unreachable("VPWidenEVLRecipe cannot be cloned"); - return nullptr; - } - - VP_CLASSOF_IMPL(VPDef::VPWidenCastEVLSC) - - VPValue *getEVL() { return getOperand(getNumOperands() - 1); } - const VPValue *getEVL() const { return getOperand(getNumOperands() - 1); } - - /// Produce a vp-intrinsic copies of the cast. - void execute(VPTransformState &State) final; - - /// Returns true if the recipe only uses the first lane of operand \p Op. - bool onlyFirstLaneUsed(const VPValue *Op) const override { - assert(is_contained(operands(), Op) && - "Op must be an operand of the recipe"); - // EVL in that recipe is always the last operand, thus any use before means - // the VPValue should be vectorized. - return getEVL() == Op; - } - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - /// Print the recipe. - void print(raw_ostream &O, const Twine &Indent, - VPSlotTracker &SlotTracker) const final; -#endif -}; - /// VPScalarCastRecipe is a recipe to create scalar cast instructions. class VPScalarCastRecipe : public VPSingleDefRecipe { Instruction::CastOps Opcode; diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index f8303a64ca14e..3b11ec77813f3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -92,7 +92,6 @@ bool VPRecipeBase::mayWriteToMemory() const { case VPVectorPointerSC: case VPWidenCanonicalIVSC: case VPWidenCastSC: - case VPWidenCastEVLSC: case VPWidenGEPSC: case VPWidenIntOrFpInductionSC: case VPWidenLoadEVLSC: @@ -139,7 +138,6 @@ bool VPRecipeBase::mayReadFromMemory() const { case VPVectorPointerSC: case VPWidenCanonicalIVSC: case VPWidenCastSC: - case VPWidenCastEVLSC: case VPWidenGEPSC: case VPWidenIntOrFpInductionSC: case VPWidenPHISC: @@ -180,7 +178,6 @@ bool VPRecipeBase::mayHaveSideEffects() const { case VPVectorPointerSC: case VPWidenCanonicalIVSC: case VPWidenCastSC: - case VPWidenCastEVLSC: case VPWidenGEPSC: case VPWidenIntOrFpInductionSC: case VPWidenPHISC: @@ -1557,40 +1554,6 @@ InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF, dyn_cast_if_present(getUnderlyingValue())); } -void VPWidenCastEVLRecipe::execute(VPTransformState &State) { - unsigned Opcode = getOpcode(); - State.setDebugLocFrom(getDebugLoc()); - assert(State.UF == 1 && "Expected only UF == 1 when vectorizing with " - "explicit vector length."); - - assert(State.get(getOperand(0), 0)->getType()->isVectorTy() && - "VPWidenCastEVLRecipe should not be used for scalars"); - - // TODO: add more cast instruction, eg: fptoint/inttofp/inttoptr/fptofp - if (Opcode == Instruction::SExt || Opcode == Instruction::ZExt || - Opcode == Instruction::Trunc) { - Value *SrcVal = State.get(getOperand(0), 0); - VectorType *SrcTy = cast(SrcVal->getType()); - VectorType *DsType = - VectorType::get(getResultType(), SrcTy->getElementCount()); - - IRBuilderBase &BuilderIR = State.Builder; - VectorBuilder Builder(BuilderIR); - Value *Mask = BuilderIR.CreateVectorSplat(State.VF, BuilderIR.getTrue()); - - Builder.setMask(Mask).setEVL(State.get(getEVL(), 0, /*NeedsScalar=*/true)); - Value *VPInst = - Builder.createVectorInstruction(Opcode, DsType, {SrcVal}, "vp.cast"); - if (VPInst) { - if (auto *VecOp = dyn_cast(VPInst)) - VecOp->copyIRFlags(getUnderlyingInstr()); - } - State.set(this, VPInst, 0); - State.addMetadata(VPInst, - dyn_cast_or_null(getUnderlyingValue())); - } -} - #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPWidenCastRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { @@ -1601,16 +1564,6 @@ void VPWidenCastRecipe::print(raw_ostream &O, const Twine &Indent, printOperands(O, SlotTracker); O << " to " << *getResultType(); } - -void VPWidenCastEVLRecipe::print(raw_ostream &O, const Twine &Indent, - VPSlotTracker &SlotTracker) const { - O << Indent << "WIDEN-VP "; - printAsOperand(O, SlotTracker); - O << " = vp." << Instruction::getOpcodeName(getOpcode()) << " "; - printFlags(O); - printOperands(O, SlotTracker); - O << " to " << *getResultType(); -} #endif InstructionCost VPHeaderPHIRecipe::computeCost(ElementCount VF, diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 57801b90739a3..ba791c679b4ae 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1477,15 +1477,6 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { return nullptr; return new VPWidenEVLRecipe(*W, EVL); }) - .Case( - [&](VPWidenCastRecipe *W) -> VPRecipeBase * { - unsigned Opcode = W->getOpcode(); - if (Opcode != Instruction::SExt && - Opcode != Instruction::ZExt && - Opcode != Instruction::Trunc) - return nullptr; - return new VPWidenCastEVLRecipe(*W, EVL); - }) .Case([&](VPReductionRecipe *Red) { VPValue *NewMask = GetNewMask(Red->getCondOp()); return new VPReductionEVLRecipe(*Red, EVL, NewMask); diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index 97fc2e88f412d..691b0d40823cf 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -337,7 +337,6 @@ class VPDef { VPWidenCallSC, VPWidenCanonicalIVSC, VPWidenCastSC, - VPWidenCastEVLSC, VPWidenGEPSC, VPWidenIntrinsicSC, VPWidenLoadEVLSC, diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll index 9ee1708f175ab..d273a88a84fea 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll @@ -200,7 +200,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) { ; IF-EVL-INLOOP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP7]] ; IF-EVL-INLOOP-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[TMP8]], i32 0 ; IF-EVL-INLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv8i16.p0(ptr align 2 [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP6]]) -; IF-EVL-INLOOP-NEXT: [[VP_CAST:%.*]] = call @llvm.vp.sext.nxv8i32.nxv8i16( [[VP_OP_LOAD]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP6]]) +; IF-EVL-INLOOP-NEXT: [[VP_CAST:%.*]] = sext [[VP_OP_LOAD]] to ; IF-EVL-INLOOP-NEXT: [[TMP10:%.*]] = call i32 @llvm.vp.reduce.add.nxv8i32(i32 0, [[VP_CAST]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP6]]) ; IF-EVL-INLOOP-NEXT: [[TMP11]] = add i32 [[TMP10]], [[VEC_PHI]] ; IF-EVL-INLOOP-NEXT: [[INDEX_EVL_NEXT]] = add i32 [[TMP6]], [[EVL_BASED_IV]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll index f1f3d2c88301e..6894a30f30efe 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll @@ -1,91 +1,63 @@ ; REQUIRES: asserts - ; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \ ; RUN: -force-tail-folding-style=data-with-evl \ ; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \ ; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s -; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \ -; RUN: -force-tail-folding-style=none \ -; RUN: -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \ -; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=NO-VP %s - define void @vp_sext(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF={1}' { -; IF-EVL-NEXT: Live-in vp<%0> = VF * UF -; IF-EVL-NEXT: Live-in vp<%1> = vector-trip-count +; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF +; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count ; IF-EVL: vector.ph: ; IF-EVL-NEXT: Successor(s): vector loop -; IF-EVL: vector loop: { -; IF-EVL-NEXT: vector.body: -; IF-EVL-NEXT: EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%12> -; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<%4> = phi ir<0>, vp<%11> -; IF-EVL-NEXT: EMIT vp<%5> = EXPLICIT-VECTOR-LENGTH vp<%4>, ir<%N> -; IF-EVL-NEXT: vp<%6> = SCALAR-STEPS vp<%4>, ir<1> -; IF-EVL-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%6> -; IF-EVL-NEXT: vp<%7> = vector-pointer ir<%arrayidx> -; IF-EVL-NEXT: WIDEN ir<%0> = vp.load vp<%7>, vp<%5> -; IF-EVL-NEXT: WIDEN-VP vp<%8> = vp.sext ir<%0>, vp<%5> to i64 -; IF-EVL-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%6> -; IF-EVL-NEXT: vp<%9> = vector-pointer ir<%arrayidx4> -; IF-EVL-NEXT: WIDEN vp.store vp<%9>, vp<%8>, vp<%5> -; IF-EVL-NEXT: SCALAR-CAST vp<%10> = zext vp<%5> to i64 -; IF-EVL-NEXT: EMIT vp<%11> = add vp<%10>, vp<%4> -; IF-EVL-NEXT: EMIT vp<%12> = add vp<%3>, vp<%0> -; IF-EVL-NEXT: EMIT branch-on-count vp<%12>, vp<%1> -; IF-EVL-NEXT: No successors +; IF-EVL: vector loop: { +; IF-EVL-NEXT: vector.body: +; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION +; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]> +; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]> +; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1> +; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> +; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN-CAST ir<[[SEXT:%.+]]> = sext ir<[[LD1]]> to i64 +; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[SEXT]]>, vp<[[EVL]]> +; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> +; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]> +; IF-EVL-NEXT: No successors ; IF-EVL-NEXT: } +; IF-EVL-NEXT: Successor(s): middle.block -; NO-VP: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF>=1' { -; NO-VP-NEXT: Live-in vp<%0> = VF * UF -; NO-VP-NEXT: Live-in vp<%1> = vector-trip-count -; NO-VP-NEXT: Live-in ir<%N> = original trip-count - -; NO-VP: vector.ph: -; NO-VP-NEXT: Successor(s): vector loop - -; NO-VP: vector loop: { -; NO-VP-NEXT: vector.body: -; NO-VP-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%6> -; NO-VP-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1> -; NO-VP-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%3> -; NO-VP-NEXT: vp<%4> = vector-pointer ir<%arrayidx> -; NO-VP-NEXT: WIDEN ir<%0> = load vp<%4> -; NO-VP-NEXT: WIDEN-CAST ir<%conv2> = sext ir<%0> to i64 -; NO-VP-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%3> -; NO-VP-NEXT: vp<%5> = vector-pointer ir<%arrayidx4> -; NO-VP-NEXT: WIDEN store vp<%5>, ir<%conv2> -; NO-VP-NEXT: EMIT vp<%6> = add nuw vp<%2>, vp<%0> -; NO-VP-NEXT: EMIT branch-on-count vp<%6>, vp<%1> -; NO-VP-NEXT: No successors -; NO-VP-NEXT: } entry: - %cmp8 = icmp sgt i64 %N, 0 - br i1 %cmp8, label %for.body, label %for.cond.cleanup - -for.cond.cleanup: - ret void + br label %loop -for.body: - %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] - %arrayidx = getelementptr inbounds i32, ptr %b, i64 %indvars.iv - %0 = load i32, ptr %arrayidx, align 4 +loop: + %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] + %gep = getelementptr inbounds i32, ptr %b, i64 %iv + %0 = load i32, ptr %gep, align 4 %conv2 = sext i32 %0 to i64 - %arrayidx4 = getelementptr inbounds i64, ptr %a, i64 %indvars.iv - store i64 %conv2, ptr %arrayidx4, align 8 - %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 - %exitcond.not = icmp eq i64 %indvars.iv.next, %N - br i1 %exitcond.not, label %for.cond.cleanup, label %for.body + %gep4 = getelementptr inbounds i64, ptr %a, i64 %iv + store i64 %conv2, ptr %gep4, align 8 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void } define void @vp_zext(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF={1}' { -; IF-EVL-NEXT: Live-in vp<%0> = VF * UF -; IF-EVL-NEXT: Live-in vp<%1> = vector-trip-count +; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF +; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count ; IF-EVL: vector.ph: @@ -93,135 +65,327 @@ define void @vp_zext(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL: vector loop: { ; IF-EVL-NEXT: vector.body: -; IF-EVL-NEXT: EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%12> -; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<%4> = phi ir<0>, vp<%11> -; IF-EVL-NEXT: EMIT vp<%5> = EXPLICIT-VECTOR-LENGTH vp<%4>, ir<%N> -; IF-EVL-NEXT: vp<%6> = SCALAR-STEPS vp<%4>, ir<1> -; IF-EVL-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%6> -; IF-EVL-NEXT: vp<%7> = vector-pointer ir<%arrayidx> -; IF-EVL-NEXT: WIDEN ir<%0> = vp.load vp<%7>, vp<%5> -; IF-EVL-NEXT: WIDEN-VP vp<%8> = vp.zext ir<%0>, vp<%5> to i64 -; IF-EVL-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%6> -; IF-EVL-NEXT: vp<%9> = vector-pointer ir<%arrayidx4> -; IF-EVL-NEXT: WIDEN vp.store vp<%9>, vp<%8>, vp<%5> -; IF-EVL-NEXT: SCALAR-CAST vp<%10> = zext vp<%5> to i64 -; IF-EVL-NEXT: EMIT vp<%11> = add vp<%10>, vp<%4> -; IF-EVL-NEXT: EMIT vp<%12> = add vp<%3>, vp<%0> -; IF-EVL-NEXT: EMIT branch-on-count vp<%12>, vp<%1> +; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION +; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]> +; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]> +; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1> +; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> +; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN-CAST ir<[[ZEXT:%.+]]> = zext ir<[[LD1]]> to i64 +; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[ZEXT]]>, vp<[[EVL]]> +; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> +; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]> ; IF-EVL-NEXT: No successors ; IF-EVL-NEXT: } -; NO-VP: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF>=1' { -; NO-VP-NEXT: Live-in vp<%0> = VF * UF -; NO-VP-NEXT: Live-in vp<%1> = vector-trip-count -; NO-VP-NEXT: Live-in ir<%N> = original trip-count - -; NO-VP: vector.ph: -; NO-VP-NEXT: Successor(s): vector loop - -; NO-VP: vector loop: { -; NO-VP-NEXT: vector.body: -; NO-VP-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%6> -; NO-VP-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1> -; NO-VP-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%3> -; NO-VP-NEXT: vp<%4> = vector-pointer ir<%arrayidx> -; NO-VP-NEXT: WIDEN ir<%0> = load vp<%4> -; NO-VP-NEXT: WIDEN-CAST ir<%conv2> = zext ir<%0> to i64 -; NO-VP-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%3> -; NO-VP-NEXT: vp<%5> = vector-pointer ir<%arrayidx4> -; NO-VP-NEXT: WIDEN store vp<%5>, ir<%conv2> -; NO-VP-NEXT: EMIT vp<%6> = add nuw vp<%2>, vp<%0> -; NO-VP-NEXT: EMIT branch-on-count vp<%6>, vp<%1> -; NO-VP-NEXT: No successors -; NO-VP-NEXT: } +entry: + br label %loop + +loop: + %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] + %gep = getelementptr inbounds i32, ptr %b, i64 %iv + %0 = load i32, ptr %gep, align 4 + %conv2 = zext i32 %0 to i64 + %gep4 = getelementptr inbounds i64, ptr %a, i64 %iv + store i64 %conv2, ptr %gep4, align 8 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} + +define void @vp_truncate(ptr noalias %a, ptr noalias %b, i64 %N) { +; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { +; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF +; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count +; IF-EVL-NEXT: Live-in ir<%N> = original trip-count + +; IF-EVL: vector.ph: +; IF-EVL-NEXT: Successor(s): vector loop + +; IF-EVL: vector loop: { +; IF-EVL-NEXT: vector.body: +; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION +; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]> +; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]> +; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1> +; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> +; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN-CAST ir<[[TRUNC:%.+]]> = trunc ir<[[LD1]]> to i16 +; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[TRUNC]]>, vp<[[EVL]]> +; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> +; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]> +; IF-EVL-NEXT: No successors +; IF-EVL-NEXT: } entry: - %cmp8 = icmp sgt i64 %N, 0 - br i1 %cmp8, label %for.body, label %for.cond.cleanup + br label %loop -for.cond.cleanup: +loop: + %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] + %gep = getelementptr inbounds i32, ptr %b, i64 %iv + %0 = load i32, ptr %gep, align 4 + %conv2 = trunc i32 %0 to i16 + %gep4 = getelementptr inbounds i16, ptr %a, i64 %iv + store i16 %conv2, ptr %gep4, align 2 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: ret void +} -for.body: - %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] - %arrayidx = getelementptr inbounds i32, ptr %b, i64 %indvars.iv - %0 = load i32, ptr %arrayidx, align 4 - %conv2 = zext i32 %0 to i64 - %arrayidx4 = getelementptr inbounds i64, ptr %a, i64 %indvars.iv - store i64 %conv2, ptr %arrayidx4, align 8 - %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 - %exitcond.not = icmp eq i64 %indvars.iv.next, %N - br i1 %exitcond.not, label %for.cond.cleanup, label %for.body +define void @vp_fpext(ptr noalias %a, ptr noalias %b, i64 %N) { +; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF={1}' { +; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF +; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count +; IF-EVL-NEXT: Live-in ir<%N> = original trip-count + +; IF-EVL: vector.ph: +; IF-EVL-NEXT: Successor(s): vector loop + +; IF-EVL: vector loop: { +; IF-EVL-NEXT: vector.body: +; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION +; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]> +; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]> +; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1> +; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> +; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN-CAST ir<[[FPEXT:%.+]]> = fpext ir<[[LD1]]> to +; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[FPEXT]]>, vp<[[EVL]]> +; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> +; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]> +; IF-EVL-NEXT: No successors +; IF-EVL-NEXT: } + +entry: + br label %loop + +loop: + %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] + %gep = getelementptr inbounds float, ptr %b, i64 %iv + %0 = load float, ptr %gep, align 4 + %conv2 = fpext float %0 to double + %gep4 = getelementptr inbounds double, ptr %a, i64 %iv + store double %conv2, ptr %gep4, align 8 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void } -define void @vp_truncate(ptr noalias %a, ptr noalias %b, i64 %N) { -; IF-EVL : VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { -; IF-EVL-NEXT : Live-in vp<%0> = VF * UF -; IF-EVL-NEXT : Live-in vp<%1> = vector-trip-count -; IF-EVL-NEXT : Live-in ir<%N> = original trip-count - -; IF-EVL : vector.ph: -; IF-EVL-NEXT : Successor(s): vector loop - -; IF-EVL : vector loop: { -; IF-EVL-NEXT : vector.body: -; IF-EVL-NEXT : EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%12> -; IF-EVL-NEXT : EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<%4> = phi ir<0>, vp<%11> -; IF-EVL-NEXT : EMIT vp<%5> = EXPLICIT-VECTOR-LENGTH vp<%4>, ir<%N> -; IF-EVL-NEXT : vp<%6> = SCALAR-STEPS vp<%4>, ir<1> -; IF-EVL-NEXT : CLONE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%6> -; IF-EVL-NEXT : vp<%7> = vector-pointer ir<%arrayidx> -; IF-EVL-NEXT : WIDEN ir<%0> = vp.load vp<%7>, vp<%5> -; IF-EVL-NEXT : WIDEN-VP vp<%8> = vp.trunc ir<%0>, vp<%5> to i16 -; IF-EVL-NEXT : CLONE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%6> -; IF-EVL-NEXT : vp<%9> = vector-pointer ir<%arrayidx4> -; IF-EVL-NEXT : WIDEN vp.store vp<%9>, vp<%8>, vp<%5> -; IF-EVL-NEXT : SCALAR-CAST vp<%10> = zext vp<%5> to i64 -; IF-EVL-NEXT : EMIT vp<%11> = add vp<%10>, vp<%4> -; IF-EVL-NEXT : EMIT vp<%12> = add vp<%3>, vp<%0> -; IF-EVL-NEXT : EMIT branch-on-count vp<%12>, vp<%1> -; IF-EVL-NEXT : No successors -; IF-EVL-NEXT : } - -; NO-VP: Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF>=1' { -; NO-VP-NEXT: Live-in vp<%0> = VF * UF -; NO-VP-NEXT: Live-in vp<%1> = vector-trip-count -; NO-VP-NEXT: Live-in ir<%N> = original trip-count - -; NO-VP: vector.ph: -; NO-VP-NEXT: Successor(s): vector loop - -; NO-VP: vector loop: { -; NO-VP: vector.body: -; NO-VP-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%6> -; NO-VP-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1> -; NO-VP-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%3> -; NO-VP-NEXT: vp<%4> = vector-pointer ir<%arrayidx> -; NO-VP-NEXT: WIDEN ir<%0> = load vp<%4> -; NO-VP-NEXT: WIDEN-CAST ir<%conv2> = trunc ir<%0> to i16 -; NO-VP-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%3> -; NO-VP-NEXT: vp<%5> = vector-pointer ir<%arrayidx4> -; NO-VP-NEXT: WIDEN store vp<%5>, ir<%conv2> -; NO-VP-NEXT: EMIT vp<%6> = add nuw vp<%2>, vp<%0> -; NO-VP-NEXT: EMIT branch-on-count vp<%6>, vp<%1> -; NO-VP-NEXT: No successors -; NO-VP-NEXT: } +define void @vp_fptrunct(ptr noalias %a, ptr noalias %b, i64 %N) { +; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF={1}' { +; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF +; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count +; IF-EVL-NEXT: Live-in ir<%N> = original trip-count + +; IF-EVL: vector.ph: +; IF-EVL-NEXT: Successor(s): vector loop + +; IF-EVL: vector loop: { +; IF-EVL-NEXT: vector.body: +; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION +; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]> +; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]> +; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1> +; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> +; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN-CAST ir<[[FPTRUNC:%.+]]> = fptrunc ir<[[LD1]]> to float +; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[FPTRUNC]]>, vp<[[EVL]]> +; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> +; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]> +; IF-EVL-NEXT: No successors +; IF-EVL-NEXT: } entry: - %cmp8 = icmp sgt i64 %N, 0 - br i1 %cmp8, label %for.body, label %for.cond.cleanup + br label %loop -for.cond.cleanup: +loop: + %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] + %gep = getelementptr inbounds double, ptr %b, i64 %iv + %0 = load double, ptr %gep, align 8 + %conv2 = fptrunc double %0 to float + %gep4 = getelementptr inbounds float, ptr %a, i64 %iv + store float %conv2, ptr %gep4, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: ret void +} -for.body: - %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] - %arrayidx = getelementptr inbounds i32, ptr %b, i64 %indvars.iv - %0 = load i32, ptr %arrayidx, align 4 - %conv2 = trunc i32 %0 to i16 - %arrayidx4 = getelementptr inbounds i16, ptr %a, i64 %indvars.iv - store i16 %conv2, ptr %arrayidx4, align 2 - %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 - %exitcond.not = icmp eq i64 %indvars.iv.next, %N - br i1 %exitcond.not, label %for.cond.cleanup, label %for.body +define void @vp_fptosi(ptr noalias %a, ptr noalias %b, i64 %N) { +; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { +; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF +; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count +; IF-EVL-NEXT: Live-in ir<%N> = original trip-count + +; IF-EVL: vector.ph: +; IF-EVL-NEXT: Successor(s): vector loop + +; IF-EVL: vector loop: { +; IF-EVL-NEXT: vector.body: +; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION +; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]> +; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]> +; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1> +; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> +; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN-CAST ir<[[FPTOSI:%.+]]> = fptoui ir<[[LD1]]> to i32 +; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[FPTOSI]]>, vp<[[EVL]]> +; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> +; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]> +; IF-EVL-NEXT: No successors +; IF-EVL-NEXT: } + +entry: + br label %loop + +loop: + %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] + %gep = getelementptr inbounds float, ptr %b, i64 %iv + %0 = load float, ptr %gep, align 4 + %conv2 = fptoui float %0 to i32 + %gep4 = getelementptr inbounds i32, ptr %a, i64 %iv + store i32 %conv2, ptr %gep4, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} + +define void @vp_inttofp(ptr noalias %a, ptr noalias %b, i64 %N) { +; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { +; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF +; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count +; IF-EVL-NEXT: Live-in ir<%N> = original trip-count + +; IF-EVL: vector.ph: +; IF-EVL-NEXT: Successor(s): vector loop + +; IF-EVL: vector loop: { +; IF-EVL-NEXT: vector.body: +; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION +; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]> +; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]> +; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1> +; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> +; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN-CAST ir<[[SITOFP:%.+]]> = sitofp ir<[[LD1]]> to float +; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[SITOFP]]>, vp<[[EVL]]> +; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> +; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]> +; IF-EVL-NEXT: No successors +; IF-EVL-NEXT: } + +entry: + br label %loop + +loop: + %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] + %gep = getelementptr inbounds i32, ptr %b, i64 %iv + %0 = load i32, ptr %gep, align 4 + %conv2 = sitofp i32 %0 to float + %gep4 = getelementptr inbounds float, ptr %a, i64 %iv + store float %conv2, ptr %gep4, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void } + +define void @vp_uinttofp(ptr noalias %a, ptr noalias %b, i64 %N) { +; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { +; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF +; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count +; IF-EVL-NEXT: Live-in ir<%N> = original trip-count + +; IF-EVL: vector.ph: +; IF-EVL-NEXT: Successor(s): vector loop + +; IF-EVL: vector loop: { +; IF-EVL-NEXT: vector.body: +; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION +; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]> +; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]> +; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1> +; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> +; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN-CAST ir<[[UITOFP:%.+]]> = uitofp ir<[[LD1]]> to float +; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[UITOFP]]>, vp<[[EVL]]> +; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> +; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]> +; IF-EVL-NEXT: No successors +; IF-EVL-NEXT: } + +entry: + br label %loop + +loop: + %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] + %gep = getelementptr inbounds i32, ptr %b, i64 %iv + %0 = load i32, ptr %gep, align 4 + %conv2 = uitofp i32 %0 to float + %gep4 = getelementptr inbounds float, ptr %a, i64 %iv + store float %conv2, ptr %gep4, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} \ No newline at end of file From 3f4e0e56aa3890d8516cdd2a2c7a7a3fd5001c26 Mon Sep 17 00:00:00 2001 From: "Liqin.Weng" Date: Fri, 18 Oct 2024 17:20:53 +0800 Subject: [PATCH 03/11] [VPlan] Use VPWidenIntrinsicRecipe to vp.cast --- llvm/include/llvm/IR/VectorBuilder.h | 4 +- llvm/lib/IR/VectorBuilder.cpp | 9 ++-- llvm/lib/Transforms/Utils/LoopUtils.cpp | 4 +- llvm/lib/Transforms/Vectorize/VPlan.h | 9 ++++ .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 49 ++++++++++++------- .../Transforms/Vectorize/VPlanTransforms.cpp | 18 +++++++ .../LoopVectorize/RISCV/inloop-reduction.ll | 2 +- .../RISCV/vplan-vp-cast-intrinsics.ll | 32 ++++++------ 8 files changed, 83 insertions(+), 44 deletions(-) diff --git a/llvm/include/llvm/IR/VectorBuilder.h b/llvm/include/llvm/IR/VectorBuilder.h index b0277c2b52595..830163984e37b 100644 --- a/llvm/include/llvm/IR/VectorBuilder.h +++ b/llvm/include/llvm/IR/VectorBuilder.h @@ -99,11 +99,11 @@ class VectorBuilder { const Twine &Name = Twine()); /// Emit a VP reduction intrinsic call for recurrence kind. - /// \param RdxID The intrinsic ID of llvm.vector.reduce.* + /// \param ID The intrinsic ID of call Intrinsic /// \param ValTy The type of operand which the reduction operation is /// performed. /// \param VecOpArray The operand list. - Value *createSimpleReduction(Intrinsic::ID RdxID, Type *ValTy, + Value *createSimpleIntrinsic(Intrinsic::ID ID, Type *ValTy, ArrayRef VecOpArray, const Twine &Name = Twine()); }; diff --git a/llvm/lib/IR/VectorBuilder.cpp b/llvm/lib/IR/VectorBuilder.cpp index 737f49b1334d7..d629a2fb6af7b 100644 --- a/llvm/lib/IR/VectorBuilder.cpp +++ b/llvm/lib/IR/VectorBuilder.cpp @@ -60,13 +60,12 @@ Value *VectorBuilder::createVectorInstruction(unsigned Opcode, Type *ReturnTy, return createVectorInstructionImpl(VPID, ReturnTy, InstOpArray, Name); } -Value *VectorBuilder::createSimpleReduction(Intrinsic::ID RdxID, - Type *ValTy, +Value *VectorBuilder::createSimpleIntrinsic(Intrinsic::ID ID, Type *ValTy, ArrayRef InstOpArray, const Twine &Name) { - auto VPID = VPIntrinsic::getForIntrinsic(RdxID); - assert(VPReductionIntrinsic::isVPReduction(VPID) && - "No VPIntrinsic for this reduction"); + auto VPID = VPIntrinsic::getForIntrinsic(ID); + assert(VPIntrinsic::isVPIntrinsic(VPID) && + "No VPIntrinsic for this Intrinsic"); return createVectorInstructionImpl(VPID, ValTy, InstOpArray, Name); } diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp index 70047273c3b9a..2dac2d43f7f3a 100644 --- a/llvm/lib/Transforms/Utils/LoopUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp @@ -1300,7 +1300,7 @@ Value *llvm::createSimpleReduction(VectorBuilder &VBuilder, Value *Src, Type *SrcEltTy = SrcTy->getElementType(); Value *Iden = getRecurrenceIdentity(Kind, SrcEltTy, Desc.getFastMathFlags()); Value *Ops[] = {Iden, Src}; - return VBuilder.createSimpleReduction(Id, SrcTy, Ops); + return VBuilder.createSimpleIntrinsic(Id, SrcTy, Ops); } Value *llvm::createReduction(IRBuilderBase &B, @@ -1343,7 +1343,7 @@ Value *llvm::createOrderedReduction(VectorBuilder &VBuilder, Intrinsic::ID Id = getReductionIntrinsicID(RecurKind::FAdd); auto *SrcTy = cast(Src->getType()); Value *Ops[] = {Start, Src}; - return VBuilder.createSimpleReduction(Id, SrcTy, Ops); + return VBuilder.createSimpleIntrinsic(Id, SrcTy, Ops); } void llvm::propagateIRFlags(Value *I, ArrayRef VL, Value *OpValue, diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 1b1630ebc6c23..136d9b1190759 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1659,6 +1659,15 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags { MayWriteToMemory(CI.mayWriteToMemory()), MayHaveSideEffects(CI.mayHaveSideEffects()) {} + // VPWidenIntrinsicRecipe(CastInst &CI, Intrinsic::ID VectorIntrinsicID, + // ArrayRef CallArguments, Type *Ty, + // DebugLoc DL = {}) + // : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI), + // VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), + // MayReadFromMemory(CI.mayReadFromMemory()), + // MayWriteToMemory(CI.mayWriteToMemory()), + // MayHaveSideEffects(CI.mayHaveSideEffects()) {} + VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID, ArrayRef CallArguments, Type *Ty, DebugLoc DL = {}) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 3b11ec77813f3..04fc61241eb53 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -960,24 +960,37 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) { Args.push_back(Arg); } - // Use vector version of the intrinsic. - Module *M = State.Builder.GetInsertBlock()->getModule(); - Function *VectorF = - Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl); - assert(VectorF && "Can't retrieve vector intrinsic."); - - auto *CI = cast_or_null(getUnderlyingValue()); - SmallVector OpBundles; - if (CI) - CI->getOperandBundlesAsDefs(OpBundles); - - CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles); - - setFlags(V); - - if (!V->getType()->isVoidTy()) - State.set(this, V); - State.addMetadata(V, CI); + if (VPIntrinsic::isVPIntrinsic(VectorIntrinsicID)) { + // Use vector version of the vector predicate Intrinsic + IRBuilderBase &BuilderIR = State.Builder; + VectorBuilder VBuilder(BuilderIR); + Value *Mask = BuilderIR.CreateVectorSplat(State.VF, BuilderIR.getTrue()); + VBuilder.setMask(Mask).setEVL(Args.back()); + Args.pop_back(); + Value *VPInst = VBuilder.createSimpleIntrinsic( + VectorIntrinsicID, TysForDecl[0], Args, "vp.call"); + if (!VPInst->getType()->isVoidTy()) + State.set(this, VPInst); + State.addMetadata(VPInst, + dyn_cast_or_null(getUnderlyingValue())); + } else { + // Use vector version of the intrinsic. + Module *M = State.Builder.GetInsertBlock()->getModule(); + Function *VectorF = + Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl); + assert(VectorF && "Can't retrieve vector intrinsic."); + + auto *CI = cast_or_null(getUnderlyingValue()); + SmallVector OpBundles; + if (CI) + CI->getOperandBundlesAsDefs(OpBundles); + + CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles); + setFlags(V); + if (!V->getType()->isVoidTy()) + State.set(this, V); + State.addMetadata(V, CI); + } } InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF, diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index ba791c679b4ae..7d4e64b05aa7c 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1501,6 +1501,24 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { *CI, VPID, Ops, TypeInfo.inferScalarType(CInst), CInst->getDebugLoc()); }) + .Case( + [&](VPWidenCastRecipe *CInst) -> VPRecipeBase * { + auto *CI = cast(CInst->getUnderlyingInstr()); + SmallVector Ops(CInst->operands()); + Ops.push_back(&EVL); + Intrinsic::ID VPID = + VPIntrinsic::getForOpcode(CI->getOpcode()); + if (VPID == Intrinsic::not_intrinsic) + return nullptr; + // FIXME: In fact, can we really not pass the + // underlyingInstr? In this case, how to set the Flag and + // add metadata in VPWidenIntrinsicRecipe::execute? + // return new VPWidenIntrinsicRecipe( + // *CI, VPID, Ops, CI->getType(), CI->getDebugLoc()); + return new VPWidenIntrinsicRecipe( + VPID, Ops, TypeInfo.inferScalarType(CInst), false, + false, false); + }) .Case([&](VPWidenSelectRecipe *Sel) { SmallVector Ops(Sel->operands()); Ops.push_back(&EVL); diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll index d273a88a84fea..9ee1708f175ab 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll @@ -200,7 +200,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) { ; IF-EVL-INLOOP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP7]] ; IF-EVL-INLOOP-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[TMP8]], i32 0 ; IF-EVL-INLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv8i16.p0(ptr align 2 [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP6]]) -; IF-EVL-INLOOP-NEXT: [[VP_CAST:%.*]] = sext [[VP_OP_LOAD]] to +; IF-EVL-INLOOP-NEXT: [[VP_CAST:%.*]] = call @llvm.vp.sext.nxv8i32.nxv8i16( [[VP_OP_LOAD]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP6]]) ; IF-EVL-INLOOP-NEXT: [[TMP10:%.*]] = call i32 @llvm.vp.reduce.add.nxv8i32(i32 0, [[VP_CAST]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP6]]) ; IF-EVL-INLOOP-NEXT: [[TMP11]] = add i32 [[TMP10]], [[VEC_PHI]] ; IF-EVL-INLOOP-NEXT: [[INDEX_EVL_NEXT]] = add i32 [[TMP6]], [[EVL_BASED_IV]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll index 6894a30f30efe..8692f6c802e55 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll @@ -23,10 +23,10 @@ define void @vp_sext(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-CAST ir<[[SEXT:%.+]]> = sext ir<[[LD1]]> to i64 +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SEXT:%.+]]> = call llvm.vp.sext(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[SEXT]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[SEXT]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -73,10 +73,10 @@ define void @vp_zext(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-CAST ir<[[ZEXT:%.+]]> = zext ir<[[LD1]]> to i64 +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[ZEXT:%.+]]> = call llvm.vp.zext(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[ZEXT]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[ZEXT]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -121,10 +121,10 @@ define void @vp_truncate(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-CAST ir<[[TRUNC:%.+]]> = trunc ir<[[LD1]]> to i16 +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[TRUNC:%.+]]> = call llvm.vp.trunc(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[TRUNC]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[TRUNC]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -169,10 +169,10 @@ define void @vp_fpext(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-CAST ir<[[FPEXT:%.+]]> = fpext ir<[[LD1]]> to +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[FPEXT:%.+]]> = call llvm.vp.fpext(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[FPEXT]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[FPEXT]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -217,10 +217,10 @@ define void @vp_fptrunct(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-CAST ir<[[FPTRUNC:%.+]]> = fptrunc ir<[[LD1]]> to float +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[FPTRUNC:%.+]]> = call llvm.vp.fptrunc(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[FPTRUNC]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[FPTRUNC]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -265,10 +265,10 @@ define void @vp_fptosi(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-CAST ir<[[FPTOSI:%.+]]> = fptoui ir<[[LD1]]> to i32 +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[FPTOSI:%.+]]> = call llvm.vp.fptoui(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[FPTOSI]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[FPTOSI]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -313,10 +313,10 @@ define void @vp_inttofp(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-CAST ir<[[SITOFP:%.+]]> = sitofp ir<[[LD1]]> to float +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SITOFP:%.+]]> = call llvm.vp.sitofp(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[SITOFP]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[SITOFP]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -361,10 +361,10 @@ define void @vp_uinttofp(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-CAST ir<[[UITOFP:%.+]]> = uitofp ir<[[LD1]]> to float +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[UITOFP:%.+]]> = call llvm.vp.uitofp(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[UITOFP]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[UITOFP]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> From cc04719ce8b34060d2e6a1af112ec982300ecee1 Mon Sep 17 00:00:00 2001 From: LiqinWeng Date: Thu, 31 Oct 2024 18:37:56 +0800 Subject: [PATCH 04/11] fix the comments --- llvm/lib/Transforms/Vectorize/VPlan.h | 16 +++++----- .../Transforms/Vectorize/VPlanTransforms.cpp | 8 +---- .../RISCV/vplan-vp-cast-intrinsics.ll | 32 +++++++++---------- 3 files changed, 25 insertions(+), 31 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 136d9b1190759..e84c3fb0a43af 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1659,14 +1659,14 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags { MayWriteToMemory(CI.mayWriteToMemory()), MayHaveSideEffects(CI.mayHaveSideEffects()) {} - // VPWidenIntrinsicRecipe(CastInst &CI, Intrinsic::ID VectorIntrinsicID, - // ArrayRef CallArguments, Type *Ty, - // DebugLoc DL = {}) - // : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI), - // VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), - // MayReadFromMemory(CI.mayReadFromMemory()), - // MayWriteToMemory(CI.mayWriteToMemory()), - // MayHaveSideEffects(CI.mayHaveSideEffects()) {} + VPWidenIntrinsicRecipe(CastInst &CI, Intrinsic::ID VectorIntrinsicID, + ArrayRef CallArguments, Type *Ty, + DebugLoc DL = {}) + : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI), + VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), + MayReadFromMemory(CI.mayReadFromMemory()), + MayWriteToMemory(CI.mayWriteToMemory()), + MayHaveSideEffects(CI.mayHaveSideEffects()) {} VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID, ArrayRef CallArguments, Type *Ty, diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 7d4e64b05aa7c..a1ec7f6fc248f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1510,14 +1510,8 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { VPIntrinsic::getForOpcode(CI->getOpcode()); if (VPID == Intrinsic::not_intrinsic) return nullptr; - // FIXME: In fact, can we really not pass the - // underlyingInstr? In this case, how to set the Flag and - // add metadata in VPWidenIntrinsicRecipe::execute? - // return new VPWidenIntrinsicRecipe( - // *CI, VPID, Ops, CI->getType(), CI->getDebugLoc()); return new VPWidenIntrinsicRecipe( - VPID, Ops, TypeInfo.inferScalarType(CInst), false, - false, false); + *CI, VPID, Ops, CI->getType(), CI->getDebugLoc()); }) .Case([&](VPWidenSelectRecipe *Sel) { SmallVector Ops(Sel->operands()); diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll index 8692f6c802e55..0d517407b5326 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll @@ -23,10 +23,10 @@ define void @vp_sext(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SEXT:%.+]]> = call llvm.vp.sext(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[SEXT:%.+]]> = call llvm.vp.sext(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[SEXT]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[SEXT]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -73,10 +73,10 @@ define void @vp_zext(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[ZEXT:%.+]]> = call llvm.vp.zext(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[ZEXT:%.+]]> = call llvm.vp.zext(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[ZEXT]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[ZEXT]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -121,10 +121,10 @@ define void @vp_truncate(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[TRUNC:%.+]]> = call llvm.vp.trunc(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[TRUNC:%.+]]> = call llvm.vp.trunc(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[TRUNC]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[TRUNC]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -169,10 +169,10 @@ define void @vp_fpext(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[FPEXT:%.+]]> = call llvm.vp.fpext(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[FPEXT:%.+]]> = call llvm.vp.fpext(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[FPEXT]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[FPEXT]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -217,10 +217,10 @@ define void @vp_fptrunct(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[FPTRUNC:%.+]]> = call llvm.vp.fptrunc(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[FPTRUNC:%.+]]> = call llvm.vp.fptrunc(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[FPTRUNC]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[FPTRUNC]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -265,10 +265,10 @@ define void @vp_fptosi(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[FPTOSI:%.+]]> = call llvm.vp.fptoui(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[FPTOSI:%.+]]> = call llvm.vp.fptoui(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[FPTOSI]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[FPTOSI]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -313,10 +313,10 @@ define void @vp_inttofp(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SITOFP:%.+]]> = call llvm.vp.sitofp(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[SITOFP:%.+]]> = call llvm.vp.sitofp(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[SITOFP]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[SITOFP]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -361,10 +361,10 @@ define void @vp_uinttofp(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[UITOFP:%.+]]> = call llvm.vp.uitofp(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[UITOFP:%.+]]> = call llvm.vp.uitofp(ir<[[LD1]]>, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[UITOFP]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[UITOFP]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> From 363d625238acff5a70852c60a69a76226de1c57b Mon Sep 17 00:00:00 2001 From: "Liqin.Weng" Date: Wed, 6 Nov 2024 14:05:52 +0800 Subject: [PATCH 05/11] fix the comments --- llvm/include/llvm/IR/VectorBuilder.h | 4 +- llvm/lib/IR/VectorBuilder.cpp | 9 ++-- llvm/lib/Transforms/Utils/LoopUtils.cpp | 4 +- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 49 +++++++------------ .../Transforms/Vectorize/VPlanTransforms.cpp | 11 ++++- 5 files changed, 36 insertions(+), 41 deletions(-) diff --git a/llvm/include/llvm/IR/VectorBuilder.h b/llvm/include/llvm/IR/VectorBuilder.h index 830163984e37b..b0277c2b52595 100644 --- a/llvm/include/llvm/IR/VectorBuilder.h +++ b/llvm/include/llvm/IR/VectorBuilder.h @@ -99,11 +99,11 @@ class VectorBuilder { const Twine &Name = Twine()); /// Emit a VP reduction intrinsic call for recurrence kind. - /// \param ID The intrinsic ID of call Intrinsic + /// \param RdxID The intrinsic ID of llvm.vector.reduce.* /// \param ValTy The type of operand which the reduction operation is /// performed. /// \param VecOpArray The operand list. - Value *createSimpleIntrinsic(Intrinsic::ID ID, Type *ValTy, + Value *createSimpleReduction(Intrinsic::ID RdxID, Type *ValTy, ArrayRef VecOpArray, const Twine &Name = Twine()); }; diff --git a/llvm/lib/IR/VectorBuilder.cpp b/llvm/lib/IR/VectorBuilder.cpp index d629a2fb6af7b..737f49b1334d7 100644 --- a/llvm/lib/IR/VectorBuilder.cpp +++ b/llvm/lib/IR/VectorBuilder.cpp @@ -60,12 +60,13 @@ Value *VectorBuilder::createVectorInstruction(unsigned Opcode, Type *ReturnTy, return createVectorInstructionImpl(VPID, ReturnTy, InstOpArray, Name); } -Value *VectorBuilder::createSimpleIntrinsic(Intrinsic::ID ID, Type *ValTy, +Value *VectorBuilder::createSimpleReduction(Intrinsic::ID RdxID, + Type *ValTy, ArrayRef InstOpArray, const Twine &Name) { - auto VPID = VPIntrinsic::getForIntrinsic(ID); - assert(VPIntrinsic::isVPIntrinsic(VPID) && - "No VPIntrinsic for this Intrinsic"); + auto VPID = VPIntrinsic::getForIntrinsic(RdxID); + assert(VPReductionIntrinsic::isVPReduction(VPID) && + "No VPIntrinsic for this reduction"); return createVectorInstructionImpl(VPID, ValTy, InstOpArray, Name); } diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp index 2dac2d43f7f3a..70047273c3b9a 100644 --- a/llvm/lib/Transforms/Utils/LoopUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp @@ -1300,7 +1300,7 @@ Value *llvm::createSimpleReduction(VectorBuilder &VBuilder, Value *Src, Type *SrcEltTy = SrcTy->getElementType(); Value *Iden = getRecurrenceIdentity(Kind, SrcEltTy, Desc.getFastMathFlags()); Value *Ops[] = {Iden, Src}; - return VBuilder.createSimpleIntrinsic(Id, SrcTy, Ops); + return VBuilder.createSimpleReduction(Id, SrcTy, Ops); } Value *llvm::createReduction(IRBuilderBase &B, @@ -1343,7 +1343,7 @@ Value *llvm::createOrderedReduction(VectorBuilder &VBuilder, Intrinsic::ID Id = getReductionIntrinsicID(RecurKind::FAdd); auto *SrcTy = cast(Src->getType()); Value *Ops[] = {Start, Src}; - return VBuilder.createSimpleIntrinsic(Id, SrcTy, Ops); + return VBuilder.createSimpleReduction(Id, SrcTy, Ops); } void llvm::propagateIRFlags(Value *I, ArrayRef VL, Value *OpValue, diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 04fc61241eb53..d1922b4e0755e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -960,37 +960,24 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) { Args.push_back(Arg); } - if (VPIntrinsic::isVPIntrinsic(VectorIntrinsicID)) { - // Use vector version of the vector predicate Intrinsic - IRBuilderBase &BuilderIR = State.Builder; - VectorBuilder VBuilder(BuilderIR); - Value *Mask = BuilderIR.CreateVectorSplat(State.VF, BuilderIR.getTrue()); - VBuilder.setMask(Mask).setEVL(Args.back()); - Args.pop_back(); - Value *VPInst = VBuilder.createSimpleIntrinsic( - VectorIntrinsicID, TysForDecl[0], Args, "vp.call"); - if (!VPInst->getType()->isVoidTy()) - State.set(this, VPInst); - State.addMetadata(VPInst, - dyn_cast_or_null(getUnderlyingValue())); - } else { - // Use vector version of the intrinsic. - Module *M = State.Builder.GetInsertBlock()->getModule(); - Function *VectorF = - Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl); - assert(VectorF && "Can't retrieve vector intrinsic."); - - auto *CI = cast_or_null(getUnderlyingValue()); - SmallVector OpBundles; - if (CI) - CI->getOperandBundlesAsDefs(OpBundles); - - CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles); - setFlags(V); - if (!V->getType()->isVoidTy()) - State.set(this, V); - State.addMetadata(V, CI); - } + // Use vector version of the intrinsic. + Module *M = State.Builder.GetInsertBlock()->getModule(); + Function *VectorF = + Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl); + assert(VectorF && "Can't retrieve vector intrinsic."); + + auto *CI = cast_or_null(getUnderlyingValue()); + SmallVector OpBundles; + if (CI) + CI->getOperandBundlesAsDefs(OpBundles); + + CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles); + + setFlags(V); + + if (!V->getType()->isVoidTy()) + State.set(this, V); + State.addMetadata(V, dyn_cast_or_null(getUnderlyingValue())); } InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF, diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index a1ec7f6fc248f..56e5c67590f1d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1504,12 +1504,19 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { .Case( [&](VPWidenCastRecipe *CInst) -> VPRecipeBase * { auto *CI = cast(CInst->getUnderlyingInstr()); - SmallVector Ops(CInst->operands()); - Ops.push_back(&EVL); Intrinsic::ID VPID = VPIntrinsic::getForOpcode(CI->getOpcode()); if (VPID == Intrinsic::not_intrinsic) return nullptr; + SmallVector Ops(CInst->operands()); + if (VPIntrinsic::getMaskParamPos(VPID)) { + VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue( + IntegerType::getInt1Ty(CI->getContext()))); + Ops.push_back(Mask); + } + if (VPIntrinsic::getVectorLengthParamPos(VPID)) { + Ops.push_back(&EVL); + } return new VPWidenIntrinsicRecipe( *CI, VPID, Ops, CI->getType(), CI->getDebugLoc()); }) From 0e45dc495417e3c53a62d6e27b90dac981a9ad2b Mon Sep 17 00:00:00 2001 From: LiqinWeng Date: Thu, 14 Nov 2024 17:24:01 +0800 Subject: [PATCH 06/11] fix the commets --- llvm/lib/Transforms/Vectorize/VPlan.h | 19 +++-------- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 25 +++++++++++---- .../Transforms/Vectorize/VPlanTransforms.cpp | 21 ++++++------ .../RISCV/vplan-vp-cast-intrinsics.ll | 32 +++++++++---------- 4 files changed, 51 insertions(+), 46 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index e84c3fb0a43af..65863fbd276dc 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1650,23 +1650,14 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags { bool MayHaveSideEffects; public: - VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID, + VPWidenIntrinsicRecipe(Instruction &I, Intrinsic::ID VectorIntrinsicID, ArrayRef CallArguments, Type *Ty, DebugLoc DL = {}) - : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI), + : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, I), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), - MayReadFromMemory(CI.mayReadFromMemory()), - MayWriteToMemory(CI.mayWriteToMemory()), - MayHaveSideEffects(CI.mayHaveSideEffects()) {} - - VPWidenIntrinsicRecipe(CastInst &CI, Intrinsic::ID VectorIntrinsicID, - ArrayRef CallArguments, Type *Ty, - DebugLoc DL = {}) - : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI), - VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), - MayReadFromMemory(CI.mayReadFromMemory()), - MayWriteToMemory(CI.mayWriteToMemory()), - MayHaveSideEffects(CI.mayHaveSideEffects()) {} + MayReadFromMemory(I.mayReadFromMemory()), + MayWriteToMemory(I.mayWriteToMemory()), + MayHaveSideEffects(I.mayHaveSideEffects()) {} VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID, ArrayRef CallArguments, Type *Ty, diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index d1922b4e0755e..312590093f97f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -962,18 +962,31 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) { // Use vector version of the intrinsic. Module *M = State.Builder.GetInsertBlock()->getModule(); + bool IsVPIntrinsic = VPIntrinsic::isVPIntrinsic(VectorIntrinsicID); Function *VectorF = - Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl); + IsVPIntrinsic + ? VPIntrinsic::getOrInsertDeclarationForParams(M, VectorIntrinsicID, + TysForDecl[0], Args) + : Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl); assert(VectorF && "Can't retrieve vector intrinsic."); - auto *CI = cast_or_null(getUnderlyingValue()); SmallVector OpBundles; - if (CI) - CI->getOperandBundlesAsDefs(OpBundles); + if (!IsVPIntrinsic) { + if (auto *CI = cast_or_null(getUnderlyingValue())) + CI->getOperandBundlesAsDefs(OpBundles); + } - CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles); + Value *V = State.Builder.CreateCall(VectorF, Args, OpBundles); - setFlags(V); + if (IsVPIntrinsic) { + // Currently vp-intrinsics only accept FMF flags. + // TODO: Enable other flags when support is added. + // vp_uitofp will get OperationType::NonNegOp + if (isa(V) && VectorIntrinsicID != Intrinsic::vp_uitofp) + setFlags(cast(V)); + } else { + setFlags(cast(V)); + } if (!V->getType()->isVoidTy()) State.set(this, V); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 56e5c67590f1d..aa9b0ec35546f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1503,22 +1503,23 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { }) .Case( [&](VPWidenCastRecipe *CInst) -> VPRecipeBase * { - auto *CI = cast(CInst->getUnderlyingInstr()); + auto *CI = dyn_cast(CInst->getUnderlyingInstr()); Intrinsic::ID VPID = VPIntrinsic::getForOpcode(CI->getOpcode()); if (VPID == Intrinsic::not_intrinsic) return nullptr; + SmallVector Ops(CInst->operands()); - if (VPIntrinsic::getMaskParamPos(VPID)) { - VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue( - IntegerType::getInt1Ty(CI->getContext()))); - Ops.push_back(Mask); - } - if (VPIntrinsic::getVectorLengthParamPos(VPID)) { - Ops.push_back(&EVL); - } + assert(VPIntrinsic::getMaskParamPos(VPID) && + VPIntrinsic::getVectorLengthParamPos(VPID) && + "Expected VP intrinsic"); + VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue( + IntegerType::getInt1Ty(CI->getContext()))); + Ops.push_back(Mask); + Ops.push_back(&EVL); return new VPWidenIntrinsicRecipe( - *CI, VPID, Ops, CI->getType(), CI->getDebugLoc()); + *CI, VPID, Ops, TypeInfo.inferScalarType(CInst), + CInst->getDebugLoc()); }) .Case([&](VPWidenSelectRecipe *Sel) { SmallVector Ops(Sel->operands()); diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll index 0d517407b5326..e9e5f2b2c4d2f 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll @@ -4,7 +4,7 @@ ; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \ ; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s -define void @vp_sext(ptr noalias %a, ptr noalias %b, i64 %N) { +define void @vp_sext(ptr %a, ptr %b, i64 %N) { ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count @@ -23,7 +23,7 @@ define void @vp_sext(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[SEXT:%.+]]> = call llvm.vp.sext(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[SEXT:%.+]]> = call llvm.vp.sext(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> ; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[SEXT]]>, vp<[[EVL]]> @@ -54,7 +54,7 @@ exit: ret void } -define void @vp_zext(ptr noalias %a, ptr noalias %b, i64 %N) { +define void @vp_zext(ptr %a, ptr %b, i64 %N) { ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count @@ -73,7 +73,7 @@ define void @vp_zext(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[ZEXT:%.+]]> = call llvm.vp.zext(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[ZEXT:%.+]]> = call llvm.vp.zext(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> ; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[ZEXT]]>, vp<[[EVL]]> @@ -102,7 +102,7 @@ exit: ret void } -define void @vp_truncate(ptr noalias %a, ptr noalias %b, i64 %N) { +define void @vp_truncate(ptr %a, ptr %b, i64 %N) { ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count @@ -121,7 +121,7 @@ define void @vp_truncate(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[TRUNC:%.+]]> = call llvm.vp.trunc(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[TRUNC:%.+]]> = call llvm.vp.trunc(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> ; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[TRUNC]]>, vp<[[EVL]]> @@ -150,7 +150,7 @@ exit: ret void } -define void @vp_fpext(ptr noalias %a, ptr noalias %b, i64 %N) { +define void @vp_fpext(ptr %a, ptr %b, i64 %N) { ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count @@ -169,7 +169,7 @@ define void @vp_fpext(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[FPEXT:%.+]]> = call llvm.vp.fpext(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[FPEXT:%.+]]> = call llvm.vp.fpext(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> ; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[FPEXT]]>, vp<[[EVL]]> @@ -198,7 +198,7 @@ exit: ret void } -define void @vp_fptrunct(ptr noalias %a, ptr noalias %b, i64 %N) { +define void @vp_fptrunct(ptr %a, ptr %b, i64 %N) { ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count @@ -217,7 +217,7 @@ define void @vp_fptrunct(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[FPTRUNC:%.+]]> = call llvm.vp.fptrunc(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[FPTRUNC:%.+]]> = call llvm.vp.fptrunc(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> ; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[FPTRUNC]]>, vp<[[EVL]]> @@ -246,7 +246,7 @@ exit: ret void } -define void @vp_fptosi(ptr noalias %a, ptr noalias %b, i64 %N) { +define void @vp_fptosi(ptr %a, ptr %b, i64 %N) { ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count @@ -265,7 +265,7 @@ define void @vp_fptosi(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[FPTOSI:%.+]]> = call llvm.vp.fptoui(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[FPTOSI:%.+]]> = call llvm.vp.fptoui(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> ; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[FPTOSI]]>, vp<[[EVL]]> @@ -294,7 +294,7 @@ exit: ret void } -define void @vp_inttofp(ptr noalias %a, ptr noalias %b, i64 %N) { +define void @vp_inttofp(ptr %a, ptr %b, i64 %N) { ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count @@ -313,7 +313,7 @@ define void @vp_inttofp(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[SITOFP:%.+]]> = call llvm.vp.sitofp(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[SITOFP:%.+]]> = call llvm.vp.sitofp(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> ; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[SITOFP]]>, vp<[[EVL]]> @@ -342,7 +342,7 @@ exit: ret void } -define void @vp_uinttofp(ptr noalias %a, ptr noalias %b, i64 %N) { +define void @vp_uinttofp(ptr %a, ptr %b, i64 %N) { ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count @@ -361,7 +361,7 @@ define void @vp_uinttofp(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[UITOFP:%.+]]> = call llvm.vp.uitofp(ir<[[LD1]]>, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[UITOFP:%.+]]> = call llvm.vp.uitofp(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> ; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[UITOFP]]>, vp<[[EVL]]> From 1d8f27cdb840238130328bde2fcfa325aaf6ed8a Mon Sep 17 00:00:00 2001 From: "Liqin.Weng" Date: Sun, 17 Nov 2024 15:12:05 +0800 Subject: [PATCH 07/11] fix the comments --- llvm/lib/Transforms/Vectorize/VPlan.h | 10 +++--- .../Transforms/Vectorize/VPlanTransforms.cpp | 2 +- .../LoopVectorize/RISCV/inloop-reduction.ll | 8 ++--- .../RISCV/vplan-vp-cast-intrinsics.ll | 34 +++++++++---------- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 65863fbd276dc..1b1630ebc6c23 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1650,14 +1650,14 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags { bool MayHaveSideEffects; public: - VPWidenIntrinsicRecipe(Instruction &I, Intrinsic::ID VectorIntrinsicID, + VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID, ArrayRef CallArguments, Type *Ty, DebugLoc DL = {}) - : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, I), + : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), - MayReadFromMemory(I.mayReadFromMemory()), - MayWriteToMemory(I.mayWriteToMemory()), - MayHaveSideEffects(I.mayHaveSideEffects()) {} + MayReadFromMemory(CI.mayReadFromMemory()), + MayWriteToMemory(CI.mayWriteToMemory()), + MayHaveSideEffects(CI.mayHaveSideEffects()) {} VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID, ArrayRef CallArguments, Type *Ty, diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index aa9b0ec35546f..4d8dc44cf8a67 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1518,7 +1518,7 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { Ops.push_back(Mask); Ops.push_back(&EVL); return new VPWidenIntrinsicRecipe( - *CI, VPID, Ops, TypeInfo.inferScalarType(CInst), + VPID, Ops, TypeInfo.inferScalarType(CInst), CInst->getDebugLoc()); }) .Case([&](VPWidenSelectRecipe *Sel) { diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll index 9ee1708f175ab..85f3b8066fc3d 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll @@ -142,10 +142,10 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) { ; IF-EVL-OUTLOOP-NEXT: [[TMP6:%.*]] = add i32 [[EVL_BASED_IV]], 0 ; IF-EVL-OUTLOOP-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP6]] ; IF-EVL-OUTLOOP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[TMP7]], i32 0 -; IF-EVL-OUTLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i16.p0(ptr align 2 [[TMP8]], splat (i1 true), i32 [[TMP5]]) -; IF-EVL-OUTLOOP-NEXT: [[TMP9:%.*]] = sext [[VP_OP_LOAD]] to -; IF-EVL-OUTLOOP-NEXT: [[VP_OP:%.*]] = call @llvm.vp.add.nxv4i32( [[VEC_PHI]], [[TMP9]], splat (i1 true), i32 [[TMP5]]) -; IF-EVL-OUTLOOP-NEXT: [[TMP10]] = call @llvm.vp.merge.nxv4i32( splat (i1 true), [[VP_OP]], [[VEC_PHI]], i32 [[TMP5]]) +; IF-EVL-OUTLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i16.p0(ptr align 2 [[TMP8]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP5]]) +; IF-EVL-OUTLOOP-NEXT: [[TMP9:%.*]] = call @llvm.vp.sext.nxv4i32.nxv4i16( [[VP_OP_LOAD]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP5]]) +; IF-EVL-OUTLOOP-NEXT: [[VP_OP:%.*]] = call @llvm.vp.add.nxv4i32( [[VEC_PHI]], [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP5]]) +; IF-EVL-OUTLOOP-NEXT: [[TMP10]] = call @llvm.vp.merge.nxv4i32( shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), [[VP_OP]], [[VEC_PHI]], i32 [[TMP5]]) ; IF-EVL-OUTLOOP-NEXT: [[INDEX_EVL_NEXT]] = add i32 [[TMP5]], [[EVL_BASED_IV]] ; IF-EVL-OUTLOOP-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP4]] ; IF-EVL-OUTLOOP-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll index e9e5f2b2c4d2f..05e1040df6cd8 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll @@ -23,10 +23,10 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[SEXT:%.+]]> = call llvm.vp.sext(ir<[[LD1]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SEXT:%.+]]> = call llvm.vp.sext(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[SEXT]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[SEXT]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -73,10 +73,10 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[ZEXT:%.+]]> = call llvm.vp.zext(ir<[[LD1]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[ZEXT:%.+]]> = call llvm.vp.zext(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[ZEXT]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[ZEXT]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -121,10 +121,10 @@ define void @vp_truncate(ptr %a, ptr %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[TRUNC:%.+]]> = call llvm.vp.trunc(ir<[[LD1]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[TRUNC:%.+]]> = call llvm.vp.trunc(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[TRUNC]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[TRUNC]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -169,10 +169,10 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[FPEXT:%.+]]> = call llvm.vp.fpext(ir<[[LD1]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[FPEXT:%.+]]> = call llvm.vp.fpext(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[FPEXT]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[FPEXT]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -217,10 +217,10 @@ define void @vp_fptrunct(ptr %a, ptr %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[FPTRUNC:%.+]]> = call llvm.vp.fptrunc(ir<[[LD1]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[FPTRUNC:%.+]]> = call llvm.vp.fptrunc(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[FPTRUNC]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[FPTRUNC]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -265,10 +265,10 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[FPTOSI:%.+]]> = call llvm.vp.fptoui(ir<[[LD1]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[FPTOSI:%.+]]> = call llvm.vp.fptoui(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[FPTOSI]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[FPTOSI]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -313,10 +313,10 @@ define void @vp_inttofp(ptr %a, ptr %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[SITOFP:%.+]]> = call llvm.vp.sitofp(ir<[[LD1]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SITOFP:%.+]]> = call llvm.vp.sitofp(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[SITOFP]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[SITOFP]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -361,10 +361,10 @@ define void @vp_uinttofp(ptr %a, ptr %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[UITOFP:%.+]]> = call llvm.vp.uitofp(ir<[[LD1]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[UITOFP:%.+]]> = call llvm.vp.uitofp(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[UITOFP]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[UITOFP]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -388,4 +388,4 @@ loop: exit: ret void -} \ No newline at end of file +} From 5bd40b9d801c72c22330d88dde183389ad7b1679 Mon Sep 17 00:00:00 2001 From: LiqinWeng Date: Thu, 28 Nov 2024 12:18:48 +0800 Subject: [PATCH 08/11] Rebase && add ir test && address the failed tests --- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 14 +- .../Transforms/Vectorize/VPlanTransforms.cpp | 11 +- .../LoopVectorize/RISCV/inloop-reduction.ll | 14 +- ...ize-force-tail-with-evl-call-intrinsics.ll | 8 +- ...ize-force-tail-with-evl-cast-intrinsics.ll | 1109 +++++++++++++++++ .../RISCV/vplan-vp-call-intrinsics.ll | 16 +- .../RISCV/vplan-vp-cast-intrinsics.ll | 160 ++- 7 files changed, 1277 insertions(+), 55 deletions(-) create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 312590093f97f..d9429da8dfa93 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -968,7 +968,8 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) { ? VPIntrinsic::getOrInsertDeclarationForParams(M, VectorIntrinsicID, TysForDecl[0], Args) : Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl); - assert(VectorF && "Can't retrieve vector intrinsic."); + assert(VectorF && + "Can't retrieve vector intrinsic or vector-predication intrinsics."); SmallVector OpBundles; if (!IsVPIntrinsic) { @@ -976,16 +977,15 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) { CI->getOperandBundlesAsDefs(OpBundles); } - Value *V = State.Builder.CreateCall(VectorF, Args, OpBundles); + Instruction *V = State.Builder.CreateCall(VectorF, Args, OpBundles); if (IsVPIntrinsic) { - // Currently vp-intrinsics only accept FMF flags. - // TODO: Enable other flags when support is added. - // vp_uitofp will get OperationType::NonNegOp + // Currently vp-intrinsics only accept FMF flags. llvm.vp.uitofp will get + // Flags of OperationType::NonNegOp && OperationType::FPMathOp. if (isa(V) && VectorIntrinsicID != Intrinsic::vp_uitofp) - setFlags(cast(V)); + setFlags(V); } else { - setFlags(cast(V)); + setFlags(V); } if (!V->getType()->isVoidTy()) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 4d8dc44cf8a67..f349313feec6e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1446,7 +1446,8 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { VPTypeAnalysis TypeInfo(CanonicalIVType); LLVMContext &Ctx = CanonicalIVType->getContext(); SmallVector HeaderMasks = collectAllHeaderMasks(Plan); - + VPValue *AllOneMask = + Plan.getOrAddLiveIn(ConstantInt::getTrue(IntegerType::getInt1Ty(Ctx))); for (VPUser *U : Plan.getVF().users()) { if (auto *R = dyn_cast(U)) R->setOperand(1, &EVL); @@ -1493,9 +1494,7 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { assert(VPIntrinsic::getMaskParamPos(VPID) && VPIntrinsic::getVectorLengthParamPos(VPID) && "Expected VP intrinsic"); - VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue( - IntegerType::getInt1Ty(CI->getContext()))); - Ops.push_back(Mask); + Ops.push_back(AllOneMask); Ops.push_back(&EVL); return new VPWidenIntrinsicRecipe( *CI, VPID, Ops, TypeInfo.inferScalarType(CInst), @@ -1513,9 +1512,7 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { assert(VPIntrinsic::getMaskParamPos(VPID) && VPIntrinsic::getVectorLengthParamPos(VPID) && "Expected VP intrinsic"); - VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue( - IntegerType::getInt1Ty(CI->getContext()))); - Ops.push_back(Mask); + Ops.push_back(AllOneMask); Ops.push_back(&EVL); return new VPWidenIntrinsicRecipe( VPID, Ops, TypeInfo.inferScalarType(CInst), diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll index 85f3b8066fc3d..ec1796fae03d9 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll @@ -142,10 +142,10 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) { ; IF-EVL-OUTLOOP-NEXT: [[TMP6:%.*]] = add i32 [[EVL_BASED_IV]], 0 ; IF-EVL-OUTLOOP-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP6]] ; IF-EVL-OUTLOOP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[TMP7]], i32 0 -; IF-EVL-OUTLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i16.p0(ptr align 2 [[TMP8]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP5]]) -; IF-EVL-OUTLOOP-NEXT: [[TMP9:%.*]] = call @llvm.vp.sext.nxv4i32.nxv4i16( [[VP_OP_LOAD]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP5]]) -; IF-EVL-OUTLOOP-NEXT: [[VP_OP:%.*]] = call @llvm.vp.add.nxv4i32( [[VEC_PHI]], [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP5]]) -; IF-EVL-OUTLOOP-NEXT: [[TMP10]] = call @llvm.vp.merge.nxv4i32( shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), [[VP_OP]], [[VEC_PHI]], i32 [[TMP5]]) +; IF-EVL-OUTLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i16.p0(ptr align 2 [[TMP8]], splat (i1 true), i32 [[TMP5]]) +; IF-EVL-OUTLOOP-NEXT: [[TMP9:%.*]] = call @llvm.vp.sext.nxv4i32.nxv4i16( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP5]]) +; IF-EVL-OUTLOOP-NEXT: [[VP_OP:%.*]] = call @llvm.vp.add.nxv4i32( [[VEC_PHI]], [[TMP9]], splat (i1 true), i32 [[TMP5]]) +; IF-EVL-OUTLOOP-NEXT: [[TMP10]] = call @llvm.vp.merge.nxv4i32( splat (i1 true), [[VP_OP]], [[VEC_PHI]], i32 [[TMP5]]) ; IF-EVL-OUTLOOP-NEXT: [[INDEX_EVL_NEXT]] = add i32 [[TMP5]], [[EVL_BASED_IV]] ; IF-EVL-OUTLOOP-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP4]] ; IF-EVL-OUTLOOP-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] @@ -199,9 +199,9 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) { ; IF-EVL-INLOOP-NEXT: [[TMP7:%.*]] = add i32 [[EVL_BASED_IV]], 0 ; IF-EVL-INLOOP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP7]] ; IF-EVL-INLOOP-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[TMP8]], i32 0 -; IF-EVL-INLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv8i16.p0(ptr align 2 [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP6]]) -; IF-EVL-INLOOP-NEXT: [[VP_CAST:%.*]] = call @llvm.vp.sext.nxv8i32.nxv8i16( [[VP_OP_LOAD]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP6]]) -; IF-EVL-INLOOP-NEXT: [[TMP10:%.*]] = call i32 @llvm.vp.reduce.add.nxv8i32(i32 0, [[VP_CAST]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), i32 [[TMP6]]) +; IF-EVL-INLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv8i16.p0(ptr align 2 [[TMP9]], splat (i1 true), i32 [[TMP6]]) +; IF-EVL-INLOOP-NEXT: [[TMP14:%.*]] = call @llvm.vp.sext.nxv8i32.nxv8i16( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP6]]) +; IF-EVL-INLOOP-NEXT: [[TMP10:%.*]] = call i32 @llvm.vp.reduce.add.nxv8i32(i32 0, [[TMP14]], splat (i1 true), i32 [[TMP6]]) ; IF-EVL-INLOOP-NEXT: [[TMP11]] = add i32 [[TMP10]], [[VEC_PHI]] ; IF-EVL-INLOOP-NEXT: [[INDEX_EVL_NEXT]] = add i32 [[TMP6]], [[EVL_BASED_IV]] ; IF-EVL-INLOOP-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP4]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll index 64f86c6cdeed6..66a5e086a0117 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll @@ -708,9 +708,9 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) { ; IF-EVL-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP10]] ; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i32 0 ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], splat (i1 true), i32 [[TMP9]]) -; IF-EVL-NEXT: [[TMP13:%.*]] = fpext [[VP_OP_LOAD]] to +; IF-EVL-NEXT: [[TMP13:%.*]] = call @llvm.vp.fpext.nxv4f64.nxv4f32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP14:%.*]] = call @llvm.vp.lrint.nxv4i64.nxv4f64( [[TMP13]], splat (i1 true), i32 [[TMP9]]) -; IF-EVL-NEXT: [[TMP15:%.*]] = trunc [[TMP14]] to +; IF-EVL-NEXT: [[TMP15:%.*]] = call @llvm.vp.trunc.nxv4i32.nxv4i64( [[TMP14]], splat (i1 true), i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]] ; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0 ; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0( [[TMP15]], ptr align 4 [[TMP17]], splat (i1 true), i32 [[TMP9]]) @@ -817,9 +817,9 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) { ; IF-EVL-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP10]] ; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i32 0 ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], splat (i1 true), i32 [[TMP9]]) -; IF-EVL-NEXT: [[TMP13:%.*]] = fpext [[VP_OP_LOAD]] to +; IF-EVL-NEXT: [[TMP13:%.*]] = call @llvm.vp.fpext.nxv4f64.nxv4f32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP14:%.*]] = call @llvm.vp.llrint.nxv4i64.nxv4f64( [[TMP13]], splat (i1 true), i32 [[TMP9]]) -; IF-EVL-NEXT: [[TMP15:%.*]] = trunc [[TMP14]] to +; IF-EVL-NEXT: [[TMP15:%.*]] = call @llvm.vp.trunc.nxv4i32.nxv4i64( [[TMP14]], splat (i1 true), i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]] ; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0 ; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0( [[TMP15]], ptr align 4 [[TMP17]], splat (i1 true), i32 [[TMP9]]) diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll new file mode 100644 index 0000000000000..4557e95f1e1b6 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll @@ -0,0 +1,1109 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -passes=loop-vectorize \ +; RUN: -force-tail-folding-style=data-with-evl \ +; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \ +; RUN: -mtriple=riscv64 -mattr=+v -S %s | FileCheck %s --check-prefix=IF-EVL + +; RUN: opt -passes=loop-vectorize \ +; RUN: -force-tail-folding-style=none \ +; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \ +; RUN: -mtriple=riscv64 -mattr=+v -S %s | FileCheck %s --check-prefix=NO-VP + +define void @vp_sext(ptr %a, ptr %b, i64 %N) { +; IF-EVL-LABEL: define void @vp_sext( +; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { +; IF-EVL-NEXT: [[ENTRY:.*]]: +; IF-EVL-NEXT: [[TMP0:%.*]] = sub i64 -1, [[N]] +; IF-EVL-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 2 +; IF-EVL-NEXT: [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 18, i64 [[TMP2]]) +; IF-EVL-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]] +; IF-EVL-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; IF-EVL: [[VECTOR_MEMCHECK]]: +; IF-EVL-NEXT: [[TMP5:%.*]] = shl i64 [[N]], 3 +; IF-EVL-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]] +; IF-EVL-NEXT: [[TMP6:%.*]] = shl i64 [[N]], 2 +; IF-EVL-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP6]] +; IF-EVL-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]] +; IF-EVL-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]] +; IF-EVL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] +; IF-EVL-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; IF-EVL: [[VECTOR_PH]]: +; IF-EVL-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 +; IF-EVL-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], 1 +; IF-EVL-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]] +; IF-EVL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]] +; IF-EVL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; IF-EVL-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 +; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]] +; IF-EVL: [[VECTOR_BODY]]: +; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true) +; IF-EVL-NEXT: [[TMP13:%.*]] = add i64 [[EVL_BASED_IV]], 0 +; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP13]] +; IF-EVL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0 +; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv2i32.p0(ptr align 4 [[TMP15]], splat (i1 true), i32 [[TMP12]]), !alias.scope [[META0:![0-9]+]] +; IF-EVL-NEXT: [[TMP16:%.*]] = call @llvm.vp.sext.nxv2i64.nxv2i32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP12]]) +; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP13]] +; IF-EVL-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP17]], i32 0 +; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0( [[TMP16]], ptr align 8 [[TMP18]], splat (i1 true), i32 [[TMP12]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]] +; IF-EVL-NEXT: [[TMP19:%.*]] = zext i32 [[TMP12]] to i64 +; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]] +; IF-EVL-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; IF-EVL-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] +; IF-EVL: [[MIDDLE_BLOCK]]: +; IF-EVL-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; IF-EVL: [[SCALAR_PH]]: +; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ] +; IF-EVL-NEXT: br label %[[LOOP:.*]] +; IF-EVL: [[LOOP]]: +; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ] +; IF-EVL-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] +; IF-EVL-NEXT: [[TMP21:%.*]] = load i32, ptr [[GEP]], align 4 +; IF-EVL-NEXT: [[CONV2:%.*]] = sext i32 [[TMP21]] to i64 +; IF-EVL-NEXT: [[GEP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] +; IF-EVL-NEXT: store i64 [[CONV2]], ptr [[GEP4]], align 8 +; IF-EVL-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; IF-EVL-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; IF-EVL-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]] +; IF-EVL: [[EXIT]]: +; IF-EVL-NEXT: ret void +; +; NO-VP-LABEL: define void @vp_sext( +; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { +; NO-VP-NEXT: [[ENTRY:.*]]: +; NO-VP-NEXT: br label %[[LOOP:.*]] +; NO-VP: [[LOOP]]: +; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ] +; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] +; NO-VP-NEXT: [[TMP0:%.*]] = load i32, ptr [[GEP]], align 4 +; NO-VP-NEXT: [[CONV2:%.*]] = sext i32 [[TMP0]] to i64 +; NO-VP-NEXT: [[GEP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] +; NO-VP-NEXT: store i64 [[CONV2]], ptr [[GEP4]], align 8 +; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]] +; NO-VP: [[EXIT]]: +; NO-VP-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] + %gep = getelementptr inbounds i32, ptr %b, i64 %iv + %0 = load i32, ptr %gep, align 4 + %conv2 = sext i32 %0 to i64 + %gep4 = getelementptr inbounds i64, ptr %a, i64 %iv + store i64 %conv2, ptr %gep4, align 8 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} + +define void @vp_zext(ptr %a, ptr %b, i64 %N) { +; IF-EVL-LABEL: define void @vp_zext( +; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; IF-EVL-NEXT: [[ENTRY:.*]]: +; IF-EVL-NEXT: [[TMP0:%.*]] = sub i64 -1, [[N]] +; IF-EVL-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 2 +; IF-EVL-NEXT: [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 18, i64 [[TMP2]]) +; IF-EVL-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]] +; IF-EVL-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; IF-EVL: [[VECTOR_MEMCHECK]]: +; IF-EVL-NEXT: [[TMP5:%.*]] = shl i64 [[N]], 3 +; IF-EVL-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]] +; IF-EVL-NEXT: [[TMP6:%.*]] = shl i64 [[N]], 2 +; IF-EVL-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP6]] +; IF-EVL-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]] +; IF-EVL-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]] +; IF-EVL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] +; IF-EVL-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; IF-EVL: [[VECTOR_PH]]: +; IF-EVL-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 +; IF-EVL-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], 1 +; IF-EVL-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]] +; IF-EVL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]] +; IF-EVL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; IF-EVL-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 +; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]] +; IF-EVL: [[VECTOR_BODY]]: +; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true) +; IF-EVL-NEXT: [[TMP13:%.*]] = add i64 [[EVL_BASED_IV]], 0 +; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP13]] +; IF-EVL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0 +; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv2i32.p0(ptr align 4 [[TMP15]], splat (i1 true), i32 [[TMP12]]), !alias.scope [[META9:![0-9]+]] +; IF-EVL-NEXT: [[TMP16:%.*]] = call @llvm.vp.zext.nxv2i64.nxv2i32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP12]]) +; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP13]] +; IF-EVL-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP17]], i32 0 +; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0( [[TMP16]], ptr align 8 [[TMP18]], splat (i1 true), i32 [[TMP12]]), !alias.scope [[META12:![0-9]+]], !noalias [[META9]] +; IF-EVL-NEXT: [[TMP19:%.*]] = zext i32 [[TMP12]] to i64 +; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]] +; IF-EVL-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; IF-EVL-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] +; IF-EVL: [[MIDDLE_BLOCK]]: +; IF-EVL-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; IF-EVL: [[SCALAR_PH]]: +; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ] +; IF-EVL-NEXT: br label %[[LOOP:.*]] +; IF-EVL: [[LOOP]]: +; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; IF-EVL-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] +; IF-EVL-NEXT: [[TMP21:%.*]] = load i32, ptr [[GEP]], align 4 +; IF-EVL-NEXT: [[CONV:%.*]] = zext i32 [[TMP21]] to i64 +; IF-EVL-NEXT: [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] +; IF-EVL-NEXT: store i64 [[CONV]], ptr [[GEP2]], align 8 +; IF-EVL-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; IF-EVL-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; IF-EVL-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]] +; IF-EVL: [[EXIT]]: +; IF-EVL-NEXT: ret void +; +; NO-VP-LABEL: define void @vp_zext( +; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; NO-VP-NEXT: [[ENTRY:.*]]: +; NO-VP-NEXT: br label %[[LOOP:.*]] +; NO-VP: [[LOOP]]: +; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] +; NO-VP-NEXT: [[TMP0:%.*]] = load i32, ptr [[GEP]], align 4 +; NO-VP-NEXT: [[CONV:%.*]] = zext i32 [[TMP0]] to i64 +; NO-VP-NEXT: [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] +; NO-VP-NEXT: store i64 [[CONV]], ptr [[GEP2]], align 8 +; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]] +; NO-VP: [[EXIT]]: +; NO-VP-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds i32, ptr %b, i64 %iv + %0 = load i32, ptr %gep, align 4 + %conv = zext i32 %0 to i64 + %gep2 = getelementptr inbounds i64, ptr %a, i64 %iv + store i64 %conv, ptr %gep2, align 8 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} + +define void @vp_trunc(ptr %a, ptr %b, i64 %N) { +; IF-EVL-LABEL: define void @vp_trunc( +; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; IF-EVL-NEXT: [[ENTRY:.*]]: +; IF-EVL-NEXT: [[TMP0:%.*]] = sub i64 -1, [[N]] +; IF-EVL-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 2 +; IF-EVL-NEXT: [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 18, i64 [[TMP2]]) +; IF-EVL-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]] +; IF-EVL-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; IF-EVL: [[VECTOR_MEMCHECK]]: +; IF-EVL-NEXT: [[TMP5:%.*]] = shl i64 [[N]], 2 +; IF-EVL-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]] +; IF-EVL-NEXT: [[TMP6:%.*]] = shl i64 [[N]], 3 +; IF-EVL-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP6]] +; IF-EVL-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]] +; IF-EVL-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]] +; IF-EVL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] +; IF-EVL-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; IF-EVL: [[VECTOR_PH]]: +; IF-EVL-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 +; IF-EVL-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], 1 +; IF-EVL-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]] +; IF-EVL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]] +; IF-EVL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; IF-EVL-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 +; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]] +; IF-EVL: [[VECTOR_BODY]]: +; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true) +; IF-EVL-NEXT: [[TMP13:%.*]] = add i64 [[EVL_BASED_IV]], 0 +; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP13]] +; IF-EVL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP14]], i32 0 +; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP15]], splat (i1 true), i32 [[TMP12]]), !alias.scope [[META16:![0-9]+]] +; IF-EVL-NEXT: [[TMP16:%.*]] = call @llvm.vp.trunc.nxv2i32.nxv2i64( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP12]]) +; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP13]] +; IF-EVL-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i32 0 +; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i32.p0( [[TMP16]], ptr align 4 [[TMP18]], splat (i1 true), i32 [[TMP12]]), !alias.scope [[META19:![0-9]+]], !noalias [[META16]] +; IF-EVL-NEXT: [[TMP19:%.*]] = zext i32 [[TMP12]] to i64 +; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]] +; IF-EVL-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; IF-EVL-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] +; IF-EVL: [[MIDDLE_BLOCK]]: +; IF-EVL-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; IF-EVL: [[SCALAR_PH]]: +; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ] +; IF-EVL-NEXT: br label %[[LOOP:.*]] +; IF-EVL: [[LOOP]]: +; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; IF-EVL-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]] +; IF-EVL-NEXT: [[TMP21:%.*]] = load i64, ptr [[GEP]], align 8 +; IF-EVL-NEXT: [[CONV:%.*]] = trunc i64 [[TMP21]] to i32 +; IF-EVL-NEXT: [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]] +; IF-EVL-NEXT: store i32 [[CONV]], ptr [[GEP2]], align 4 +; IF-EVL-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; IF-EVL-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; IF-EVL-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP22:![0-9]+]] +; IF-EVL: [[EXIT]]: +; IF-EVL-NEXT: ret void +; +; NO-VP-LABEL: define void @vp_trunc( +; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; NO-VP-NEXT: [[ENTRY:.*]]: +; NO-VP-NEXT: br label %[[LOOP:.*]] +; NO-VP: [[LOOP]]: +; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]] +; NO-VP-NEXT: [[TMP0:%.*]] = load i64, ptr [[GEP]], align 8 +; NO-VP-NEXT: [[CONV:%.*]] = trunc i64 [[TMP0]] to i32 +; NO-VP-NEXT: [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]] +; NO-VP-NEXT: store i32 [[CONV]], ptr [[GEP2]], align 4 +; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]] +; NO-VP: [[EXIT]]: +; NO-VP-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds i64, ptr %b, i64 %iv + %0 = load i64, ptr %gep, align 8 + %conv = trunc i64 %0 to i32 + %gep2 = getelementptr inbounds i32, ptr %a, i64 %iv + store i32 %conv, ptr %gep2, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} + +define void @vp_fpext(ptr %a, ptr %b, i64 %N) { +; IF-EVL-LABEL: define void @vp_fpext( +; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; IF-EVL-NEXT: [[ENTRY:.*]]: +; IF-EVL-NEXT: [[TMP0:%.*]] = sub i64 -1, [[N]] +; IF-EVL-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 2 +; IF-EVL-NEXT: [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 14, i64 [[TMP2]]) +; IF-EVL-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]] +; IF-EVL-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; IF-EVL: [[VECTOR_MEMCHECK]]: +; IF-EVL-NEXT: [[TMP5:%.*]] = shl i64 [[N]], 3 +; IF-EVL-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]] +; IF-EVL-NEXT: [[TMP6:%.*]] = shl i64 [[N]], 2 +; IF-EVL-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP6]] +; IF-EVL-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]] +; IF-EVL-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]] +; IF-EVL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] +; IF-EVL-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; IF-EVL: [[VECTOR_PH]]: +; IF-EVL-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 +; IF-EVL-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], 1 +; IF-EVL-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]] +; IF-EVL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]] +; IF-EVL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; IF-EVL-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 +; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]] +; IF-EVL: [[VECTOR_BODY]]: +; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true) +; IF-EVL-NEXT: [[TMP13:%.*]] = add i64 [[EVL_BASED_IV]], 0 +; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP13]] +; IF-EVL-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i32 0 +; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv2f32.p0(ptr align 4 [[TMP15]], splat (i1 true), i32 [[TMP12]]), !alias.scope [[META23:![0-9]+]] +; IF-EVL-NEXT: [[TMP16:%.*]] = call @llvm.vp.fpext.nxv2f64.nxv2f32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP12]]) +; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[TMP13]] +; IF-EVL-NEXT: [[TMP18:%.*]] = getelementptr inbounds double, ptr [[TMP17]], i32 0 +; IF-EVL-NEXT: call void @llvm.vp.store.nxv2f64.p0( [[TMP16]], ptr align 8 [[TMP18]], splat (i1 true), i32 [[TMP12]]), !alias.scope [[META26:![0-9]+]], !noalias [[META23]] +; IF-EVL-NEXT: [[TMP19:%.*]] = zext i32 [[TMP12]] to i64 +; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]] +; IF-EVL-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; IF-EVL-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]] +; IF-EVL: [[MIDDLE_BLOCK]]: +; IF-EVL-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; IF-EVL: [[SCALAR_PH]]: +; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ] +; IF-EVL-NEXT: br label %[[LOOP:.*]] +; IF-EVL: [[LOOP]]: +; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; IF-EVL-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]] +; IF-EVL-NEXT: [[TMP21:%.*]] = load float, ptr [[GEP]], align 4 +; IF-EVL-NEXT: [[CONV:%.*]] = fpext float [[TMP21]] to double +; IF-EVL-NEXT: [[GEP2:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IV]] +; IF-EVL-NEXT: store double [[CONV]], ptr [[GEP2]], align 8 +; IF-EVL-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; IF-EVL-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; IF-EVL-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP29:![0-9]+]] +; IF-EVL: [[EXIT]]: +; IF-EVL-NEXT: ret void +; +; NO-VP-LABEL: define void @vp_fpext( +; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; NO-VP-NEXT: [[ENTRY:.*]]: +; NO-VP-NEXT: br label %[[LOOP:.*]] +; NO-VP: [[LOOP]]: +; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]] +; NO-VP-NEXT: [[TMP0:%.*]] = load float, ptr [[GEP]], align 4 +; NO-VP-NEXT: [[CONV:%.*]] = fpext float [[TMP0]] to double +; NO-VP-NEXT: [[GEP2:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IV]] +; NO-VP-NEXT: store double [[CONV]], ptr [[GEP2]], align 8 +; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]] +; NO-VP: [[EXIT]]: +; NO-VP-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds float, ptr %b, i64 %iv + %0 = load float, ptr %gep, align 4 + %conv = fpext float %0 to double + %gep2 = getelementptr inbounds double, ptr %a, i64 %iv + store double %conv, ptr %gep2, align 8 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} + +define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) { +; IF-EVL-LABEL: define void @vp_fptrunc( +; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; IF-EVL-NEXT: [[ENTRY:.*]]: +; IF-EVL-NEXT: [[TMP0:%.*]] = sub i64 -1, [[N]] +; IF-EVL-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 2 +; IF-EVL-NEXT: [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 14, i64 [[TMP2]]) +; IF-EVL-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]] +; IF-EVL-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; IF-EVL: [[VECTOR_MEMCHECK]]: +; IF-EVL-NEXT: [[TMP5:%.*]] = shl i64 [[N]], 2 +; IF-EVL-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]] +; IF-EVL-NEXT: [[TMP6:%.*]] = shl i64 [[N]], 3 +; IF-EVL-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP6]] +; IF-EVL-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]] +; IF-EVL-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]] +; IF-EVL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] +; IF-EVL-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; IF-EVL: [[VECTOR_PH]]: +; IF-EVL-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 +; IF-EVL-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], 1 +; IF-EVL-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]] +; IF-EVL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]] +; IF-EVL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; IF-EVL-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 +; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]] +; IF-EVL: [[VECTOR_BODY]]: +; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true) +; IF-EVL-NEXT: [[TMP13:%.*]] = add i64 [[EVL_BASED_IV]], 0 +; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[TMP13]] +; IF-EVL-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i32 0 +; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv2f64.p0(ptr align 8 [[TMP15]], splat (i1 true), i32 [[TMP12]]), !alias.scope [[META30:![0-9]+]] +; IF-EVL-NEXT: [[TMP16:%.*]] = call @llvm.vp.fptrunc.nxv2f32.nxv2f64( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP12]]) +; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP13]] +; IF-EVL-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0 +; IF-EVL-NEXT: call void @llvm.vp.store.nxv2f32.p0( [[TMP16]], ptr align 4 [[TMP18]], splat (i1 true), i32 [[TMP12]]), !alias.scope [[META33:![0-9]+]], !noalias [[META30]] +; IF-EVL-NEXT: [[TMP19:%.*]] = zext i32 [[TMP12]] to i64 +; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP19]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP11]] +; IF-EVL-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; IF-EVL-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]] +; IF-EVL: [[MIDDLE_BLOCK]]: +; IF-EVL-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; IF-EVL: [[SCALAR_PH]]: +; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ] +; IF-EVL-NEXT: br label %[[LOOP:.*]] +; IF-EVL: [[LOOP]]: +; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; IF-EVL-NEXT: [[GEP:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[IV]] +; IF-EVL-NEXT: [[TMP21:%.*]] = load double, ptr [[GEP]], align 8 +; IF-EVL-NEXT: [[CONV:%.*]] = fptrunc double [[TMP21]] to float +; IF-EVL-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]] +; IF-EVL-NEXT: store float [[CONV]], ptr [[GEP2]], align 4 +; IF-EVL-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; IF-EVL-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; IF-EVL-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP36:![0-9]+]] +; IF-EVL: [[EXIT]]: +; IF-EVL-NEXT: ret void +; +; NO-VP-LABEL: define void @vp_fptrunc( +; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; NO-VP-NEXT: [[ENTRY:.*]]: +; NO-VP-NEXT: br label %[[LOOP:.*]] +; NO-VP: [[LOOP]]: +; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[IV]] +; NO-VP-NEXT: [[TMP0:%.*]] = load double, ptr [[GEP]], align 8 +; NO-VP-NEXT: [[CONV:%.*]] = fptrunc double [[TMP0]] to float +; NO-VP-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]] +; NO-VP-NEXT: store float [[CONV]], ptr [[GEP2]], align 4 +; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]] +; NO-VP: [[EXIT]]: +; NO-VP-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds double, ptr %b, i64 %iv + %0 = load double, ptr %gep, align 8 + %conv = fptrunc double %0 to float + %gep2 = getelementptr inbounds float, ptr %a, i64 %iv + store float %conv, ptr %gep2, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} + +define void @vp_sitofp(ptr %a, ptr %b, i64 %N) { +; IF-EVL-LABEL: define void @vp_sitofp( +; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; IF-EVL-NEXT: [[ENTRY:.*]]: +; IF-EVL-NEXT: [[B2:%.*]] = ptrtoint ptr [[B]] to i64 +; IF-EVL-NEXT: [[A1:%.*]] = ptrtoint ptr [[A]] to i64 +; IF-EVL-NEXT: [[TMP0:%.*]] = sub i64 -1, [[N]] +; IF-EVL-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 4 +; IF-EVL-NEXT: [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]]) +; IF-EVL-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]] +; IF-EVL-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; IF-EVL: [[VECTOR_MEMCHECK]]: +; IF-EVL-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 +; IF-EVL-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 +; IF-EVL-NEXT: [[TMP8:%.*]] = sub i64 [[A1]], [[B2]] +; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]] +; IF-EVL-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; IF-EVL: [[VECTOR_PH]]: +; IF-EVL-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 +; IF-EVL-NEXT: [[TMP11:%.*]] = sub i64 [[TMP10]], 1 +; IF-EVL-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]] +; IF-EVL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]] +; IF-EVL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; IF-EVL-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 +; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]] +; IF-EVL: [[VECTOR_BODY]]: +; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; IF-EVL-NEXT: [[TMP15:%.*]] = add i64 [[EVL_BASED_IV]], 0 +; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP15]] +; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0 +; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP17]], splat (i1 true), i32 [[TMP14]]) +; IF-EVL-NEXT: [[TMP18:%.*]] = call @llvm.vp.sitofp.nxv4f32.nxv4i32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP14]]) +; IF-EVL-NEXT: [[TMP19:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP15]] +; IF-EVL-NEXT: [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP19]], i32 0 +; IF-EVL-NEXT: call void @llvm.vp.store.nxv4f32.p0( [[TMP18]], ptr align 4 [[TMP20]], splat (i1 true), i32 [[TMP14]]) +; IF-EVL-NEXT: [[TMP21:%.*]] = zext i32 [[TMP14]] to i64 +; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP21]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]] +; IF-EVL-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; IF-EVL-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]] +; IF-EVL: [[MIDDLE_BLOCK]]: +; IF-EVL-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; IF-EVL: [[SCALAR_PH]]: +; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ] +; IF-EVL-NEXT: br label %[[LOOP:.*]] +; IF-EVL: [[LOOP]]: +; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; IF-EVL-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] +; IF-EVL-NEXT: [[TMP23:%.*]] = load i32, ptr [[GEP]], align 4 +; IF-EVL-NEXT: [[CONV:%.*]] = sitofp i32 [[TMP23]] to float +; IF-EVL-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]] +; IF-EVL-NEXT: store float [[CONV]], ptr [[GEP2]], align 4 +; IF-EVL-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; IF-EVL-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; IF-EVL-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP38:![0-9]+]] +; IF-EVL: [[EXIT]]: +; IF-EVL-NEXT: ret void +; +; NO-VP-LABEL: define void @vp_sitofp( +; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; NO-VP-NEXT: [[ENTRY:.*]]: +; NO-VP-NEXT: br label %[[LOOP:.*]] +; NO-VP: [[LOOP]]: +; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] +; NO-VP-NEXT: [[TMP0:%.*]] = load i32, ptr [[GEP]], align 4 +; NO-VP-NEXT: [[CONV:%.*]] = sitofp i32 [[TMP0]] to float +; NO-VP-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]] +; NO-VP-NEXT: store float [[CONV]], ptr [[GEP2]], align 4 +; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]] +; NO-VP: [[EXIT]]: +; NO-VP-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds i32, ptr %b, i64 %iv + %0 = load i32, ptr %gep, align 4 + %conv = sitofp i32 %0 to float + %gep2 = getelementptr inbounds float, ptr %a, i64 %iv + store float %conv, ptr %gep2, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} + +define void @vp_uitofp(ptr %a, ptr %b, i64 %N) { +; IF-EVL-LABEL: define void @vp_uitofp( +; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; IF-EVL-NEXT: [[ENTRY:.*]]: +; IF-EVL-NEXT: [[B2:%.*]] = ptrtoint ptr [[B]] to i64 +; IF-EVL-NEXT: [[A1:%.*]] = ptrtoint ptr [[A]] to i64 +; IF-EVL-NEXT: [[TMP0:%.*]] = sub i64 -1, [[N]] +; IF-EVL-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 4 +; IF-EVL-NEXT: [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]]) +; IF-EVL-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]] +; IF-EVL-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; IF-EVL: [[VECTOR_MEMCHECK]]: +; IF-EVL-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 +; IF-EVL-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 +; IF-EVL-NEXT: [[TMP8:%.*]] = sub i64 [[A1]], [[B2]] +; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]] +; IF-EVL-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; IF-EVL: [[VECTOR_PH]]: +; IF-EVL-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 +; IF-EVL-NEXT: [[TMP11:%.*]] = sub i64 [[TMP10]], 1 +; IF-EVL-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]] +; IF-EVL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]] +; IF-EVL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; IF-EVL-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 +; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]] +; IF-EVL: [[VECTOR_BODY]]: +; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; IF-EVL-NEXT: [[TMP15:%.*]] = add i64 [[EVL_BASED_IV]], 0 +; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP15]] +; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0 +; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP17]], splat (i1 true), i32 [[TMP14]]) +; IF-EVL-NEXT: [[TMP18:%.*]] = call @llvm.vp.uitofp.nxv4f32.nxv4i32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP14]]) +; IF-EVL-NEXT: [[TMP19:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP15]] +; IF-EVL-NEXT: [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP19]], i32 0 +; IF-EVL-NEXT: call void @llvm.vp.store.nxv4f32.p0( [[TMP18]], ptr align 4 [[TMP20]], splat (i1 true), i32 [[TMP14]]) +; IF-EVL-NEXT: [[TMP21:%.*]] = zext i32 [[TMP14]] to i64 +; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP21]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]] +; IF-EVL-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; IF-EVL-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]] +; IF-EVL: [[MIDDLE_BLOCK]]: +; IF-EVL-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; IF-EVL: [[SCALAR_PH]]: +; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ] +; IF-EVL-NEXT: br label %[[LOOP:.*]] +; IF-EVL: [[LOOP]]: +; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; IF-EVL-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] +; IF-EVL-NEXT: [[TMP23:%.*]] = load i32, ptr [[GEP]], align 4 +; IF-EVL-NEXT: [[CONV:%.*]] = uitofp i32 [[TMP23]] to float +; IF-EVL-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]] +; IF-EVL-NEXT: store float [[CONV]], ptr [[GEP2]], align 4 +; IF-EVL-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; IF-EVL-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; IF-EVL-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP40:![0-9]+]] +; IF-EVL: [[EXIT]]: +; IF-EVL-NEXT: ret void +; +; NO-VP-LABEL: define void @vp_uitofp( +; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; NO-VP-NEXT: [[ENTRY:.*]]: +; NO-VP-NEXT: br label %[[LOOP:.*]] +; NO-VP: [[LOOP]]: +; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] +; NO-VP-NEXT: [[TMP0:%.*]] = load i32, ptr [[GEP]], align 4 +; NO-VP-NEXT: [[CONV:%.*]] = uitofp i32 [[TMP0]] to float +; NO-VP-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]] +; NO-VP-NEXT: store float [[CONV]], ptr [[GEP2]], align 4 +; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]] +; NO-VP: [[EXIT]]: +; NO-VP-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds i32, ptr %b, i64 %iv + %0 = load i32, ptr %gep, align 4 + %conv = uitofp i32 %0 to float + %gep2 = getelementptr inbounds float, ptr %a, i64 %iv + store float %conv, ptr %gep2, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} + +define void @vp_fptosi(ptr %a, ptr %b, i64 %N) { +; IF-EVL-LABEL: define void @vp_fptosi( +; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; IF-EVL-NEXT: [[ENTRY:.*]]: +; IF-EVL-NEXT: [[B2:%.*]] = ptrtoint ptr [[B]] to i64 +; IF-EVL-NEXT: [[A1:%.*]] = ptrtoint ptr [[A]] to i64 +; IF-EVL-NEXT: [[TMP0:%.*]] = sub i64 -1, [[N]] +; IF-EVL-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 4 +; IF-EVL-NEXT: [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]]) +; IF-EVL-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]] +; IF-EVL-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; IF-EVL: [[VECTOR_MEMCHECK]]: +; IF-EVL-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 +; IF-EVL-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 +; IF-EVL-NEXT: [[TMP8:%.*]] = sub i64 [[A1]], [[B2]] +; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]] +; IF-EVL-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; IF-EVL: [[VECTOR_PH]]: +; IF-EVL-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 +; IF-EVL-NEXT: [[TMP11:%.*]] = sub i64 [[TMP10]], 1 +; IF-EVL-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]] +; IF-EVL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]] +; IF-EVL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; IF-EVL-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 +; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]] +; IF-EVL: [[VECTOR_BODY]]: +; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; IF-EVL-NEXT: [[TMP15:%.*]] = add i64 [[EVL_BASED_IV]], 0 +; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP15]] +; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i32 0 +; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP17]], splat (i1 true), i32 [[TMP14]]) +; IF-EVL-NEXT: [[TMP18:%.*]] = call @llvm.vp.fptosi.nxv4i32.nxv4f32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP14]]) +; IF-EVL-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP15]] +; IF-EVL-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i32 0 +; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0( [[TMP18]], ptr align 4 [[TMP20]], splat (i1 true), i32 [[TMP14]]) +; IF-EVL-NEXT: [[TMP21:%.*]] = zext i32 [[TMP14]] to i64 +; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP21]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]] +; IF-EVL-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; IF-EVL-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] +; IF-EVL: [[MIDDLE_BLOCK]]: +; IF-EVL-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; IF-EVL: [[SCALAR_PH]]: +; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ] +; IF-EVL-NEXT: br label %[[LOOP:.*]] +; IF-EVL: [[LOOP]]: +; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; IF-EVL-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]] +; IF-EVL-NEXT: [[TMP23:%.*]] = load float, ptr [[GEP]], align 4 +; IF-EVL-NEXT: [[CONV:%.*]] = fptosi float [[TMP23]] to i32 +; IF-EVL-NEXT: [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]] +; IF-EVL-NEXT: store i32 [[CONV]], ptr [[GEP2]], align 4 +; IF-EVL-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; IF-EVL-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; IF-EVL-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP42:![0-9]+]] +; IF-EVL: [[EXIT]]: +; IF-EVL-NEXT: ret void +; +; NO-VP-LABEL: define void @vp_fptosi( +; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; NO-VP-NEXT: [[ENTRY:.*]]: +; NO-VP-NEXT: br label %[[LOOP:.*]] +; NO-VP: [[LOOP]]: +; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]] +; NO-VP-NEXT: [[TMP0:%.*]] = load float, ptr [[GEP]], align 4 +; NO-VP-NEXT: [[CONV:%.*]] = fptosi float [[TMP0]] to i32 +; NO-VP-NEXT: [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]] +; NO-VP-NEXT: store i32 [[CONV]], ptr [[GEP2]], align 4 +; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]] +; NO-VP: [[EXIT]]: +; NO-VP-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds float, ptr %b, i64 %iv + %0 = load float, ptr %gep, align 4 + %conv = fptosi float %0 to i32 + %gep2 = getelementptr inbounds i32, ptr %a, i64 %iv + store i32 %conv, ptr %gep2, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} + +define void @vp_fptoui(ptr %a, ptr %b, i64 %N) { +; IF-EVL-LABEL: define void @vp_fptoui( +; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; IF-EVL-NEXT: [[ENTRY:.*]]: +; IF-EVL-NEXT: [[B2:%.*]] = ptrtoint ptr [[B]] to i64 +; IF-EVL-NEXT: [[A1:%.*]] = ptrtoint ptr [[A]] to i64 +; IF-EVL-NEXT: [[TMP0:%.*]] = sub i64 -1, [[N]] +; IF-EVL-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 4 +; IF-EVL-NEXT: [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]]) +; IF-EVL-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]] +; IF-EVL-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; IF-EVL: [[VECTOR_MEMCHECK]]: +; IF-EVL-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 +; IF-EVL-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 +; IF-EVL-NEXT: [[TMP8:%.*]] = sub i64 [[A1]], [[B2]] +; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]] +; IF-EVL-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; IF-EVL: [[VECTOR_PH]]: +; IF-EVL-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 +; IF-EVL-NEXT: [[TMP11:%.*]] = sub i64 [[TMP10]], 1 +; IF-EVL-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]] +; IF-EVL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]] +; IF-EVL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; IF-EVL-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 +; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]] +; IF-EVL: [[VECTOR_BODY]]: +; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; IF-EVL-NEXT: [[TMP15:%.*]] = add i64 [[EVL_BASED_IV]], 0 +; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP15]] +; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i32 0 +; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP17]], splat (i1 true), i32 [[TMP14]]) +; IF-EVL-NEXT: [[TMP18:%.*]] = call @llvm.vp.fptoui.nxv4i32.nxv4f32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP14]]) +; IF-EVL-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP15]] +; IF-EVL-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i32 0 +; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0( [[TMP18]], ptr align 4 [[TMP20]], splat (i1 true), i32 [[TMP14]]) +; IF-EVL-NEXT: [[TMP21:%.*]] = zext i32 [[TMP14]] to i64 +; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP21]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]] +; IF-EVL-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; IF-EVL-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]] +; IF-EVL: [[MIDDLE_BLOCK]]: +; IF-EVL-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; IF-EVL: [[SCALAR_PH]]: +; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ] +; IF-EVL-NEXT: br label %[[LOOP:.*]] +; IF-EVL: [[LOOP]]: +; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; IF-EVL-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]] +; IF-EVL-NEXT: [[TMP23:%.*]] = load float, ptr [[GEP]], align 4 +; IF-EVL-NEXT: [[CONV:%.*]] = fptoui float [[TMP23]] to i32 +; IF-EVL-NEXT: [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]] +; IF-EVL-NEXT: store i32 [[CONV]], ptr [[GEP2]], align 4 +; IF-EVL-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; IF-EVL-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; IF-EVL-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP44:![0-9]+]] +; IF-EVL: [[EXIT]]: +; IF-EVL-NEXT: ret void +; +; NO-VP-LABEL: define void @vp_fptoui( +; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; NO-VP-NEXT: [[ENTRY:.*]]: +; NO-VP-NEXT: br label %[[LOOP:.*]] +; NO-VP: [[LOOP]]: +; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]] +; NO-VP-NEXT: [[TMP0:%.*]] = load float, ptr [[GEP]], align 4 +; NO-VP-NEXT: [[CONV:%.*]] = fptoui float [[TMP0]] to i32 +; NO-VP-NEXT: [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]] +; NO-VP-NEXT: store i32 [[CONV]], ptr [[GEP2]], align 4 +; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]] +; NO-VP: [[EXIT]]: +; NO-VP-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds float, ptr %b, i64 %iv + %0 = load float, ptr %gep, align 4 + %conv = fptoui float %0 to i32 + %gep2 = getelementptr inbounds i32, ptr %a, i64 %iv + store i32 %conv, ptr %gep2, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} + +define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) { +; IF-EVL-LABEL: define void @vp_inttoptr( +; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; IF-EVL-NEXT: [[ENTRY:.*]]: +; IF-EVL-NEXT: [[B2:%.*]] = ptrtoint ptr [[B]] to i64 +; IF-EVL-NEXT: [[A1:%.*]] = ptrtoint ptr [[A]] to i64 +; IF-EVL-NEXT: [[TMP0:%.*]] = sub i64 -1, [[N]] +; IF-EVL-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 2 +; IF-EVL-NEXT: [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]]) +; IF-EVL-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]] +; IF-EVL-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; IF-EVL: [[VECTOR_MEMCHECK]]: +; IF-EVL-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 +; IF-EVL-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 8 +; IF-EVL-NEXT: [[TMP8:%.*]] = sub i64 [[A1]], [[B2]] +; IF-EVL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]] +; IF-EVL-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; IF-EVL: [[VECTOR_PH]]: +; IF-EVL-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 +; IF-EVL-NEXT: [[TMP11:%.*]] = sub i64 [[TMP10]], 1 +; IF-EVL-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]] +; IF-EVL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]] +; IF-EVL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; IF-EVL-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; IF-EVL-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 2 +; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]] +; IF-EVL: [[VECTOR_BODY]]: +; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true) +; IF-EVL-NEXT: [[TMP15:%.*]] = add i64 [[EVL_BASED_IV]], 0 +; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP15]] +; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i32 0 +; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP17]], splat (i1 true), i32 [[TMP14]]) +; IF-EVL-NEXT: [[TMP18:%.*]] = call @llvm.vp.inttoptr.nxv2p0.nxv2i64( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP14]]) +; IF-EVL-NEXT: [[TMP19:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[TMP15]] +; IF-EVL-NEXT: [[TMP20:%.*]] = getelementptr inbounds ptr, ptr [[TMP19]], i32 0 +; IF-EVL-NEXT: call void @llvm.vp.store.nxv2p0.p0( [[TMP18]], ptr align 8 [[TMP20]], splat (i1 true), i32 [[TMP14]]) +; IF-EVL-NEXT: [[TMP21:%.*]] = zext i32 [[TMP14]] to i64 +; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP21]], [[EVL_BASED_IV]] +; IF-EVL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]] +; IF-EVL-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; IF-EVL-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]] +; IF-EVL: [[MIDDLE_BLOCK]]: +; IF-EVL-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; IF-EVL: [[SCALAR_PH]]: +; IF-EVL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ] +; IF-EVL-NEXT: br label %[[LOOP:.*]] +; IF-EVL: [[LOOP]]: +; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; IF-EVL-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]] +; IF-EVL-NEXT: [[TMP23:%.*]] = load i64, ptr [[GEP]], align 8 +; IF-EVL-NEXT: [[TMP24:%.*]] = inttoptr i64 [[TMP23]] to ptr +; IF-EVL-NEXT: [[GEP2:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[IV]] +; IF-EVL-NEXT: store ptr [[TMP24]], ptr [[GEP2]], align 8 +; IF-EVL-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; IF-EVL-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; IF-EVL-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP46:![0-9]+]] +; IF-EVL: [[EXIT]]: +; IF-EVL-NEXT: ret void +; +; NO-VP-LABEL: define void @vp_inttoptr( +; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; NO-VP-NEXT: [[ENTRY:.*]]: +; NO-VP-NEXT: br label %[[LOOP:.*]] +; NO-VP: [[LOOP]]: +; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]] +; NO-VP-NEXT: [[TMP0:%.*]] = load i64, ptr [[GEP]], align 8 +; NO-VP-NEXT: [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr +; NO-VP-NEXT: [[GEP2:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[IV]] +; NO-VP-NEXT: store ptr [[TMP1]], ptr [[GEP2]], align 8 +; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]] +; NO-VP: [[EXIT]]: +; NO-VP-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds i64, ptr %b, i64 %iv + %0 = load i64, ptr %gep, align 8 + %1 = inttoptr i64 %0 to ptr + %gep2 = getelementptr inbounds ptr, ptr %a, i64 %iv + store ptr %1, ptr %gep2, align 8 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} + +define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) { +; IF-EVL-LABEL: define void @vp_ptrtoint( +; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; IF-EVL-NEXT: [[ENTRY:.*]]: +; IF-EVL-NEXT: br label %[[LOOP:.*]] +; IF-EVL: [[LOOP]]: +; IF-EVL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; IF-EVL-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] +; IF-EVL-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[GEP]] to i64 +; IF-EVL-NEXT: [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] +; IF-EVL-NEXT: store i64 [[TMP0]], ptr [[GEP2]], align 8 +; IF-EVL-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; IF-EVL-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; IF-EVL-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]] +; IF-EVL: [[EXIT]]: +; IF-EVL-NEXT: ret void +; +; NO-VP-LABEL: define void @vp_ptrtoint( +; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; NO-VP-NEXT: [[ENTRY:.*]]: +; NO-VP-NEXT: br label %[[LOOP:.*]] +; NO-VP: [[LOOP]]: +; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; NO-VP-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] +; NO-VP-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[GEP]] to i64 +; NO-VP-NEXT: [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] +; NO-VP-NEXT: store i64 [[TMP0]], ptr [[GEP2]], align 8 +; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]] +; NO-VP: [[EXIT]]: +; NO-VP-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds i32, ptr %b, i64 %iv + %0 = ptrtoint ptr %gep to i64 + %gep2 = getelementptr inbounds i64, ptr %a, i64 %iv + store i64 %0, ptr %gep2, align 8 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} +;. +; IF-EVL: [[META0]] = !{[[META1:![0-9]+]]} +; IF-EVL: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]} +; IF-EVL: [[META2]] = distinct !{[[META2]], !"LVerDomain"} +; IF-EVL: [[META3]] = !{[[META4:![0-9]+]]} +; IF-EVL: [[META4]] = distinct !{[[META4]], [[META2]]} +; IF-EVL: [[LOOP5]] = distinct !{[[LOOP5]], [[META6:![0-9]+]], [[META7:![0-9]+]]} +; IF-EVL: [[META6]] = !{!"llvm.loop.isvectorized", i32 1} +; IF-EVL: [[META7]] = !{!"llvm.loop.unroll.runtime.disable"} +; IF-EVL: [[LOOP8]] = distinct !{[[LOOP8]], [[META6]]} +; IF-EVL: [[META9]] = !{[[META10:![0-9]+]]} +; IF-EVL: [[META10]] = distinct !{[[META10]], [[META11:![0-9]+]]} +; IF-EVL: [[META11]] = distinct !{[[META11]], !"LVerDomain"} +; IF-EVL: [[META12]] = !{[[META13:![0-9]+]]} +; IF-EVL: [[META13]] = distinct !{[[META13]], [[META11]]} +; IF-EVL: [[LOOP14]] = distinct !{[[LOOP14]], [[META6]], [[META7]]} +; IF-EVL: [[LOOP15]] = distinct !{[[LOOP15]], [[META6]]} +; IF-EVL: [[META16]] = !{[[META17:![0-9]+]]} +; IF-EVL: [[META17]] = distinct !{[[META17]], [[META18:![0-9]+]]} +; IF-EVL: [[META18]] = distinct !{[[META18]], !"LVerDomain"} +; IF-EVL: [[META19]] = !{[[META20:![0-9]+]]} +; IF-EVL: [[META20]] = distinct !{[[META20]], [[META18]]} +; IF-EVL: [[LOOP21]] = distinct !{[[LOOP21]], [[META6]], [[META7]]} +; IF-EVL: [[LOOP22]] = distinct !{[[LOOP22]], [[META6]]} +; IF-EVL: [[META23]] = !{[[META24:![0-9]+]]} +; IF-EVL: [[META24]] = distinct !{[[META24]], [[META25:![0-9]+]]} +; IF-EVL: [[META25]] = distinct !{[[META25]], !"LVerDomain"} +; IF-EVL: [[META26]] = !{[[META27:![0-9]+]]} +; IF-EVL: [[META27]] = distinct !{[[META27]], [[META25]]} +; IF-EVL: [[LOOP28]] = distinct !{[[LOOP28]], [[META6]], [[META7]]} +; IF-EVL: [[LOOP29]] = distinct !{[[LOOP29]], [[META6]]} +; IF-EVL: [[META30]] = !{[[META31:![0-9]+]]} +; IF-EVL: [[META31]] = distinct !{[[META31]], [[META32:![0-9]+]]} +; IF-EVL: [[META32]] = distinct !{[[META32]], !"LVerDomain"} +; IF-EVL: [[META33]] = !{[[META34:![0-9]+]]} +; IF-EVL: [[META34]] = distinct !{[[META34]], [[META32]]} +; IF-EVL: [[LOOP35]] = distinct !{[[LOOP35]], [[META6]], [[META7]]} +; IF-EVL: [[LOOP36]] = distinct !{[[LOOP36]], [[META6]]} +; IF-EVL: [[LOOP37]] = distinct !{[[LOOP37]], [[META6]], [[META7]]} +; IF-EVL: [[LOOP38]] = distinct !{[[LOOP38]], [[META6]]} +; IF-EVL: [[LOOP39]] = distinct !{[[LOOP39]], [[META6]], [[META7]]} +; IF-EVL: [[LOOP40]] = distinct !{[[LOOP40]], [[META6]]} +; IF-EVL: [[LOOP41]] = distinct !{[[LOOP41]], [[META6]], [[META7]]} +; IF-EVL: [[LOOP42]] = distinct !{[[LOOP42]], [[META6]]} +; IF-EVL: [[LOOP43]] = distinct !{[[LOOP43]], [[META6]], [[META7]]} +; IF-EVL: [[LOOP44]] = distinct !{[[LOOP44]], [[META6]]} +; IF-EVL: [[LOOP45]] = distinct !{[[LOOP45]], [[META6]], [[META7]]} +; IF-EVL: [[LOOP46]] = distinct !{[[LOOP46]], [[META6]]} +;. diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll index d64ae380799bf..5c47f07f4436c 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll @@ -332,12 +332,12 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-CAST ir<[[FPEXT:%.+]]> = fpext ir<[[LD1]]> to double -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[LRINT:%.+]]> = call llvm.vp.lrint(ir<[[FPEXT]]>, ir, vp<[[EVL]]>) -; IF-EVL-NEXT: WIDEN-CAST ir<[[TRUNC:%.+]]> = trunc ir<[[LRINT]]> to i32 +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[FPEXT:%.+]]> = call llvm.vp.fpext(ir<[[LD1]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[LRINT:%.+]]> = call llvm.vp.lrint(vp<[[FPEXT]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[TRUNC:%.+]]> = call llvm.vp.trunc(ir<[[LRINT]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[TRUNC]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[TRUNC]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -384,12 +384,12 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-CAST ir<[[FPEXT:%.+]]> = fpext ir<[[LD1]]> to double -; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[LLRINT:%.+]]> = call llvm.vp.llrint(ir<[[FPEXT]]>, ir, vp<[[EVL]]>) -; IF-EVL-NEXT: WIDEN-CAST ir<[[TRUNC:%.+]]> = trunc ir<[[LLRINT]]> to i32 +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[FPEXT:%.+]]> = call llvm.vp.fpext(ir<[[LD1]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[LLRINT:%.+]]> = call llvm.vp.llrint(vp<[[FPEXT]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[TRUNC:%.+]]> = call llvm.vp.trunc(ir<[[LLRINT]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[TRUNC]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[TRUNC]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll index 05e1040df6cd8..c8611c60966e3 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll @@ -102,7 +102,7 @@ exit: ret void } -define void @vp_truncate(ptr %a, ptr %b, i64 %N) { +define void @vp_trunc(ptr %a, ptr %b, i64 %N) { ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count @@ -198,7 +198,7 @@ exit: ret void } -define void @vp_fptrunct(ptr %a, ptr %b, i64 %N) { +define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) { ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count @@ -246,7 +246,7 @@ exit: ret void } -define void @vp_fptosi(ptr %a, ptr %b, i64 %N) { +define void @vp_sitofp(ptr %a, ptr %b, i64 %N) { ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count @@ -265,10 +265,10 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[FPTOSI:%.+]]> = call llvm.vp.fptoui(ir<[[LD1]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SITOFP:%.+]]> = call llvm.vp.sitofp(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[FPTOSI]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[SITOFP]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -281,11 +281,11 @@ entry: loop: %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] - %gep = getelementptr inbounds float, ptr %b, i64 %iv - %0 = load float, ptr %gep, align 4 - %conv2 = fptoui float %0 to i32 - %gep4 = getelementptr inbounds i32, ptr %a, i64 %iv - store i32 %conv2, ptr %gep4, align 4 + %gep = getelementptr inbounds i32, ptr %b, i64 %iv + %0 = load i32, ptr %gep, align 4 + %conv2 = sitofp i32 %0 to float + %gep4 = getelementptr inbounds float, ptr %a, i64 %iv + store float %conv2, ptr %gep4, align 4 %iv.next = add nuw nsw i64 %iv, 1 %exitcond.not = icmp eq i64 %iv.next, %N br i1 %exitcond.not, label %exit, label %loop @@ -294,7 +294,7 @@ exit: ret void } -define void @vp_inttofp(ptr %a, ptr %b, i64 %N) { +define void @vp_uitofp(ptr %a, ptr %b, i64 %N) { ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count @@ -313,10 +313,10 @@ define void @vp_inttofp(ptr %a, ptr %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SITOFP:%.+]]> = call llvm.vp.sitofp(ir<[[LD1]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[UITOFP:%.+]]> = call llvm.vp.uitofp(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[SITOFP]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[UITOFP]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -331,7 +331,7 @@ loop: %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] %gep = getelementptr inbounds i32, ptr %b, i64 %iv %0 = load i32, ptr %gep, align 4 - %conv2 = sitofp i32 %0 to float + %conv2 = uitofp i32 %0 to float %gep4 = getelementptr inbounds float, ptr %a, i64 %iv store float %conv2, ptr %gep4, align 4 %iv.next = add nuw nsw i64 %iv, 1 @@ -342,7 +342,7 @@ exit: ret void } -define void @vp_uinttofp(ptr %a, ptr %b, i64 %N) { +define void @vp_fptosi(ptr %a, ptr %b, i64 %N) { ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count @@ -361,10 +361,10 @@ define void @vp_uinttofp(ptr %a, ptr %b, i64 %N) { ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> -; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[UITOFP:%.+]]> = call llvm.vp.uitofp(ir<[[LD1]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[FPTOSI:%.+]]> = call llvm.vp.fptosi(ir<[[LD1]]>, ir, vp<[[EVL]]>) ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> -; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[UITOFP]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[FPTOSI]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> @@ -377,11 +377,59 @@ entry: loop: %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] - %gep = getelementptr inbounds i32, ptr %b, i64 %iv - %0 = load i32, ptr %gep, align 4 - %conv2 = uitofp i32 %0 to float - %gep4 = getelementptr inbounds float, ptr %a, i64 %iv - store float %conv2, ptr %gep4, align 4 + %gep = getelementptr inbounds float, ptr %b, i64 %iv + %0 = load float, ptr %gep, align 4 + %conv2 = fptosi float %0 to i32 + %gep4 = getelementptr inbounds i32, ptr %a, i64 %iv + store i32 %conv2, ptr %gep4, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} + +define void @vp_fptoui(ptr %a, ptr %b, i64 %N) { +; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { +; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF +; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count +; IF-EVL-NEXT: Live-in ir<%N> = original trip-count + +; IF-EVL: vector.ph: +; IF-EVL-NEXT: Successor(s): vector loop + +; IF-EVL: vector loop: { +; IF-EVL-NEXT: vector.body: +; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION +; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]> +; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]> +; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1> +; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> +; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[FPTOUI:%.+]]> = call llvm.vp.fptoui(ir<[[LD1]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[FPTOUI]]>, vp<[[EVL]]> +; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> +; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]> +; IF-EVL-NEXT: No successors +; IF-EVL-NEXT: } + +entry: + br label %loop + +loop: + %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] + %gep = getelementptr inbounds float, ptr %b, i64 %iv + %0 = load float, ptr %gep, align 4 + %conv2 = fptoui float %0 to i32 + %gep4 = getelementptr inbounds i32, ptr %a, i64 %iv + store i32 %conv2, ptr %gep4, align 4 %iv.next = add nuw nsw i64 %iv, 1 %exitcond.not = icmp eq i64 %iv.next, %N br i1 %exitcond.not, label %exit, label %loop @@ -389,3 +437,71 @@ loop: exit: ret void } + +define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) { +; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2},UF={1}' { +; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF +; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count +; IF-EVL-NEXT: Live-in ir<%N> = original trip-count + +; IF-EVL: vector.ph: +; IF-EVL-NEXT: Successor(s): vector loop + +; IF-EVL: vector loop: { +; IF-EVL-NEXT: vector.body: +; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION +; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]> +; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]> +; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1> +; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> +; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> +; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[INTTOPTR:%.+]]> = call llvm.vp.inttoptr(ir<[[LD1]]>, ir, vp<[[EVL]]>) +; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> +; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> +; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, vp<[[INTTOPTR]]>, vp<[[EVL]]> +; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> +; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> +; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]> +; IF-EVL-NEXT: No successors +; IF-EVL-NEXT: } + +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds i64, ptr %b, i64 %iv + %0 = load i64, ptr %gep, align 8 + %1 = inttoptr i64 %0 to ptr + %gep2 = getelementptr inbounds ptr, ptr %a, i64 %iv + store ptr %1, ptr %gep2, align 8 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} + +define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) { +; IF-EVL: LV: Vectorization is possible but not beneficial. + +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds i32, ptr %b, i64 %iv + %0 = ptrtoint ptr %gep to i64 + %gep2 = getelementptr inbounds i64, ptr %a, i64 %iv + store i64 %0, ptr %gep2, align 8 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} \ No newline at end of file From 6b8fb807603d6a72ee4da943a29e10003f7cf492 Mon Sep 17 00:00:00 2001 From: "Liqin.Weng" Date: Tue, 3 Dec 2024 11:53:36 +0800 Subject: [PATCH 09/11] fix the comments --- llvm/lib/Analysis/VectorUtils.cpp | 4 +++ .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 26 ++++++------------- .../Transforms/Vectorize/VPlanTransforms.cpp | 20 ++++++++------ .../LoopVectorize/RISCV/inloop-reduction.ll | 2 +- 4 files changed, 25 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp index 989090b80e1c8..6d03071c5026f 100644 --- a/llvm/lib/Analysis/VectorUtils.cpp +++ b/llvm/lib/Analysis/VectorUtils.cpp @@ -144,6 +144,10 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg( if (TTI && Intrinsic::isTargetIntrinsic(ID)) return TTI->isVectorIntrinsicWithOverloadTypeAtArg(ID, OpdIdx); + if (VPCastIntrinsic::isVPCast(ID)) { + return OpdIdx == -1 || OpdIdx == 0; + } + switch (ID) { case Intrinsic::fptosi_sat: case Intrinsic::fptoui_sat: diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index aabf6243f236b..f729c93c47624 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -962,31 +962,21 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) { // Use vector version of the intrinsic. Module *M = State.Builder.GetInsertBlock()->getModule(); - bool IsVPIntrinsic = VPIntrinsic::isVPIntrinsic(VectorIntrinsicID); Function *VectorF = - IsVPIntrinsic - ? VPIntrinsic::getOrInsertDeclarationForParams(M, VectorIntrinsicID, - TysForDecl[0], Args) - : Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl); + Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl); assert(VectorF && "Can't retrieve vector intrinsic or vector-predication intrinsics."); + auto *CI = cast_or_null(getUnderlyingValue()); SmallVector OpBundles; - if (!IsVPIntrinsic) { - if (auto *CI = cast_or_null(getUnderlyingValue())) - CI->getOperandBundlesAsDefs(OpBundles); - } + if (CI) + CI->getOperandBundlesAsDefs(OpBundles); - Instruction *V = State.Builder.CreateCall(VectorF, Args, OpBundles); + CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles); - if (IsVPIntrinsic) { - // Currently vp-intrinsics only accept FMF flags. llvm.vp.uitofp will get - // Flags of OperationType::NonNegOp && OperationType::FPMathOp. - if (isa(V) && VectorIntrinsicID != Intrinsic::vp_uitofp) - setFlags(V); - } else { - setFlags(V); - } + // FIXME: vp.cast and vp.select dont pass the underlying instruction into the + // recipe, which set the flags and metadata be needed. + setFlags(V); if (!V->getType()->isVoidTy()) State.set(this, V); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 1092e10104dad..604e0fa18779e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1445,8 +1445,7 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { VPTypeAnalysis TypeInfo(CanonicalIVType); LLVMContext &Ctx = CanonicalIVType->getContext(); SmallVector HeaderMasks = collectAllHeaderMasks(Plan); - VPValue *AllOneMask = - Plan.getOrAddLiveIn(ConstantInt::getTrue(IntegerType::getInt1Ty(Ctx))); + for (VPUser *U : Plan.getVF().users()) { if (auto *R = dyn_cast(U)) R->setOperand(1, &EVL); @@ -1486,14 +1485,17 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { auto *CI = cast(CInst->getUnderlyingInstr()); Intrinsic::ID VPID = VPIntrinsic::getForIntrinsic( CI->getCalledFunction()->getIntrinsicID()); - if (VPID == Intrinsic::not_intrinsic) - return nullptr; + assert(VPID != Intrinsic::not_intrinsic && + "Expected VP Instrinsic"); SmallVector Ops(CInst->operands()); assert(VPIntrinsic::getMaskParamPos(VPID) && VPIntrinsic::getVectorLengthParamPos(VPID) && "Expected VP intrinsic"); - Ops.push_back(AllOneMask); + VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue( + IntegerType::getInt1Ty(CI->getContext()))); + Ops.push_back(Mask); + ; Ops.push_back(&EVL); return new VPWidenIntrinsicRecipe( *CI, VPID, Ops, TypeInfo.inferScalarType(CInst), @@ -1504,14 +1506,16 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { auto *CI = dyn_cast(CInst->getUnderlyingInstr()); Intrinsic::ID VPID = VPIntrinsic::getForOpcode(CI->getOpcode()); - if (VPID == Intrinsic::not_intrinsic) - return nullptr; + assert(VPID != Intrinsic::not_intrinsic && + "Expected vp.casts Instrinsic"); SmallVector Ops(CInst->operands()); assert(VPIntrinsic::getMaskParamPos(VPID) && VPIntrinsic::getVectorLengthParamPos(VPID) && "Expected VP intrinsic"); - Ops.push_back(AllOneMask); + VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue( + IntegerType::getInt1Ty(CI->getContext()))); + Ops.push_back(Mask); Ops.push_back(&EVL); return new VPWidenIntrinsicRecipe( VPID, Ops, TypeInfo.inferScalarType(CInst), diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll index 80c58afb746f3..14818199072c2 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll @@ -203,7 +203,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) { ; IF-EVL-INLOOP-NEXT: [[TMP14:%.*]] = call @llvm.vp.sext.nxv8i32.nxv8i16( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP6]]) ; IF-EVL-INLOOP-NEXT: [[TMP10:%.*]] = call i32 @llvm.vp.reduce.add.nxv8i32(i32 0, [[TMP14]], splat (i1 true), i32 [[TMP6]]) ; IF-EVL-INLOOP-NEXT: [[TMP11]] = add i32 [[TMP10]], [[VEC_PHI]] -; IF-EVL-INLOOP-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP5]], [[EVL_BASED_IV]] +; IF-EVL-INLOOP-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP6]], [[EVL_BASED_IV]] ; IF-EVL-INLOOP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP4]] ; IF-EVL-INLOOP-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; IF-EVL-INLOOP-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] From 1e54505497e76e25beb7a3e878c3a7e9d5e91eb5 Mon Sep 17 00:00:00 2001 From: LiqinWeng Date: Tue, 10 Dec 2024 11:30:07 +0800 Subject: [PATCH 10/11] address the comments --- llvm/lib/Analysis/VectorUtils.cpp | 3 +-- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp index 6d03071c5026f..e10dd9c29a4a2 100644 --- a/llvm/lib/Analysis/VectorUtils.cpp +++ b/llvm/lib/Analysis/VectorUtils.cpp @@ -144,9 +144,8 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg( if (TTI && Intrinsic::isTargetIntrinsic(ID)) return TTI->isVectorIntrinsicWithOverloadTypeAtArg(ID, OpdIdx); - if (VPCastIntrinsic::isVPCast(ID)) { + if (VPCastIntrinsic::isVPCast(ID)) return OpdIdx == -1 || OpdIdx == 0; - } switch (ID) { case Intrinsic::fptosi_sat: diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index f729c93c47624..57bdd7659f7a5 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -974,8 +974,6 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) { CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles); - // FIXME: vp.cast and vp.select dont pass the underlying instruction into the - // recipe, which set the flags and metadata be needed. setFlags(V); if (!V->getType()->isVoidTy()) From b7bc8a81f564b90c62ba59a6ceafe93dea16fe3b Mon Sep 17 00:00:00 2001 From: LiqinWeng Date: Tue, 10 Dec 2024 18:23:29 +0800 Subject: [PATCH 11/11] address the comments --- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 57bdd7659f7a5..fa978e24b21e3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -978,7 +978,7 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) { if (!V->getType()->isVoidTy()) State.set(this, V); - State.addMetadata(V, dyn_cast_or_null(getUnderlyingValue())); + State.addMetadata(V, CI); } InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF, diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 604e0fa18779e..21e43214a6332 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1485,8 +1485,8 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { auto *CI = cast(CInst->getUnderlyingInstr()); Intrinsic::ID VPID = VPIntrinsic::getForIntrinsic( CI->getCalledFunction()->getIntrinsicID()); - assert(VPID != Intrinsic::not_intrinsic && - "Expected VP Instrinsic"); + if (VPID == Intrinsic::not_intrinsic) + return nullptr; SmallVector Ops(CInst->operands()); assert(VPIntrinsic::getMaskParamPos(VPID) && @@ -1495,7 +1495,6 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue( IntegerType::getInt1Ty(CI->getContext()))); Ops.push_back(Mask); - ; Ops.push_back(&EVL); return new VPWidenIntrinsicRecipe( *CI, VPID, Ops, TypeInfo.inferScalarType(CInst),