diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index db40ce2d20b81..99fd97eb71cad 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2335,8 +2335,9 @@ class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPSingleDefRecipe { return Idx == 0 ? getOperand(1) : getOperand(Idx * 2 + !isNormalized()); } - /// Generate the phi/select nodes. - void execute(VPTransformState &State) override; + void execute(VPTransformState &State) override { + llvm_unreachable("VPBlendRecipe should be expanded by simplifyBlends"); + } /// Return the cost of this VPWidenMemoryRecipe. InstructionCost computeCost(ElementCount VF, diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 1fbc3f3e725ee..241ac42b685a9 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -2411,42 +2411,6 @@ void VPVectorPointerRecipe::print(raw_ostream &O, const Twine &Indent, } #endif -void VPBlendRecipe::execute(VPTransformState &State) { - assert(isNormalized() && "Expected blend to be normalized!"); - // We know that all PHIs in non-header blocks are converted into - // selects, so we don't have to worry about the insertion order and we - // can just use the builder. - // At this point we generate the predication tree. There may be - // duplications since this is a simple recursive scan, but future - // optimizations will clean it up. - - unsigned NumIncoming = getNumIncomingValues(); - - // Generate a sequence of selects of the form: - // SELECT(Mask3, In3, - // SELECT(Mask2, In2, - // SELECT(Mask1, In1, - // In0))) - // Note that Mask0 is never used: lanes for which no path reaches this phi and - // are essentially undef are taken from In0. - bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this); - Value *Result = nullptr; - for (unsigned In = 0; In < NumIncoming; ++In) { - // We might have single edge PHIs (blocks) - use an identity - // 'select' for the first PHI operand. - Value *In0 = State.get(getIncomingValue(In), OnlyFirstLaneUsed); - if (In == 0) - Result = In0; // Initialize with the first incoming value. - else { - // Select between the current value and the previous incoming edge - // based on the incoming mask. - Value *Cond = State.get(getMask(In), OnlyFirstLaneUsed); - Result = State.Builder.CreateSelect(Cond, In0, Result, "predphi"); - } - } - State.set(this, Result, OnlyFirstLaneUsed); -} - InstructionCost VPBlendRecipe::computeCost(ElementCount VF, VPCostContext &Ctx) const { // Handle cases where only the first lane is used the same way as the legacy diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index cb370fe0e5b65..5da43b61c672e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2711,6 +2711,18 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan, continue; } + // Expand VPBlendRecipe into VPInstruction::Select. + VPBuilder Builder(&R); + if (auto *Blend = dyn_cast(&R)) { + VPValue *Select = Blend->getIncomingValue(0); + for (unsigned I = 1; I != Blend->getNumIncomingValues(); ++I) + Select = Builder.createSelect(Blend->getMask(I), + Blend->getIncomingValue(I), Select, + R.getDebugLoc(), "predphi"); + Blend->replaceAllUsesWith(Select); + ToRemove.push_back(Blend); + } + if (auto *Expr = dyn_cast(&R)) { Expr->decompose(); ToRemove.push_back(Expr); @@ -2724,7 +2736,6 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan, // Expand WideIVStep. auto *VPI = cast(&R); - VPBuilder Builder(VPI); Type *IVTy = TypeInfo.inferScalarType(VPI); if (TypeInfo.inferScalarType(VectorStep) != IVTy) { Instruction::CastOps CastOp = IVTy->isFloatingPointTy()