diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 34b2abf449ece..c4a6994276274 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2242,48 +2242,50 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { } } - // Try to optimize header mask recipes away to their EVL variants. + // Replace header masks with a mask equivalent to predicating by EVL: + // + // icmp ule widen-canonical-iv backedge-taken-count + // -> + // icmp ult step-vector, EVL + VPRecipeBase *EVLR = EVL.getDefiningRecipe(); + VPBuilder Builder(EVLR->getParent(), std::next(EVLR->getIterator())); + Type *EVLType = TypeInfo.inferScalarType(&EVL); + VPValue *EVLMask = Builder.createICmp( + CmpInst::ICMP_ULT, + Builder.createNaryOp(VPInstruction::StepVector, {}, EVLType), &EVL); for (VPValue *HeaderMask : collectAllHeaderMasks(Plan)) { - // TODO: Split optimizeMaskToEVL out and move into - // VPlanTransforms::optimize. transformRecipestoEVLRecipes should be run in - // tryToBuildVPlanWithVPRecipes beforehand. - for (VPUser *U : collectUsersRecursively(HeaderMask)) { - auto *CurRecipe = cast(U); - VPRecipeBase *EVLRecipe = - optimizeMaskToEVL(HeaderMask, *CurRecipe, TypeInfo, *AllOneMask, EVL); - if (!EVLRecipe) - continue; - - [[maybe_unused]] unsigned NumDefVal = EVLRecipe->getNumDefinedValues(); - assert(NumDefVal == CurRecipe->getNumDefinedValues() && - "New recipe must define the same number of values as the " - "original."); - assert( - NumDefVal <= 1 && - "Only supports recipes with a single definition or without users."); - EVLRecipe->insertBefore(CurRecipe); - if (isa(EVLRecipe)) { - VPValue *CurVPV = CurRecipe->getVPSingleValue(); - CurVPV->replaceAllUsesWith(EVLRecipe->getVPSingleValue()); - } - ToErase.push_back(CurRecipe); - } - - // Replace header masks with a mask equivalent to predicating by EVL: - // - // icmp ule widen-canonical-iv backedge-taken-count - // -> - // icmp ult step-vector, EVL - VPRecipeBase *EVLR = EVL.getDefiningRecipe(); - VPBuilder Builder(EVLR->getParent(), std::next(EVLR->getIterator())); - Type *EVLType = TypeInfo.inferScalarType(&EVL); - VPValue *EVLMask = Builder.createICmp( - CmpInst::ICMP_ULT, - Builder.createNaryOp(VPInstruction::StepVector, {}, EVLType), &EVL); HeaderMask->replaceAllUsesWith(EVLMask); ToErase.push_back(HeaderMask->getDefiningRecipe()); } + // Try to optimize header mask recipes away to their EVL variants. + // TODO: Split optimizeMaskToEVL out and move into + // VPlanTransforms::optimize. transformRecipestoEVLRecipes should be run in + // tryToBuildVPlanWithVPRecipes beforehand. + for (VPUser *U : collectUsersRecursively(EVLMask)) { + auto *CurRecipe = cast(U); + VPRecipeBase *EVLRecipe = + optimizeMaskToEVL(EVLMask, *CurRecipe, TypeInfo, *AllOneMask, EVL); + if (!EVLRecipe) + continue; + + [[maybe_unused]] unsigned NumDefVal = EVLRecipe->getNumDefinedValues(); + assert(NumDefVal == CurRecipe->getNumDefinedValues() && + "New recipe must define the same number of values as the " + "original."); + assert(NumDefVal <= 1 && + "Only supports recipes with a single definition or without users."); + EVLRecipe->insertBefore(CurRecipe); + if (isa(EVLRecipe)) { + VPValue *CurVPV = CurRecipe->getVPSingleValue(); + CurVPV->replaceAllUsesWith(EVLRecipe->getVPSingleValue()); + } + ToErase.push_back(CurRecipe); + } + // Remove dead EVL mask. + if (EVLMask->getNumUsers() == 0) + EVLMask->getDefiningRecipe()->eraseFromParent(); + for (VPRecipeBase *R : reverse(ToErase)) { SmallVector PossiblyDead(R->operands()); R->eraseFromParent();