diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 5a7b18f51fcdc..2e5be862ac450 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2883,9 +2883,9 @@ void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) { PHINode *NewPhi = cast(State.get(VPPhi)); // Make sure the builder has a valid insert point. Builder.SetInsertPoint(NewPhi); - for (unsigned Idx = 0; Idx < VPPhi->getNumOperands(); ++Idx) { + for (unsigned Idx = 0; Idx < VPPhi->getNumIncoming(); ++Idx) { VPValue *Inc = VPPhi->getIncomingValue(Idx); - VPBasicBlock *VPBB = VPPhi->getIncomingBlock(Idx); + const VPBasicBlock *VPBB = VPPhi->getIncomingBlock(Idx); NewPhi->addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]); } } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 701e4bfe0623e..ff7fabaaa4d7a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -790,6 +790,25 @@ InstructionCost VPBasicBlock::cost(ElementCount VF, VPCostContext &Ctx) { return Cost; } +const VPBasicBlock *VPBasicBlock::getCFGPredecessor(unsigned Idx) const { + const VPBlockBase *Pred = nullptr; + if (getNumPredecessors() > 0) { + Pred = getPredecessors()[Idx]; + } else { + auto *Region = getParent(); + assert(Region && !Region->isReplicator() && Region->getEntry() == this && + "must be in the entry block of a non-replicate region"); + assert(Idx < 2 && Region->getNumPredecessors() == 1 && + "loop region has a single predecessor (preheader), its entry block " + "has 2 incoming blocks"); + + // Idx == 0 selects the predecessor of the region, Idx == 1 selects the + // region itself whose exiting block feeds the phi across the backedge. + Pred = Idx == 0 ? Region->getSinglePredecessor() : Region; + } + return Pred->getExitingBasicBlock(); +} + InstructionCost VPRegionBlock::cost(ElementCount VF, VPCostContext &Ctx) { if (!isReplicator()) { InstructionCost Cost = 0; diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 239ee220e2443..4149d9c7e7fe6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1166,6 +1166,28 @@ class VPIRInstruction : public VPRecipeBase { void extractLastLaneOfFirstOperand(VPBuilder &Builder); }; +/// Helper type to provide functions to access incoming values and blocks for +/// phi-like recipes. +class VPPhiAccessors { +protected: + /// Return a VPRecipeBase* to the current object. + virtual const VPRecipeBase *getAsRecipe() const = 0; + +public: + virtual ~VPPhiAccessors() = default; + + /// Returns the incoming VPValue with index \p Idx. + VPValue *getIncomingValue(unsigned Idx) const { + return getAsRecipe()->getOperand(Idx); + } + + /// Returns the incoming block with index \p Idx. + const VPBasicBlock *getIncomingBlock(unsigned Idx) const; + + /// Returns the number of incoming values, also number of incoming blocks. + unsigned getNumIncoming() const { return getAsRecipe()->getNumOperands(); } +}; + /// An overlay for VPIRInstructions wrapping PHI nodes enabling convenient use /// cast/dyn_cast/isa and execute() implementation. A single VPValue operand is /// allowed, and it is used to add a new incoming value for the single @@ -1969,10 +1991,13 @@ class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe, /// recipe is placed in an entry block to a (non-replicate) region, it must have /// exactly 2 incoming values, the first from the predecessor of the region and /// the second from the exiting block of the region. -class VPWidenPHIRecipe : public VPSingleDefRecipe { +class VPWidenPHIRecipe : public VPSingleDefRecipe, public VPPhiAccessors { /// Name to use for the generated IR instruction for the widened phi. std::string Name; +protected: + const VPRecipeBase *getAsRecipe() const override { return this; } + public: /// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start and /// debug location \p DL. @@ -2000,12 +2025,6 @@ class VPWidenPHIRecipe : public VPSingleDefRecipe { void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override; #endif - - /// Returns the \p I th incoming VPBasicBlock. - VPBasicBlock *getIncomingBlock(unsigned I); - - /// Returns the \p I th incoming VPValue. - VPValue *getIncomingValue(unsigned I) { return getOperand(I); } }; /// A recipe for handling first-order recurrence phis. The start value is the @@ -3324,6 +3343,12 @@ class VPBasicBlock : public VPBlockBase { /// the cloned recipes. VPBasicBlock *clone() override; + /// Returns the predecessor block at index \p Idx with the predecessors as per + /// the corresponding plain CFG. If the block is an entry block to a region, + /// the first predecessor is the single predecessor of a region, and the + /// second predecessor is the exiting block of the region. + const VPBasicBlock *getCFGPredecessor(unsigned Idx) const; + protected: /// Execute the recipes in the IR basic block \p BB. void executeRecipes(VPTransformState *State, BasicBlock *BB); @@ -3338,6 +3363,11 @@ class VPBasicBlock : public VPBlockBase { BasicBlock *createEmptyBasicBlock(VPTransformState &State); }; +inline const VPBasicBlock * +VPPhiAccessors::getIncomingBlock(unsigned Idx) const { + return getAsRecipe()->getParent()->getCFGPredecessor(Idx); +} + /// A special type of VPBasicBlock that wraps an existing IR basic block. /// Recipes of the block get added before the first non-phi instruction in the /// wrapped block. diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 500622f57f89f..265aa7a3ae4df 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -3686,27 +3686,6 @@ void VPReductionPHIRecipe::print(raw_ostream &O, const Twine &Indent, } #endif -VPBasicBlock *VPWidenPHIRecipe::getIncomingBlock(unsigned I) { - VPBasicBlock *Parent = getParent(); - VPBlockBase *Pred = nullptr; - if (Parent->getNumPredecessors() > 0) { - Pred = Parent->getPredecessors()[I]; - } else { - auto *Region = Parent->getParent(); - assert(Region && !Region->isReplicator() && Region->getEntry() == Parent && - "must be in the entry block of a non-replicate region"); - assert( - I < 2 && getNumOperands() == 2 && - "when placed in an entry block, only 2 incoming blocks are available"); - - // I == 0 selects the predecessor of the region, I == 1 selects the region - // itself whose exiting block feeds the phi across the backedge. - Pred = I == 0 ? Region->getSinglePredecessor() : Region; - } - - return Pred->getExitingBasicBlock(); -} - void VPWidenPHIRecipe::execute(VPTransformState &State) { assert(EnableVPlanNativePath && "Non-native vplans are not expected to have VPWidenPHIRecipes.");