From b357af496024fe12bc56a39b89dbff036b65b952 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 27 Feb 2025 21:49:14 +0000 Subject: [PATCH 01/11] [VPlan] Add VPPhiAccessors to provide interface for phi recipes (NFC) Add a VPPhiAccessors class to provide interfaces to access incoming values and blocks, with corresponding iterators. The first user is VPWidenPhiRecipe, with the other phi-like recipes following soon. This will also be used to verify def-use chains where users are phi-like recipes, simplifying https://github.com/llvm/llvm-project/pull/124838. --- .../Transforms/Vectorize/LoopVectorize.cpp | 6 +- llvm/lib/Transforms/Vectorize/VPlan.h | 65 +++++++++++++++++-- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 46 +++++++------ 3 files changed, 87 insertions(+), 30 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index e2612698b6b0f..98ff1a4cdc449 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3069,11 +3069,9 @@ void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) { PHINode *NewPhi = cast(State.get(VPPhi)); // Make sure the builder has a valid insert point. Builder.SetInsertPoint(NewPhi); - for (unsigned Idx = 0; Idx < VPPhi->getNumOperands(); ++Idx) { - VPValue *Inc = VPPhi->getIncomingValue(Idx); - VPBasicBlock *VPBB = VPPhi->getIncomingBlock(Idx); + + for (const auto &[Inc, VPBB] : VPPhi->incoming_values_and_blocks()) NewPhi->addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]); - } } } } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 70e684826ed2d..4d5f2014a40dc 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1085,6 +1085,62 @@ class VPIRInstruction : public VPRecipeBase { void extractLastLaneOfOperand(VPBuilder &Builder); }; +/// Helper type to provide functions to access incoming values and blocks for +/// phi-like recipes. RecipeTy must be a sub-class of VPRecipeBase. +template class VPPhiAccessors { + /// Return a VPRecipeBase* to the current object. + const VPRecipeBase *getAsRecipe() const { + return static_cast(this); + } + +public: + /// Returns the \p I th incoming VPValue. + VPValue *getIncomingValue(unsigned I) const { + return getAsRecipe()->getOperand(I); + } + + /// Returns an interator range over the incoming values + VPUser::const_operand_range incoming_values() const { + return getAsRecipe()->operands(); + } + + /// Returns the \p I th incoming block. + const VPBasicBlock *getIncomingBlock(unsigned Idx) const; + + using const_incoming_block_iterator = + mapped_iterator>; + using const_incoming_blocks_range = + iterator_range; + + const_incoming_block_iterator incoming_block_begin() const { + return const_incoming_block_iterator( + detail::index_iterator(0), + [this](size_t Idx) { return getIncomingBlock(Idx); }); + } + const_incoming_block_iterator incoming_block_end() const { + return const_incoming_block_iterator( + detail::index_iterator(getAsRecipe()->getVPDefID() == + VPDef::VPWidenIntOrFpInductionSC + ? 2 + : getAsRecipe()->getNumOperands()), + [this](size_t Idx) { return getIncomingBlock(Idx); }); + } + + /// Returns an iterator range over the incoming blocks. + const_incoming_blocks_range incoming_blocks() const { + return make_range(incoming_block_begin(), incoming_block_end()); + } + + /// Returns an iterator range over pairs of incoming values and corrsponding + /// incoming blocks. + detail::zippy + incoming_values_and_blocks() const { + return zip(incoming_values(), incoming_blocks()); + } +}; + /// VPWidenRecipe is a recipe for producing a widened instruction using the /// opcode and operands of the recipe. This recipe covers most of the /// traditional vectorization cases where each recipe transforms into a @@ -1944,7 +2000,8 @@ class VPScalarPHIRecipe : public VPHeaderPHIRecipe { /// recipe is placed in an entry block to a (non-replicate) region, it must have /// exactly 2 incoming values, the first from the predecessor of the region and /// the second from the exiting block of the region. -class VPWidenPHIRecipe : public VPSingleDefRecipe { +class VPWidenPHIRecipe : public VPSingleDefRecipe, + public VPPhiAccessors { public: /// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start and /// debug location \p DL. @@ -1970,12 +2027,6 @@ class VPWidenPHIRecipe : public VPSingleDefRecipe { void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override; #endif - - /// Returns the \p I th incoming VPBasicBlock. - VPBasicBlock *getIncomingBlock(unsigned I); - - /// Returns the \p I th incoming VPValue. - VPValue *getIncomingValue(unsigned I) { return getOperand(I); } }; /// A recipe for handling first-order recurrence phis. The start value is the diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index e9f50e88867b2..efac05785203c 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1031,6 +1031,29 @@ void VPIRInstruction::print(raw_ostream &O, const Twine &Indent, } #endif +static const VPBasicBlock *getIncomingBlockForRecipe(const VPRecipeBase *R, + unsigned I) { + const VPBasicBlock *Parent = R->getParent(); + const VPBlockBase *Pred = nullptr; + if (Parent->getNumPredecessors() > 0) { + Pred = Parent->getPredecessors()[I]; + } else { + auto *Region = Parent->getParent(); + assert(Region && !Region->isReplicator() && Region->getEntry() == Parent && + "must be in the entry block of a non-replicate region"); + assert( + I < 2 && + (R->getNumOperands() == 2 || isa(R)) && + "when placed in an entry block, only 2 incoming blocks are available"); + + // I == 0 selects the predecessor of the region, I == 1 selects the region + // itself whose exiting block feeds the phi across the backedge. + Pred = I == 0 ? Region->getSinglePredecessor() : Region; + } + + return Pred->getExitingBasicBlock(); +} + void VPWidenCallRecipe::execute(VPTransformState &State) { assert(State.VF.isVector() && "not widening"); State.setDebugLocFrom(getDebugLoc()); @@ -3580,25 +3603,10 @@ void VPReductionPHIRecipe::print(raw_ostream &O, const Twine &Indent, } #endif -VPBasicBlock *VPWidenPHIRecipe::getIncomingBlock(unsigned I) { - VPBasicBlock *Parent = getParent(); - VPBlockBase *Pred = nullptr; - if (Parent->getNumPredecessors() > 0) { - Pred = Parent->getPredecessors()[I]; - } else { - auto *Region = Parent->getParent(); - assert(Region && !Region->isReplicator() && Region->getEntry() == Parent && - "must be in the entry block of a non-replicate region"); - assert( - I < 2 && getNumOperands() == 2 && - "when placed in an entry block, only 2 incoming blocks are available"); - - // I == 0 selects the predecessor of the region, I == 1 selects the region - // itself whose exiting block feeds the phi across the backedge. - Pred = I == 0 ? Region->getSinglePredecessor() : Region; - } - - return Pred->getExitingBasicBlock(); +template <> +const VPBasicBlock * +VPPhiAccessors::getIncomingBlock(unsigned Idx) const { + return getIncomingBlockForRecipe(getAsRecipe(), Idx); } void VPWidenPHIRecipe::execute(VPTransformState &State) { From c61233f4096202789ebeaa11061d70bb53cdd2af Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 13 Mar 2025 16:25:03 +0000 Subject: [PATCH 02/11] !fixup address latest comments, thanks --- llvm/lib/Transforms/Vectorize/VPlan.h | 15 ++++++--------- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 17 ++++++++++------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 1915b90675193..63d946a0dede9 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1095,9 +1095,9 @@ template class VPPhiAccessors { } public: - /// Returns the \p I th incoming VPValue. - VPValue *getIncomingValue(unsigned I) const { - return getAsRecipe()->getOperand(I); + /// Returns the incoming VPValue with index \p Idx. + VPValue *getIncomingValue(unsigned Idx) const { + return getAsRecipe()->getOperand(Idx); } /// Returns an interator range over the incoming values @@ -1105,7 +1105,7 @@ template class VPPhiAccessors { return getAsRecipe()->operands(); } - /// Returns the \p I th incoming block. + /// Returns the incoming block with index \p Idx. const VPBasicBlock *getIncomingBlock(unsigned Idx) const; using const_incoming_block_iterator = @@ -1121,10 +1121,7 @@ template class VPPhiAccessors { } const_incoming_block_iterator incoming_block_end() const { return const_incoming_block_iterator( - detail::index_iterator(getAsRecipe()->getVPDefID() == - VPDef::VPWidenIntOrFpInductionSC - ? 2 - : getAsRecipe()->getNumOperands()), + detail::index_iterator(getAsRecipe()->getNumOperands()), [this](size_t Idx) { return getIncomingBlock(Idx); }); } @@ -1133,7 +1130,7 @@ template class VPPhiAccessors { return make_range(incoming_block_begin(), incoming_block_end()); } - /// Returns an iterator range over pairs of incoming values and corrsponding + /// Returns an iterator range over pairs of incoming values and correspondingx /// incoming blocks. detail::zippy diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 8f2142b6c3260..73f0c329e5b66 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1042,24 +1042,27 @@ void VPIRInstruction::print(raw_ostream &O, const Twine &Indent, } #endif +/// Returns the incoming block at index \p Idx for \p R. This handles both +/// recipes placed in entry blocks of loop regions (incoming blocks are the +/// region's predecessor and the region's exit) and other locations (incoming +/// blocks are the direct predecessors). static const VPBasicBlock *getIncomingBlockForRecipe(const VPRecipeBase *R, - unsigned I) { + unsigned Idx) { const VPBasicBlock *Parent = R->getParent(); const VPBlockBase *Pred = nullptr; if (Parent->getNumPredecessors() > 0) { - Pred = Parent->getPredecessors()[I]; + Pred = Parent->getPredecessors()[Idx]; } else { auto *Region = Parent->getParent(); assert(Region && !Region->isReplicator() && Region->getEntry() == Parent && "must be in the entry block of a non-replicate region"); assert( - I < 2 && - (R->getNumOperands() == 2 || isa(R)) && + Idx < 2 && R->getNumOperands() == 2 && "when placed in an entry block, only 2 incoming blocks are available"); - // I == 0 selects the predecessor of the region, I == 1 selects the region - // itself whose exiting block feeds the phi across the backedge. - Pred = I == 0 ? Region->getSinglePredecessor() : Region; + // Idx == 0 selects the predecessor of the region, Idx == 1 selects the + // region itself whose exiting block feeds the phi across the backedge. + Pred = Idx == 0 ? Region->getSinglePredecessor() : Region; } return Pred->getExitingBasicBlock(); From f768b97a4b0945c07826f5fef66adbbb85c65f6a Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 26 Apr 2025 21:45:06 +0100 Subject: [PATCH 03/11] !fixup address latest comments, thanks --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 1 - llvm/lib/Transforms/Vectorize/VPlan.h | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 6ade01cf7f7e5..e58ba815a4379 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2890,7 +2890,6 @@ void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) { PHINode *NewPhi = cast(State.get(VPPhi)); // Make sure the builder has a valid insert point. Builder.SetInsertPoint(NewPhi); - for (const auto &[Inc, VPBB] : VPPhi->incoming_values_and_blocks()) NewPhi->addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]); } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index c986b5f845f4e..ca7df407c3c3b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1215,7 +1215,7 @@ template class VPPhiAccessors { return make_range(incoming_block_begin(), incoming_block_end()); } - /// Returns an iterator range over pairs of incoming values and correspondingx + /// Returns an iterator range over pairs of incoming values and corresponding /// incoming blocks. detail::zippy @@ -2028,6 +2028,7 @@ class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe, /// the second from the exiting block of the region. class VPWidenPHIRecipe : public VPSingleDefRecipe, public VPPhiAccessors { + /// Name to use for the generated IR instruction for the widened phi. std::string Name; public: From 595b05720f812c9e3f5df43f83e923059d0ea432 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 1 May 2025 22:03:25 +0100 Subject: [PATCH 04/11] !fixup address latest comments, thanks --- .../Transforms/Vectorize/LoopVectorize.cpp | 5 ++- llvm/lib/Transforms/Vectorize/VPlan.cpp | 20 +++++++++ llvm/lib/Transforms/Vectorize/VPlan.h | 43 +++++-------------- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 32 -------------- 4 files changed, 34 insertions(+), 66 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 6c1c7126269b0..d2bbc10642114 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2883,8 +2883,11 @@ void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) { PHINode *NewPhi = cast(State.get(VPPhi)); // Make sure the builder has a valid insert point. Builder.SetInsertPoint(NewPhi); - for (const auto &[Inc, VPBB] : VPPhi->incoming_values_and_blocks()) + for (unsigned Idx = 0; Idx < VPPhi->getNumIncomingValues(); ++Idx) { + VPValue *Inc = VPPhi->getIncomingValue(Idx); + const VPBasicBlock *VPBB = VPPhi->getIncomingBlock(Idx); NewPhi->addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]); + } } } } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 701e4bfe0623e..d5fa50ed1a839 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -790,6 +790,26 @@ InstructionCost VPBasicBlock::cost(ElementCount VF, VPCostContext &Ctx) { return Cost; } +const VPBasicBlock *VPBasicBlock::getCFGPredecessor(unsigned Idx) const { + const VPBlockBase *Pred = nullptr; + if (getNumPredecessors() > 0) { + Pred = getPredecessors()[Idx]; + } else { + auto *Region = getParent(); + assert(Region && !Region->isReplicator() && Region->getEntry() == this && + "must be in the entry block of a non-replicate region"); + assert( + Idx < 2 && Region->getNumPredecessors() == 1 && + "when placed in an entry block, only 2 incoming blocks are available"); + + // Idx == 0 selects the predecessor of the region, Idx == 1 selects the + // region itself whose exiting block feeds the phi across the backedge. + Pred = Idx == 0 ? Region->getSinglePredecessor() : Region; + } + + return Pred->getExitingBasicBlock(); +} + InstructionCost VPRegionBlock::cost(ElementCount VF, VPCostContext &Ctx) { if (!isReplicator()) { InstructionCost Cost = 0; diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 83632a1c03b18..7ae52b8285867 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1180,42 +1180,13 @@ template class VPPhiAccessors { return getAsRecipe()->getOperand(Idx); } - /// Returns an interator range over the incoming values - VPUser::const_operand_range incoming_values() const { - return getAsRecipe()->operands(); - } - /// Returns the incoming block with index \p Idx. - const VPBasicBlock *getIncomingBlock(unsigned Idx) const; - - using const_incoming_block_iterator = - mapped_iterator>; - using const_incoming_blocks_range = - iterator_range; - - const_incoming_block_iterator incoming_block_begin() const { - return const_incoming_block_iterator( - detail::index_iterator(0), - [this](size_t Idx) { return getIncomingBlock(Idx); }); - } - const_incoming_block_iterator incoming_block_end() const { - return const_incoming_block_iterator( - detail::index_iterator(getAsRecipe()->getNumOperands()), - [this](size_t Idx) { return getIncomingBlock(Idx); }); + const VPBasicBlock *getIncomingBlock(unsigned Idx) const { + return getAsRecipe()->getParent()->getCFGPredecessor(Idx); } - /// Returns an iterator range over the incoming blocks. - const_incoming_blocks_range incoming_blocks() const { - return make_range(incoming_block_begin(), incoming_block_end()); - } - - /// Returns an iterator range over pairs of incoming values and corresponding - /// incoming blocks. - detail::zippy - incoming_values_and_blocks() const { - return zip(incoming_values(), incoming_blocks()); + unsigned getNumIncomingValues() const { + return getAsRecipe()->getNumOperands(); } }; @@ -3372,6 +3343,12 @@ class VPBasicBlock : public VPBlockBase { /// the cloned recipes. VPBasicBlock *clone() override; + /// Returns the predecessor block at index \p Idx with the predecessors as per + /// the corresponding plain CFG. If the block is an entry block to a region, + /// the first predecessor is the single predecessor of a region, and the + /// second predecessor is the exiting block of the region. + const VPBasicBlock *getCFGPredecessor(unsigned Idx) const; + protected: /// Execute the recipes in the IR basic block \p BB. void executeRecipes(VPTransformState *State, BasicBlock *BB); diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index b853babd61cf7..265aa7a3ae4df 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1205,32 +1205,6 @@ void VPIRPhi::print(raw_ostream &O, const Twine &Indent, } #endif -/// Returns the incoming block at index \p Idx for \p R. This handles both -/// recipes placed in entry blocks of loop regions (incoming blocks are the -/// region's predecessor and the region's exit) and other locations (incoming -/// blocks are the direct predecessors). -static const VPBasicBlock *getIncomingBlockForRecipe(const VPRecipeBase *R, - unsigned Idx) { - const VPBasicBlock *Parent = R->getParent(); - const VPBlockBase *Pred = nullptr; - if (Parent->getNumPredecessors() > 0) { - Pred = Parent->getPredecessors()[Idx]; - } else { - auto *Region = Parent->getParent(); - assert(Region && !Region->isReplicator() && Region->getEntry() == Parent && - "must be in the entry block of a non-replicate region"); - assert( - Idx < 2 && R->getNumOperands() == 2 && - "when placed in an entry block, only 2 incoming blocks are available"); - - // Idx == 0 selects the predecessor of the region, Idx == 1 selects the - // region itself whose exiting block feeds the phi across the backedge. - Pred = Idx == 0 ? Region->getSinglePredecessor() : Region; - } - - return Pred->getExitingBasicBlock(); -} - void VPIRMetadata::applyMetadata(Instruction &I) const { for (const auto &[Kind, Node] : Metadata) I.setMetadata(Kind, Node); @@ -3712,12 +3686,6 @@ void VPReductionPHIRecipe::print(raw_ostream &O, const Twine &Indent, } #endif -template <> -const VPBasicBlock * -VPPhiAccessors::getIncomingBlock(unsigned Idx) const { - return getIncomingBlockForRecipe(getAsRecipe(), Idx); -} - void VPWidenPHIRecipe::execute(VPTransformState &State) { assert(EnableVPlanNativePath && "Non-native vplans are not expected to have VPWidenPHIRecipes."); From 46780ab5e27274843eb0bdb6bfc04baa06d28d93 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 3 May 2025 10:52:21 +0100 Subject: [PATCH 05/11] !fixup address comments, make getAsRecipe pure virtual. --- llvm/lib/Transforms/Vectorize/VPlan.cpp | 7 +++---- llvm/lib/Transforms/Vectorize/VPlan.h | 26 +++++++++++++++---------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index d5fa50ed1a839..ff7fabaaa4d7a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -798,15 +798,14 @@ const VPBasicBlock *VPBasicBlock::getCFGPredecessor(unsigned Idx) const { auto *Region = getParent(); assert(Region && !Region->isReplicator() && Region->getEntry() == this && "must be in the entry block of a non-replicate region"); - assert( - Idx < 2 && Region->getNumPredecessors() == 1 && - "when placed in an entry block, only 2 incoming blocks are available"); + assert(Idx < 2 && Region->getNumPredecessors() == 1 && + "loop region has a single predecessor (preheader), its entry block " + "has 2 incoming blocks"); // Idx == 0 selects the predecessor of the region, Idx == 1 selects the // region itself whose exiting block feeds the phi across the backedge. Pred = Idx == 0 ? Region->getSinglePredecessor() : Region; } - return Pred->getExitingBasicBlock(); } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 7ae52b8285867..2768c32d24164 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1167,23 +1167,22 @@ class VPIRInstruction : public VPRecipeBase { }; /// Helper type to provide functions to access incoming values and blocks for -/// phi-like recipes. RecipeTy must be a sub-class of VPRecipeBase. -template class VPPhiAccessors { +/// phi-like recipes. +class VPPhiAccessors { +protected: /// Return a VPRecipeBase* to the current object. - const VPRecipeBase *getAsRecipe() const { - return static_cast(this); - } + virtual const VPRecipeBase *getAsRecipe() const = 0; public: + virtual ~VPPhiAccessors() = default; + /// Returns the incoming VPValue with index \p Idx. VPValue *getIncomingValue(unsigned Idx) const { return getAsRecipe()->getOperand(Idx); } /// Returns the incoming block with index \p Idx. - const VPBasicBlock *getIncomingBlock(unsigned Idx) const { - return getAsRecipe()->getParent()->getCFGPredecessor(Idx); - } + const VPBasicBlock *getIncomingBlock(unsigned Idx) const; unsigned getNumIncomingValues() const { return getAsRecipe()->getNumOperands(); @@ -1993,11 +1992,13 @@ class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe, /// recipe is placed in an entry block to a (non-replicate) region, it must have /// exactly 2 incoming values, the first from the predecessor of the region and /// the second from the exiting block of the region. -class VPWidenPHIRecipe : public VPSingleDefRecipe, - public VPPhiAccessors { +class VPWidenPHIRecipe : public VPSingleDefRecipe, public VPPhiAccessors { /// Name to use for the generated IR instruction for the widened phi. std::string Name; +protected: + const VPRecipeBase *getAsRecipe() const override { return this; } + public: /// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start and /// debug location \p DL. @@ -3363,6 +3364,11 @@ class VPBasicBlock : public VPBlockBase { BasicBlock *createEmptyBasicBlock(VPTransformState &State); }; +inline const VPBasicBlock * +VPPhiAccessors::getIncomingBlock(unsigned Idx) const { + return getAsRecipe()->getParent()->getCFGPredecessor(Idx); +} + /// A special type of VPBasicBlock that wraps an existing IR basic block. /// Recipes of the block get added before the first non-phi instruction in the /// wrapped block. From 5fee077e2cc337c8db74b234b8e60fcc00daf6c4 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 2 May 2025 15:32:30 +0100 Subject: [PATCH 06/11] Use virtual [VPlan] Use VPPhiAccessors for VPIRPhi. astOf --- llvm/lib/Transforms/Vectorize/VPlan.h | 54 +++++++++++++++++-- .../Transforms/Vectorize/VPlanVerifier.cpp | 37 ++++++++----- 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 2768c32d24164..84ef0ba986461 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1185,7 +1185,10 @@ class VPPhiAccessors { const VPBasicBlock *getIncomingBlock(unsigned Idx) const; unsigned getNumIncomingValues() const { - return getAsRecipe()->getNumOperands(); + auto *R = getAsRecipe(); + return R->getVPDefID() == VPDef::VPWidenIntOrFpInductionSC + ? 1 + : R->getNumOperands(); } }; @@ -1193,7 +1196,7 @@ class VPPhiAccessors { /// cast/dyn_cast/isa and execute() implementation. A single VPValue operand is /// allowed, and it is used to add a new incoming value for the single /// predecessor VPBB. -struct VPIRPhi : public VPIRInstruction { +struct VPIRPhi : public VPIRInstruction, public VPPhiAccessors { VPIRPhi(PHINode &PN) : VPIRInstruction(PN) {} static inline bool classof(const VPRecipeBase *U) { @@ -1210,6 +1213,9 @@ struct VPIRPhi : public VPIRInstruction { void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override; #endif + +protected: + const VPRecipeBase *getAsRecipe() const override { return this; } }; /// Helper to manage IR metadata for recipes. It filters out metadata that @@ -1735,13 +1741,15 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags, /// * VPWidenPointerInductionRecipe: Generate vector and scalar values for a /// pointer induction. Produces either a vector PHI per-part or scalar values /// per-lane based on the canonical induction. -class VPHeaderPHIRecipe : public VPSingleDefRecipe { +class VPHeaderPHIRecipe : public VPSingleDefRecipe, public VPPhiAccessors { protected: VPHeaderPHIRecipe(unsigned char VPDefID, Instruction *UnderlyingInstr, VPValue *Start, DebugLoc DL = {}) : VPSingleDefRecipe(VPDefID, ArrayRef({Start}), UnderlyingInstr, DL) { } + const VPRecipeBase *getAsRecipe() const override { return this; } + public: ~VPHeaderPHIRecipe() override = default; @@ -3218,6 +3226,46 @@ class VPScalarIVStepsRecipe : public VPRecipeWithIRFlags, } }; +/// Casting from VPRecipeBase -> VPPhiAccessors is supported for all recipe +/// types implementing VPPhiAccessors. +template <> struct CastIsPossible { + static inline bool isPossible(const VPRecipeBase *f) { + return isa(f); + } +}; +/// Support casting from VPRecipeBase -> VPPhiAccessors, by down-casting to the +/// recipe types implementing VPPhiAccessors. +template <> +struct CastInfo + : public CastIsPossible { + + using Self = CastInfo; + + using CastReturnType = + typename cast_retty::ret_type; + + static inline VPPhiAccessors *doCast(const VPRecipeBase *R) { + return const_cast([R]() -> const VPPhiAccessors * { + switch (R->getVPDefID()) { + case VPDef::VPIRInstructionSC: + return cast(R); + case VPDef::VPWidenPHISC: + return cast(R); + default: + return cast(R); + } + }()); + } + + static inline VPPhiAccessors *castFailed() { return nullptr; } + + static inline VPPhiAccessors *doCastIfPossible(const VPRecipeBase *f) { + if (!Self::isPossible(f)) + return castFailed(); + return doCast(f); + } +}; + /// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It /// holds a sequence of zero or more VPRecipe's each representing a sequence of /// output IR instructions. All PHI-like recipes must come before any non-PHI recipes. diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 3f10b1756d7a1..04f30003f372b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -179,8 +179,7 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { if (!verifyPhiRecipes(VPBB)) return false; - // Verify that defs in VPBB dominate all their uses. The current - // implementation is still incomplete. + // Verify that defs in VPBB dominate all their uses. DenseMap RecipeNumbering; unsigned Cnt = 0; for (const VPRecipeBase &R : *VPBB) @@ -207,25 +206,37 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { for (const VPUser *U : V->users()) { auto *UI = cast(U); - // TODO: check dominance of incoming values for phis properly. - if (!UI || - isa(UI) || - (isa(UI) && - cast(UI)->getOpcode() == Instruction::PHI)) + const VPBlockBase *UserVPBB = UI->getParent(); + if (auto *Phi = dyn_cast(UI)) { + for (unsigned Idx = 0; Idx != Phi->getNumIncomingValues(); ++Idx) { + VPValue *IncVPV = Phi->getIncomingValue(Idx); + const VPBasicBlock *IncVPBB = Phi->getIncomingBlock(Idx); + if (IncVPV != V) + continue; + if (IncVPBB != VPBB && !VPDT.dominates(VPBB, IncVPBB)) { + errs() << "Use before def!\n"; + return false; + } + } + continue; + } + if (isa(UI)) continue; - // If the user is in the same block, check it comes after R in the - // block. - if (UI->getParent() == VPBB) { + if (auto *VPI = dyn_cast(UI)) { + if (VPI->getOpcode() == Instruction::PHI) + continue; + } + // If the user is in the same block, check it comes after R in + // the block. + if (UserVPBB == VPBB) { if (RecipeNumbering[UI] < RecipeNumbering[&R]) { errs() << "Use before def!\n"; return false; } - continue; } - if (!VPDT.dominates(VPBB, UI->getParent())) { + if (!VPDT.dominates(VPBB, UserVPBB)) { errs() << "Use before def!\n"; return false; } From 60d5ec95f83354a57578f443bdaa4b02433722f5 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 11 May 2025 19:11:17 +0100 Subject: [PATCH 07/11] !fixup update after merging f2e62cfca5e571. --- llvm/lib/Transforms/Vectorize/VPlan.h | 9 ++++++++- llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | 4 ---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 0d7890e81e4f8..4068d5b7c1119 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1135,7 +1135,14 @@ class VPPhiAccessors { const VPBasicBlock *getIncomingBlock(unsigned Idx) const; /// Returns the number of incoming values, also number of incoming blocks. - unsigned getNumIncoming() const { return getAsRecipe()->getNumOperands(); } + /// Note that at the moment, VPWidenIntOrFpInductionRecipes only have a single + /// incoming value, its start value. + unsigned getNumIncoming() const { + const VPRecipeBase *R = getAsRecipe(); + return R->getVPDefID() == VPDef::VPWidenIntOrFpInductionSC + ? 1 + : getAsRecipe()->getNumOperands(); + } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// Print the recipe. diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index a955b165e8133..e36f12ae47c09 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -235,10 +235,6 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { if (isa(UI)) continue; - if (auto *VPI = dyn_cast(UI)) { - if (VPI->getOpcode() == Instruction::PHI) - continue; - } // If the user is in the same block, check it comes after R in // the block. if (UserVPBB == VPBB) { From ba48230a9c59038b78057a7ae27b01d2adc24da1 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 11 May 2025 22:43:19 +0100 Subject: [PATCH 08/11] !fixup fix formatting --- llvm/lib/Transforms/Vectorize/VPlan.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 4068d5b7c1119..b826e2a6ebab0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3317,7 +3317,7 @@ struct CastInfo return const_cast([R]() -> const VPPhiAccessors * { switch (R->getVPDefID()) { case VPDef::VPInstructionSC: - return cast (R); + return cast(R); case VPDef::VPIRInstructionSC: return cast(R); case VPDef::VPWidenPHISC: From 56d478496ecea1b14f723ece0503bd4ee2acd06b Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 13 May 2025 11:15:38 +0100 Subject: [PATCH 09/11] !fxup improve verifier message, add verification test. --- .../Transforms/Vectorize/VPlanVerifier.cpp | 11 +++++- .../Vectorize/VPlanVerifierTest.cpp | 37 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 297daf6ea4a41..769b52586c823 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -219,7 +219,6 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { for (const VPUser *U : V->users()) { auto *UI = cast(U); - const VPBlockBase *UserVPBB = UI->getParent(); if (auto *Phi = dyn_cast(UI)) { for (unsigned Idx = 0; Idx != Phi->getNumIncoming(); ++Idx) { VPValue *IncVPV = Phi->getIncomingValue(Idx); @@ -227,7 +226,15 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { if (IncVPV != V) continue; if (IncVPBB != VPBB && !VPDT.dominates(VPBB, IncVPBB)) { - errs() << "Use before def!\n"; + errs() << "Incoming def at index " << Idx + << " does not dominate incoming block!\n"; +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + VPSlotTracker Tracker(VPBB->getPlan()); + IncVPV->getDefiningRecipe()->print(errs(), " ", Tracker); + errs() << "\n does not dominate " << IncVPBB->getName() + << " for\n"; + UI->print(errs(), " ", Tracker); +#endif return false; } } diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp index 84b7e33146811..9e0196c44f441 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp @@ -143,6 +143,43 @@ TEST_F(VPVerifierTest, VPBlendUseBeforeDefDifferentBB) { delete Phi; } +TEST_F(VPVerifierTest, VPPhiIncomingValueDoesntDominateIncomingBlock) { + VPlan &Plan = getPlan(); + IntegerType *Int32 = IntegerType::get(C, 32); + VPValue *Zero = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 0)); + + VPBasicBlock *VPBB1 = Plan.getEntry(); + VPBasicBlock *VPBB2 = Plan.createVPBasicBlock(""); + VPBasicBlock *VPBB3 = Plan.createVPBasicBlock(""); + + VPInstruction *DefI = new VPInstruction(Instruction::Add, {Zero}); + VPPhi *Phi = new VPPhi({DefI}, {}); + VPBB2->appendRecipe(Phi); + VPBB2->appendRecipe(DefI); + auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {}); + VPBB3->appendRecipe(CanIV); + + VPRegionBlock *R1 = Plan.createVPRegionBlock(VPBB3, VPBB3, "R1"); + VPBlockUtils::connectBlocks(VPBB1, VPBB2); + VPBlockUtils::connectBlocks(VPBB2, R1); +#if GTEST_HAS_STREAM_REDIRECTION + ::testing::internal::CaptureStderr(); +#endif + EXPECT_FALSE(verifyVPlanIsValid(Plan)); +#if GTEST_HAS_STREAM_REDIRECTION +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + EXPECT_STREQ("Incoming def at index 0 does not dominate incoming block!\n" + " EMIT vp<%2> = add ir<0>\n" + " does not dominate preheader for\n" + " EMIT vp<%1> = phi [ vp<%2>, preheader ]", + ::testing::internal::GetCapturedStderr().c_str()); +#else + EXPECT_STREQ("Use before def!\n", + ::testing::internal::GetCapturedStderr().c_str()); +#endif +#endif +} + TEST_F(VPVerifierTest, DuplicateSuccessorsOutsideRegion) { VPlan &Plan = getPlan(); VPValue *Zero = Plan.getOrAddLiveIn(ConstantInt::get(Type::getInt32Ty(C), 0)); From d371f1066e22802269fd294c4cf2cf08d801d38d Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 14 May 2025 19:32:58 +0100 Subject: [PATCH 10/11] !fixup address latest comments, thanks! --- llvm/lib/Transforms/Vectorize/VPlan.h | 21 +++++++------------ .../Transforms/Vectorize/VPlanVerifier.cpp | 5 +++-- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index b826e2a6ebab0..35e7554754406 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1135,13 +1135,8 @@ class VPPhiAccessors { const VPBasicBlock *getIncomingBlock(unsigned Idx) const; /// Returns the number of incoming values, also number of incoming blocks. - /// Note that at the moment, VPWidenIntOrFpInductionRecipes only have a single - /// incoming value, its start value. - unsigned getNumIncoming() const { - const VPRecipeBase *R = getAsRecipe(); - return R->getVPDefID() == VPDef::VPWidenIntOrFpInductionSC - ? 1 - : getAsRecipe()->getNumOperands(); + virtual unsigned getNumIncoming() const { + return getAsRecipe()->getNumOperands(); } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) @@ -1992,6 +1987,11 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe { return isUnrolled() ? getOperand(getNumOperands() - 2) : nullptr; } + /// Returns the number of incoming values, also number of incoming blocks. + /// Note that at the moment, VPWidenIntOrFpInductionRecipes only have a single + /// incoming value, its start value. + unsigned getNumIncoming() const override { return 1; } + /// Returns the first defined value as TruncInst, if it is one or nullptr /// otherwise. TruncInst *getTruncInst() { return Trunc; } @@ -3310,9 +3310,6 @@ struct CastInfo using Self = CastInfo; - using CastReturnType = - typename cast_retty::ret_type; - static inline VPPhiAccessors *doCast(const VPRecipeBase *R) { return const_cast([R]() -> const VPPhiAccessors * { switch (R->getVPDefID()) { @@ -3328,11 +3325,9 @@ struct CastInfo }()); } - static inline VPPhiAccessors *castFailed() { return nullptr; } - static inline VPPhiAccessors *doCastIfPossible(const VPRecipeBase *f) { if (!Self::isPossible(f)) - return castFailed(); + return nullptr; return doCast(f); } }; diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 769b52586c823..49130e0d61537 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -222,10 +222,10 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { if (auto *Phi = dyn_cast(UI)) { for (unsigned Idx = 0; Idx != Phi->getNumIncoming(); ++Idx) { VPValue *IncVPV = Phi->getIncomingValue(Idx); - const VPBasicBlock *IncVPBB = Phi->getIncomingBlock(Idx); if (IncVPV != V) continue; - if (IncVPBB != VPBB && !VPDT.dominates(VPBB, IncVPBB)) { + const VPBasicBlock *IncVPBB = Phi->getIncomingBlock(Idx); + if (!VPDT.dominates(VPBB, IncVPBB)) { errs() << "Incoming def at index " << Idx << " does not dominate incoming block!\n"; #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) @@ -240,6 +240,7 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { } continue; } + // TODO: Also verify VPPredInstPHIRecipe. if (isa(UI)) continue; From 5fa41af8e4aa01db7dc06e64aa98ba976a8dc4d7 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 15 May 2025 11:05:56 +0100 Subject: [PATCH 11/11] !fixup address latest comments, thanks! --- llvm/lib/Transforms/Vectorize/VPlan.h | 7 +++-- .../Transforms/Vectorize/VPlanVerifier.cpp | 28 ++++++++++--------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 35e7554754406..5fd7a369bf735 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3296,20 +3296,22 @@ class VPScalarIVStepsRecipe : public VPRecipeWithIRFlags, }; /// Casting from VPRecipeBase -> VPPhiAccessors is supported for all recipe -/// types implementing VPPhiAccessors. +/// types implementing VPPhiAccessors. Used by isa<> & co. template <> struct CastIsPossible { static inline bool isPossible(const VPRecipeBase *f) { + // TODO: include VPPredInstPHIRecipe too, once it implements VPPhiAccessors. return isa(f); } }; /// Support casting from VPRecipeBase -> VPPhiAccessors, by down-casting to the -/// recipe types implementing VPPhiAccessors. +/// recipe types implementing VPPhiAccessors. Used by cast<>, dyn_cast<> & co. template <> struct CastInfo : public CastIsPossible { using Self = CastInfo; + /// doCast is used by cast<>. static inline VPPhiAccessors *doCast(const VPRecipeBase *R) { return const_cast([R]() -> const VPPhiAccessors * { switch (R->getVPDefID()) { @@ -3325,6 +3327,7 @@ struct CastInfo }()); } + /// doCastIfPossible is used by dyn_cast<>. static inline VPPhiAccessors *doCastIfPossible(const VPRecipeBase *f) { if (!Self::isPossible(f)) return nullptr; diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 49130e0d61537..1e7e039a18d56 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -221,22 +221,24 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { auto *UI = cast(U); if (auto *Phi = dyn_cast(UI)) { for (unsigned Idx = 0; Idx != Phi->getNumIncoming(); ++Idx) { - VPValue *IncVPV = Phi->getIncomingValue(Idx); - if (IncVPV != V) + VPValue *IncomingVPV = Phi->getIncomingValue(Idx); + if (IncomingVPV != V) continue; - const VPBasicBlock *IncVPBB = Phi->getIncomingBlock(Idx); - if (!VPDT.dominates(VPBB, IncVPBB)) { - errs() << "Incoming def at index " << Idx - << " does not dominate incoming block!\n"; + + const VPBasicBlock *IncomingVPBB = Phi->getIncomingBlock(Idx); + if (VPDT.dominates(VPBB, IncomingVPBB)) + continue; + + errs() << "Incoming def at index " << Idx + << " does not dominate incoming block!\n"; #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - VPSlotTracker Tracker(VPBB->getPlan()); - IncVPV->getDefiningRecipe()->print(errs(), " ", Tracker); - errs() << "\n does not dominate " << IncVPBB->getName() - << " for\n"; - UI->print(errs(), " ", Tracker); + VPSlotTracker Tracker(VPBB->getPlan()); + IncomingVPV->getDefiningRecipe()->print(errs(), " ", Tracker); + errs() << "\n does not dominate " << IncomingVPBB->getName() + << " for\n"; + UI->print(errs(), " ", Tracker); #endif - return false; - } + return false; } continue; }