diff --git a/llvm/include/llvm/Support/GenericLoopInfo.h b/llvm/include/llvm/Support/GenericLoopInfo.h index d560ca648132c..2775a8734dd47 100644 --- a/llvm/include/llvm/Support/GenericLoopInfo.h +++ b/llvm/include/llvm/Support/GenericLoopInfo.h @@ -294,6 +294,10 @@ template class LoopBase { /// Otherwise return null. BlockT *getUniqueExitBlock() const; + /// Return the unique exit block for the latch, or null if there are multiple + /// different exit blocks or the latch is not exiting. + BlockT *getUniqueLatchExitBlock() const; + /// Return true if this loop does not have any exit blocks. bool hasNoExitBlocks() const; diff --git a/llvm/include/llvm/Support/GenericLoopInfoImpl.h b/llvm/include/llvm/Support/GenericLoopInfoImpl.h index d19022729ace3..35eecd340d7d0 100644 --- a/llvm/include/llvm/Support/GenericLoopInfoImpl.h +++ b/llvm/include/llvm/Support/GenericLoopInfoImpl.h @@ -159,6 +159,17 @@ BlockT *LoopBase::getUniqueExitBlock() const { return getExitBlockHelper(this, true).first; } +template +BlockT *LoopBase::getUniqueLatchExitBlock() const { + BlockT *Latch = getLoopLatch(); + assert(Latch && "Latch block must exists"); + auto IsExitBlock = [this](BlockT *BB, bool AllowRepeats) -> BlockT * { + assert(!AllowRepeats && "Unexpected parameter value."); + return !contains(BB) ? BB : nullptr; + }; + return find_singleton(children(Latch), IsExitBlock); +} + /// getExitEdges - Return all pairs of (_inside_block_,_outside_block_). template void LoopBase::getExitEdges( diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 58fcba93f1a18..9e64d2b8de7bf 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -634,10 +634,6 @@ class InnerLoopVectorizer { /// Middle Block between the vector and the scalar. BasicBlock *LoopMiddleBlock; - /// The unique ExitBlock of the scalar loop if one exists. Note that - /// there can be multiple exiting edges reaching this block. - BasicBlock *LoopExitBlock; - /// A list of all bypass blocks. The first block is the entry of the loop. SmallVector LoopBypassBlocks; @@ -1365,8 +1361,8 @@ class LoopVectorizationCostModel { // If we might exit from anywhere but the latch, must run the exiting // iteration in scalar form. if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) { - LLVM_DEBUG( - dbgs() << "LV: Loop requires scalar epilogue: multiple exits\n"); + LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: not exiting " + "from latch block\n"); return true; } if (IsVectorizing && InterleaveInfo.requiresScalarEpilogue()) { @@ -2025,8 +2021,7 @@ class GeneratedRTChecks { /// adjusts the branches to branch to the vector preheader or \p Bypass, /// depending on the generated condition. BasicBlock *emitSCEVChecks(BasicBlock *Bypass, - BasicBlock *LoopVectorPreHeader, - BasicBlock *LoopExitBlock) { + BasicBlock *LoopVectorPreHeader) { if (!SCEVCheckCond) return nullptr; @@ -2522,7 +2517,7 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) { BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) { BasicBlock *const SCEVCheckBlock = - RTChecks.emitSCEVChecks(Bypass, LoopVectorPreHeader, LoopExitBlock); + RTChecks.emitSCEVChecks(Bypass, LoopVectorPreHeader); if (!SCEVCheckBlock) return nullptr; @@ -2576,8 +2571,8 @@ BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(BasicBlock *Bypass) { void InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) { LoopVectorPreHeader = OrigLoop->getLoopPreheader(); assert(LoopVectorPreHeader && "Invalid loop structure"); - LoopExitBlock = OrigLoop->getUniqueExitBlock(); // may be nullptr - assert((LoopExitBlock || Cost->requiresScalarEpilogue(VF.isVector())) && + assert((OrigLoop->getUniqueExitBlock() || + Cost->requiresScalarEpilogue(VF.isVector())) && "multiple exit loop without required epilogue?"); LoopMiddleBlock = @@ -7927,7 +7922,7 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton( // If there is an epilogue which must run, there's no edge from the // middle block to exit blocks and thus no need to update the immediate // dominator of the exit blocks. - DT->changeImmediateDominator(LoopExitBlock, + DT->changeImmediateDominator(OrigLoop->getUniqueLatchExitBlock(), EPI.EpilogueIterationCountCheck); // Keep track of bypass blocks, as they feed start values to the induction and