From 1b83ae508b9c9e73a0ff98b04be4af699217bac3 Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Thu, 10 Apr 2025 17:32:46 -0700 Subject: [PATCH 1/8] [LV] Check if the VF is scalar by VFRange instead of VPlan in `handleUncountableEarlyExit` NFC. This patch check if the plan contains scalar VF by VFRange instead of Plan. Then we can furthur clamp the range after this transformation. Split by #113903. --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 5 +++-- llvm/lib/Transforms/Vectorize/VPlanTransforms.h | 4 +++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index a4d546f698d5f..0cec4c5c87f67 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -9744,7 +9744,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { Legal->getUncountableEarlyExitingBlock()) { VPlanTransforms::runPass(VPlanTransforms::handleUncountableEarlyExit, *Plan, *PSE.getSE(), OrigLoop, UncountableExitingBlock, - RecipeBuilder); + RecipeBuilder, Range); } DenseMap IVEndValues; addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 67a55aa67c978..b49a0c452fa1e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -17,6 +17,7 @@ #include "VPlanAnalysis.h" #include "VPlanCFG.h" #include "VPlanDominatorTree.h" +#include "VPlanHelpers.h" #include "VPlanPatternMatch.h" #include "VPlanUtils.h" #include "VPlanVerifier.h" @@ -2380,7 +2381,7 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) { void VPlanTransforms::handleUncountableEarlyExit( VPlan &Plan, ScalarEvolution &SE, Loop *OrigLoop, - BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder) { + BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder, VFRange &Range) { VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion(); auto *LatchVPBB = cast(LoopRegion->getExiting()); VPBuilder Builder(LatchVPBB->getTerminator()); @@ -2436,7 +2437,7 @@ void VPlanTransforms::handleUncountableEarlyExit( ExitIRI->extractLastLaneOfOperand(MiddleBuilder); } // Add the incoming value from the early exit. - if (!IncomingFromEarlyExit->isLiveIn() && !Plan.hasScalarVFOnly()) { + if (!IncomingFromEarlyExit->isLiveIn() && !Range.Start.isScalar()) { VPValue *FirstActiveLane = EarlyExitB.createNaryOp( VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr, "first.active.lane"); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index c23ff38265670..85649367c3478 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -28,6 +28,7 @@ class PredicatedScalarEvolution; class TargetLibraryInfo; class VPBuilder; class VPRecipeBuilder; +class VFRange; extern cl::opt VerifyEachVPlan; @@ -174,7 +175,8 @@ struct VPlanTransforms { static void handleUncountableEarlyExit(VPlan &Plan, ScalarEvolution &SE, Loop *OrigLoop, BasicBlock *UncountableExitingBlock, - VPRecipeBuilder &RecipeBuilder); + VPRecipeBuilder &RecipeBuilder, + VFRange &Range); /// Lower abstract recipes to concrete ones, that can be codegen'd. static void convertToConcreteRecipes(VPlan &Plan); From b7eca73760804353e09ed9810c9cbb4e1e0fb1ec Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Thu, 10 Apr 2025 18:23:23 -0700 Subject: [PATCH 2/8] !fixup, formatting. --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index b49a0c452fa1e..c551cfa6ba513 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2381,7 +2381,8 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) { void VPlanTransforms::handleUncountableEarlyExit( VPlan &Plan, ScalarEvolution &SE, Loop *OrigLoop, - BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder, VFRange &Range) { + BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder, + VFRange &Range) { VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion(); auto *LatchVPBB = cast(LoopRegion->getExiting()); VPBuilder Builder(LatchVPBB->getTerminator()); From 9e96e1d6f56797562fd7fa3152c7e281893097b9 Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Fri, 11 Apr 2025 01:26:11 -0700 Subject: [PATCH 3/8] !fixup, add const. --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 +- llvm/lib/Transforms/Vectorize/VPlanTransforms.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index c551cfa6ba513..1cd1f5acda305 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2382,7 +2382,7 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) { void VPlanTransforms::handleUncountableEarlyExit( VPlan &Plan, ScalarEvolution &SE, Loop *OrigLoop, BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder, - VFRange &Range) { + const VFRange &Range) { VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion(); auto *LatchVPBB = cast(LoopRegion->getExiting()); VPBuilder Builder(LatchVPBB->getTerminator()); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index 85649367c3478..401cd8b7bd4e7 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -176,7 +176,7 @@ struct VPlanTransforms { Loop *OrigLoop, BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder, - VFRange &Range); + const VFRange &Range); /// Lower abstract recipes to concrete ones, that can be codegen'd. static void convertToConcreteRecipes(VPlan &Plan); From 01625c087a3fc0ed63b47fa8da853b260972293d Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Fri, 11 Apr 2025 02:04:02 -0700 Subject: [PATCH 4/8] !fixup, clamp the range to make sure the plan has scalar/vector VF only. --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 9 +++++++-- llvm/lib/Transforms/Vectorize/VPlanTransforms.h | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 1cd1f5acda305..59c5dff35b37c 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2382,13 +2382,18 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) { void VPlanTransforms::handleUncountableEarlyExit( VPlan &Plan, ScalarEvolution &SE, Loop *OrigLoop, BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder, - const VFRange &Range) { + VFRange &Range) { VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion(); auto *LatchVPBB = cast(LoopRegion->getExiting()); VPBuilder Builder(LatchVPBB->getTerminator()); auto *MiddleVPBB = Plan.getMiddleBlock(); VPValue *IsEarlyExitTaken = nullptr; + // Clamp the range that make sure we insert extractElement for incoming value + // correctly. + bool IsScalarVF = LoopVectorizationPlanner::getDecisionAndClampRange( + [&](ElementCount VF) { return VF.isScalar(); }, Range); + // Process the uncountable exiting block. Update IsEarlyExitTaken, which // tracks if the uncountable early exit has been taken. Also split the middle // block and have it conditionally branch to the early exit block if @@ -2438,7 +2443,7 @@ void VPlanTransforms::handleUncountableEarlyExit( ExitIRI->extractLastLaneOfOperand(MiddleBuilder); } // Add the incoming value from the early exit. - if (!IncomingFromEarlyExit->isLiveIn() && !Range.Start.isScalar()) { + if (!IncomingFromEarlyExit->isLiveIn() && !IsScalarVF) { VPValue *FirstActiveLane = EarlyExitB.createNaryOp( VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr, "first.active.lane"); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index 401cd8b7bd4e7..85649367c3478 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -176,7 +176,7 @@ struct VPlanTransforms { Loop *OrigLoop, BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder, - const VFRange &Range); + VFRange &Range); /// Lower abstract recipes to concrete ones, that can be codegen'd. static void convertToConcreteRecipes(VPlan &Plan); From 65bf3359885ff630fa5a17b94f4f3ad9df69e672 Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Sun, 13 Apr 2025 20:43:16 -0700 Subject: [PATCH 5/8] !fixup, address comment. Only clamp the range when extract is needed. --- .../Transforms/Vectorize/VPlanTransforms.cpp | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 59c5dff35b37c..e2fcc769566cd 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2389,11 +2389,6 @@ void VPlanTransforms::handleUncountableEarlyExit( auto *MiddleVPBB = Plan.getMiddleBlock(); VPValue *IsEarlyExitTaken = nullptr; - // Clamp the range that make sure we insert extractElement for incoming value - // correctly. - bool IsScalarVF = LoopVectorizationPlanner::getDecisionAndClampRange( - [&](ElementCount VF) { return VF.isScalar(); }, Range); - // Process the uncountable exiting block. Update IsEarlyExitTaken, which // tracks if the uncountable early exit has been taken. Also split the middle // block and have it conditionally branch to the early exit block if @@ -2443,13 +2438,20 @@ void VPlanTransforms::handleUncountableEarlyExit( ExitIRI->extractLastLaneOfOperand(MiddleBuilder); } // Add the incoming value from the early exit. - if (!IncomingFromEarlyExit->isLiveIn() && !IsScalarVF) { - VPValue *FirstActiveLane = EarlyExitB.createNaryOp( - VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr, - "first.active.lane"); - IncomingFromEarlyExit = EarlyExitB.createNaryOp( - Instruction::ExtractElement, {IncomingFromEarlyExit, FirstActiveLane}, - nullptr, "early.exit.value"); + if (!IncomingFromEarlyExit->isLiveIn()) { + // Limit range to scalar VF only, if the range contains the scalar VF. + bool IsVectorVF = LoopVectorizationPlanner::getDecisionAndClampRange( + [&](ElementCount VF) { return VF.isVector(); }, Range); + + if (IsVectorVF) { + VPValue *FirstActiveLane = EarlyExitB.createNaryOp( + VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr, + "first.active.lane"); + IncomingFromEarlyExit = + EarlyExitB.createNaryOp(Instruction::ExtractElement, + {IncomingFromEarlyExit, FirstActiveLane}, + nullptr, "early.exit.value"); + } } ExitIRI->addOperand(IncomingFromEarlyExit); } From 53c168b9058a1644c2058035dedd7dc515b41c55 Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Mon, 14 Apr 2025 16:31:20 -0700 Subject: [PATCH 6/8] !fixup, address comment. --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index e2fcc769566cd..4971bd9fe1eca 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2441,7 +2441,7 @@ void VPlanTransforms::handleUncountableEarlyExit( if (!IncomingFromEarlyExit->isLiveIn()) { // Limit range to scalar VF only, if the range contains the scalar VF. bool IsVectorVF = LoopVectorizationPlanner::getDecisionAndClampRange( - [&](ElementCount VF) { return VF.isVector(); }, Range); + [](ElementCount VF) { return VF.isVector(); }, Range); if (IsVectorVF) { VPValue *FirstActiveLane = EarlyExitB.createNaryOp( From ce30391a149cb704185de909010ecd03870ecb03 Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Tue, 15 Apr 2025 01:06:14 -0700 Subject: [PATCH 7/8] !fixup, address comments. --- .../Transforms/Vectorize/VPlanTransforms.cpp | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 4971bd9fe1eca..06d299a2d8f08 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2437,21 +2437,18 @@ void VPlanTransforms::handleUncountableEarlyExit( ExitIRI->addOperand(IncomingFromLatch); ExitIRI->extractLastLaneOfOperand(MiddleBuilder); } + + auto IsVector = [](ElementCount VF) { return VF.isVector(); }; // Add the incoming value from the early exit. - if (!IncomingFromEarlyExit->isLiveIn()) { - // Limit range to scalar VF only, if the range contains the scalar VF. - bool IsVectorVF = LoopVectorizationPlanner::getDecisionAndClampRange( - [](ElementCount VF) { return VF.isVector(); }, Range); - - if (IsVectorVF) { - VPValue *FirstActiveLane = EarlyExitB.createNaryOp( - VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr, - "first.active.lane"); - IncomingFromEarlyExit = - EarlyExitB.createNaryOp(Instruction::ExtractElement, - {IncomingFromEarlyExit, FirstActiveLane}, - nullptr, "early.exit.value"); - } + if (!IncomingFromEarlyExit->isLiveIn() && + // Limit range to scalar VF only, if the range contains the scalar VF. + LoopVectorizationPlanner::getDecisionAndClampRange(IsVector, Range)) { + VPValue *FirstActiveLane = EarlyExitB.createNaryOp( + VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr, + "first.active.lane"); + IncomingFromEarlyExit = EarlyExitB.createNaryOp( + Instruction::ExtractElement, {IncomingFromEarlyExit, FirstActiveLane}, + nullptr, "early.exit.value"); } ExitIRI->addOperand(IncomingFromEarlyExit); } From dcaed17c60ad69ab05744499bbb848b59813904f Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Wed, 16 Apr 2025 17:08:44 -0700 Subject: [PATCH 8/8] !fixup, update comments. --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 06d299a2d8f08..e6651a505d2a1 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2439,9 +2439,11 @@ void VPlanTransforms::handleUncountableEarlyExit( } auto IsVector = [](ElementCount VF) { return VF.isVector(); }; - // Add the incoming value from the early exit. + // When the VFs are vectors, need to add `extract` to get the incoming value + // from early exit. When the range contains scalar VF, limit the range to + // scalar VF to prevent mis-compilation for the range containing both scalar + // and vector VFs. if (!IncomingFromEarlyExit->isLiveIn() && - // Limit range to scalar VF only, if the range contains the scalar VF. LoopVectorizationPlanner::getDecisionAndClampRange(IsVector, Range)) { VPValue *FirstActiveLane = EarlyExitB.createNaryOp( VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr,