@@ -970,7 +970,8 @@ class BoUpSLP {
970
970
/// Vectorize the tree but with the list of externally used values \p
971
971
/// ExternallyUsedValues. Values in this MapVector can be replaced but the
972
972
/// generated extractvalue instructions.
973
- Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
973
+ Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues,
974
+ Instruction *ReductionRoot = nullptr);
974
975
975
976
/// \returns the cost incurred by unwanted spills and fills, caused by
976
977
/// holding live values over call sites.
@@ -9002,8 +9003,8 @@ struct ShuffledInsertData {
9002
9003
};
9003
9004
} // namespace
9004
9005
9005
- Value *
9006
- BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues ) {
9006
+ Value *BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues,
9007
+ Instruction *ReductionRoot ) {
9007
9008
// All blocks must be scheduled before any instructions are inserted.
9008
9009
for (auto &BSIter : BlocksSchedules) {
9009
9010
scheduleBlock(BSIter.second.get());
@@ -9020,7 +9021,8 @@ BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
9020
9021
EntryToLastInstruction.try_emplace(E.get(), LastInst);
9021
9022
}
9022
9023
9023
- Builder.SetInsertPoint(&F->getEntryBlock().front());
9024
+ Builder.SetInsertPoint(ReductionRoot ? ReductionRoot
9025
+ : &F->getEntryBlock().front());
9024
9026
auto *VectorRoot = vectorizeTree(VectorizableTree[0].get());
9025
9027
9026
9028
// If the vectorized tree can be rewritten in a smaller type, we truncate the
@@ -11944,16 +11946,18 @@ class HorizontalReduction {
11944
11946
11945
11947
Builder.setFastMathFlags(RdxFMF);
11946
11948
11947
- // Vectorize a tree.
11948
- Value *VectorizedRoot = V.vectorizeTree(LocalExternallyUsedValues);
11949
-
11950
11949
// Emit a reduction. If the root is a select (min/max idiom), the insert
11951
11950
// point is the compare condition of that select.
11952
11951
Instruction *RdxRootInst = cast<Instruction>(ReductionRoot);
11952
+ Instruction *InsertPt = RdxRootInst;
11953
11953
if (IsCmpSelMinMax)
11954
- Builder.SetInsertPoint(GetCmpForMinMaxReduction(RdxRootInst));
11955
- else
11956
- Builder.SetInsertPoint(RdxRootInst);
11954
+ InsertPt = GetCmpForMinMaxReduction(RdxRootInst);
11955
+
11956
+ // Vectorize a tree.
11957
+ Value *VectorizedRoot =
11958
+ V.vectorizeTree(LocalExternallyUsedValues, InsertPt);
11959
+
11960
+ Builder.SetInsertPoint(InsertPt);
11957
11961
11958
11962
// To prevent poison from leaking across what used to be sequential,
11959
11963
// safe, scalar boolean logic operations, the reduction operand must be
0 commit comments