Skip to content

Commit 2372421

Browse files
fhahnfhahn
committed
!fixup fix after merge.
1 parent fe03d51 commit 2372421

17 files changed

+266
-105
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7500,10 +7500,10 @@ LoopVectorizationPlanner::executePlan(
75007500
using namespace llvm::VPlanPatternMatch;
75017501
if (MiddleVPBB->begin() != MiddleVPBB->end() &&
75027502
match(&MiddleVPBB->back(), m_BranchOnCond(m_VPValue()))) {
7503-
cast<VPIRWrapperBlock>(MiddleVPBB->getSuccessors()[1])
7503+
cast<VPIRBasicBlock>(MiddleVPBB->getSuccessors()[1])
75047504
->resetBlock(OrigLoop->getLoopPreheader());
75057505
} else
7506-
cast<VPIRWrapperBlock>(MiddleVPBB->getSuccessors()[0])
7506+
cast<VPIRBasicBlock>(MiddleVPBB->getSuccessors()[0])
75077507
->resetBlock(OrigLoop->getLoopPreheader());
75087508

75097509
// Only use noalias metadata when using memory checks guaranteeing no overlap

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Lines changed: 9 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -442,10 +442,13 @@ VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
442442
return NewBB;
443443
}
444444

445-
void VPIRWrapperBlock::execute(VPTransformState *State) {
445+
void VPIRBasicBlock::execute(VPTransformState *State) {
446446
assert(getHierarchicalSuccessors().empty() &&
447447
"VPIRBasicBlock cannot have successors at the moment");
448448

449+
State->Builder.SetInsertPoint(getIRBasicBlock()->getTerminator());
450+
executeRecipes(State, getIRBasicBlock());
451+
449452
for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) {
450453
VPBasicBlock *PredVPBB = PredVPBlock->getExitingBasicBlock();
451454
auto &PredVPSuccessors = PredVPBB->getHierarchicalSuccessors();
@@ -462,7 +465,7 @@ void VPIRWrapperBlock::execute(VPTransformState *State) {
462465
unsigned idx = PredVPSuccessors.front() == this ? 0 : 1;
463466
assert(!TermBr->getSuccessor(idx) &&
464467
"Trying to reset an existing successor block.");
465-
TermBr->setSuccessor(idx, WrappedBlock);
468+
TermBr->setSuccessor(idx, IRBB);
466469
}
467470
}
468471
}
@@ -663,12 +666,6 @@ void VPBasicBlock::print(raw_ostream &O, const Twine &Indent,
663666
printSuccessors(O, Indent);
664667
}
665668

666-
void VPIRWrapperBlock::print(raw_ostream &O, const Twine &Indent,
667-
VPSlotTracker &SlotTracker) const {
668-
O << Indent << "ir-bb<" << getName() << ">\n";
669-
printSuccessors(O, Indent);
670-
}
671-
672669
#endif
673670

674671
static std::pair<VPBlockBase *, VPBlockBase *> cloneSESE(VPBlockBase *Entry);
@@ -840,7 +837,7 @@ VPlanPtr VPlan::createInitialVPlan(const SCEV *TripCount, ScalarEvolution &SE,
840837

841838
BasicBlock *EB = TheLoop->getUniqueExitBlock();
842839
if (RequiresScalarEpilogueCheck) {
843-
VPIRWrapperBlock *EBWrapper = new VPIRWrapperBlock(EB);
840+
auto *EBWrapper = new VPIRBasicBlock(EB);
844841
VPBlockUtils::insertBlockAfter(EBWrapper, MiddleVPBB);
845842

846843
auto *ScalarLatchTerm = TheLoop->getLoopLatch()->getTerminator();
@@ -863,7 +860,7 @@ VPlanPtr VPlan::createInitialVPlan(const SCEV *TripCount, ScalarEvolution &SE,
863860
MiddleVPBB->appendRecipe(Br);
864861
}
865862
BasicBlock *Header = TheLoop->getHeader();
866-
VPIRWrapperBlock *PHWrapper = new VPIRWrapperBlock(Header);
863+
VPIRBasicBlock *PHWrapper = new VPIRBasicBlock(Header);
867864
VPBlockUtils::connectBlocks(MiddleVPBB, PHWrapper);
868865

869866
return Plan;
@@ -1218,8 +1215,6 @@ void VPlanPrinter::dumpBlock(const VPBlockBase *Block) {
12181215
dumpBasicBlock(BasicBlock);
12191216
else if (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
12201217
dumpRegion(Region);
1221-
else if (const auto *Wrapper = dyn_cast<VPIRWrapperBlock>(Block))
1222-
dumpIRWrapperBlock(Wrapper);
12231218
else
12241219
llvm_unreachable("Unsupported kind of VPBlock.");
12251220
}
@@ -1229,9 +1224,9 @@ void VPlanPrinter::drawEdge(const VPBlockBase *From, const VPBlockBase *To,
12291224
// Due to "dot" we print an edge between two regions as an edge between the
12301225
// exiting basic block and the entry basic of the respective regions.
12311226
const VPBlockBase *Tail =
1232-
isa<VPIRWrapperBlock>(From) ? From : From->getExitingBasicBlock();
1227+
isa<VPIRBasicBlock>(From) ? From : From->getExitingBasicBlock();
12331228
const VPBlockBase *Head =
1234-
isa<VPIRWrapperBlock>(To) ? To : To->getEntryBasicBlock();
1229+
isa<VPIRBasicBlock>(To) ? To : To->getEntryBasicBlock();
12351230
OS << Indent << getUID(Tail) << " -> " << getUID(Head);
12361231
OS << " [ label=\"" << Label << '\"';
12371232
if (Tail != From)
@@ -1287,34 +1282,6 @@ void VPlanPrinter::dumpBasicBlock(const VPBasicBlock *BasicBlock) {
12871282
dumpEdges(BasicBlock);
12881283
}
12891284

1290-
void VPlanPrinter::dumpIRWrapperBlock(const VPIRWrapperBlock *WrapperBlock) {
1291-
OS << Indent << getUID(WrapperBlock) << " [label =\n";
1292-
bumpIndent(1);
1293-
std::string Str;
1294-
raw_string_ostream SS(Str);
1295-
// Use no indentation as we need to wrap the lines into quotes ourselves.
1296-
WrapperBlock->print(SS, "", SlotTracker);
1297-
1298-
// We need to process each line of the output separately, so split
1299-
// single-string plain-text dump.
1300-
SmallVector<StringRef, 0> Lines;
1301-
StringRef(Str).rtrim('\n').split(Lines, "\n");
1302-
1303-
auto EmitLine = [&](StringRef Line, StringRef Suffix) {
1304-
OS << Indent << '"' << DOT::EscapeString(Line.str()) << "\\l\"" << Suffix;
1305-
};
1306-
1307-
// Don't need the "+" after the last line.
1308-
for (auto Line : make_range(Lines.begin(), Lines.end() - 1))
1309-
EmitLine(Line, " +\n");
1310-
EmitLine(Lines.back(), "\n");
1311-
1312-
bumpIndent(-1);
1313-
OS << Indent << "]\n";
1314-
1315-
dumpEdges(WrapperBlock);
1316-
}
1317-
13181285
void VPlanPrinter::dumpRegion(const VPRegionBlock *Region) {
13191286
OS << Indent << "subgraph " << getUID(Region) << " {\n";
13201287
bumpIndent(1);

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 2 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -2833,43 +2833,6 @@ class VPScalarIVStepsRecipe : public VPRecipeWithIRFlags {
28332833
}
28342834
};
28352835

2836-
class VPIRWrapperBlock : public VPBlockBase {
2837-
BasicBlock *WrappedBlock;
2838-
2839-
public:
2840-
VPIRWrapperBlock(BasicBlock *WrappedBlock)
2841-
: VPBlockBase(VPIRWrapperBlockSC, WrappedBlock->getName().str()),
2842-
WrappedBlock(WrappedBlock) {}
2843-
2844-
static inline bool classof(const VPBlockBase *V) {
2845-
return V->getVPBlockID() == VPBlockBase::VPIRWrapperBlockSC;
2846-
}
2847-
2848-
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2849-
/// Print this VPBsicBlock to \p O, prefixing all lines with \p Indent. \p
2850-
/// SlotTracker is used to print unnamed VPValue's using consequtive numbers.
2851-
///
2852-
/// Note that the numbering is applied to the whole VPlan, so printing
2853-
/// individual blocks is consistent with the whole VPlan printing.
2854-
void print(raw_ostream &O, const Twine &Indent,
2855-
VPSlotTracker &SlotTracker) const override;
2856-
using VPBlockBase::print; // Get the print(raw_stream &O) version.
2857-
#endif
2858-
/// The method which generates the output IR instructions that correspond to
2859-
/// this VPBasicBlock, thereby "executing" the VPlan.
2860-
void execute(VPTransformState *State) override;
2861-
2862-
VPIRWrapperBlock *clone() override {
2863-
return new VPIRWrapperBlock(WrappedBlock);
2864-
}
2865-
2866-
void dropAllReferences(VPValue *NewValue) override {}
2867-
2868-
void resetBlock(BasicBlock *N) { WrappedBlock = N; }
2869-
2870-
BasicBlock *getWrappedBlock() { return WrappedBlock; }
2871-
};
2872-
28732836
/// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It
28742837
/// holds a sequence of zero or more VPRecipe's each representing a sequence of
28752838
/// output IR instructions. All PHI-like recipes must come before any non-PHI recipes.
@@ -3038,6 +3001,8 @@ class VPIRBasicBlock : public VPBasicBlock {
30383001
}
30393002

30403003
BasicBlock *getIRBasicBlock() const { return IRBB; }
3004+
3005+
void resetBlock(BasicBlock *BB) { IRBB = BB; }
30413006
};
30423007

30433008
/// VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks
@@ -3412,8 +3377,6 @@ class VPlanPrinter {
34123377
/// its successor blocks.
34133378
void dumpBasicBlock(const VPBasicBlock *BasicBlock);
34143379

3415-
void dumpIRWrapperBlock(const VPIRWrapperBlock *WrapperBlock);
3416-
34173380
/// Print a given \p Region of the Plan.
34183381
void dumpRegion(const VPRegionBlock *Region);
34193382

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -367,7 +367,8 @@ static bool mergeBlocksIntoPredecessors(VPlan &Plan) {
367367
vp_depth_first_deep(Plan.getEntry()))) {
368368
auto *PredVPBB =
369369
dyn_cast_or_null<VPBasicBlock>(VPBB->getSinglePredecessor());
370-
if (PredVPBB && PredVPBB->getNumSuccessors() == 1)
370+
if (PredVPBB && PredVPBB->getNumSuccessors() == 1 &&
371+
!isa<VPIRBasicBlock>(VPBB))
371372
WorkList.push_back(VPBB);
372373
}
373374

‎llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -160,10 +160,6 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
160160
errs() << "Same IR basic block used by multiple wrapper blocks!\n";
161161
return false;
162162
}
163-
if (IRBB != IRBB->getPlan()->getPreheader()) {
164-
errs() << "VPIRBasicBlock can only be used as pre-header at the moment!\n";
165-
return false;
166-
}
167163
return true;
168164
}
169165

‎llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,14 @@ target triple = "aarch64-unknown-linux-gnu"
3535
; CHECK-NEXT: Successor(s): middle.block
3636
; CHECK-EMPTY:
3737
; CHECK-NEXT: middle.block:
38+
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, vp<[[VTC]]>
39+
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
40+
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, ir-bb<for.body>
41+
; CHECK-EMPTY:
42+
; CHECK-NEXT: ir-bb<for.cond.cleanup>
43+
; CHECK-NEXT: No successors
44+
; CHECK-EMPTY:
45+
; CHECK-NEXT: ir-bb<for.body>
3846
; CHECK-NEXT: No successors
3947
; CHECK-NEXT: }
4048

@@ -64,6 +72,14 @@ target triple = "aarch64-unknown-linux-gnu"
6472
; CHECK-NEXT: Successor(s): middle.block
6573
; CHECK-EMPTY:
6674
; CHECK-NEXT: middle.block:
75+
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, vp<[[VTC]]>
76+
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
77+
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, ir-bb<for.body>
78+
; CHECK-EMPTY:
79+
; CHECK-NEXT: ir-bb<for.cond.cleanup>
80+
; CHECK-NEXT: No successors
81+
; CHECK-EMPTY:
82+
; CHECK-NEXT: ir-bb<for.body>
6783
; CHECK-NEXT: No successors
6884
; CHECK-NEXT: }
6985

@@ -98,6 +114,14 @@ target triple = "aarch64-unknown-linux-gnu"
98114
; CHECK-NEXT: Successor(s): middle.block
99115
; CHECK-EMPTY:
100116
; CHECK-NEXT: middle.block:
117+
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, vp<[[VTC]]>
118+
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
119+
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, ir-bb<for.body>
120+
; CHECK-EMPTY:
121+
; CHECK-NEXT: ir-bb<for.cond.cleanup>
122+
; CHECK-NEXT: No successors
123+
; CHECK-EMPTY:
124+
; CHECK-NEXT: ir-bb<for.body>
101125
; CHECK-NEXT: No successors
102126
; CHECK-NEXT: }
103127

@@ -127,6 +151,14 @@ target triple = "aarch64-unknown-linux-gnu"
127151
; CHECK-NEXT: Successor(s): middle.block
128152
; CHECK-EMPTY:
129153
; CHECK-NEXT: middle.block:
154+
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, vp<[[VTC]]>
155+
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
156+
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, ir-bb<for.body>
157+
; CHECK-EMPTY:
158+
; CHECK-NEXT: ir-bb<for.cond.cleanup>
159+
; CHECK-NEXT: No successors
160+
; CHECK-EMPTY:
161+
; CHECK-NEXT: ir-bb<for.body>
130162
; CHECK-NEXT: No successors
131163
; CHECK-NEXT: }
132164

@@ -160,6 +192,14 @@ target triple = "aarch64-unknown-linux-gnu"
160192
; CHECK-NEXT: Successor(s): middle.block
161193
; CHECK-EMPTY:
162194
; CHECK-NEXT: middle.block:
195+
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, vp<[[VTC]]>
196+
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
197+
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, ir-bb<for.body>
198+
; CHECK-EMPTY:
199+
; CHECK-NEXT: ir-bb<for.cond.cleanup>
200+
; CHECK-NEXT: No successors
201+
; CHECK-EMPTY:
202+
; CHECK-NEXT: ir-bb<for.body>
163203
; CHECK-NEXT: No successors
164204
; CHECK-NEXT: }
165205

@@ -189,6 +229,14 @@ target triple = "aarch64-unknown-linux-gnu"
189229
; CHECK-NEXT: Successor(s): middle.block
190230
; CHECK-EMPTY:
191231
; CHECK-NEXT: middle.block:
232+
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, vp<[[VTC]]>
233+
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
234+
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, ir-bb<for.body>
235+
; CHECK-EMPTY:
236+
; CHECK-NEXT: ir-bb<for.cond.cleanup>
237+
; CHECK-NEXT: No successors
238+
; CHECK-EMPTY:
239+
; CHECK-NEXT: ir-bb<for.body>
192240
; CHECK-NEXT: No successors
193241
; CHECK-NEXT: }
194242

‎llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,14 @@ target triple = "arm64-apple-ios"
3333
; CHECK-NEXT: Successor(s): middle.block
3434
; CHECK-EMPTY:
3535
; CHECK-NEXT: middle.block:
36+
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, vp<[[VTC]]>
37+
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
38+
; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<loop>
39+
; CHECK-EMPTY:
40+
; CHECK-NEXT: ir-bb<exit>
41+
; CHECK-NEXT: No successors
42+
; CHECK-EMPTY:
43+
; CHECK-NEXT: ir-bb<loop>
3644
; CHECK-NEXT: No successors
3745
; CHECK-NEXT: }
3846

@@ -62,6 +70,14 @@ target triple = "arm64-apple-ios"
6270
; CHECK-NEXT: Successor(s): middle.block
6371
; CHECK-EMPTY:
6472
; CHECK-NEXT: middle.block:
73+
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, vp<[[VTC]]>
74+
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
75+
; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<loop>
76+
; CHECK-EMPTY:
77+
; CHECK-NEXT: ir-bb<exit>
78+
; CHECK-NEXT: No successors
79+
; CHECK-EMPTY:
80+
; CHECK-NEXT: ir-bb<loop>
6581
; CHECK-NEXT: No successors
6682
; CHECK-NEXT: }
6783
;

‎llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
8282
; CHECK: middle.block:
8383
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]>
8484
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
85-
; CHECK-NEXT: Successor(s): for.cond.cleanup.loopexit, for.body
85+
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup.loopexit>, ir-bb<for.body>
8686
; CHECK-EMPTY:
8787
; CHECK-NEXT: ir-bb<for.cond.cleanup.loopexit>
8888
; CHECK-NEXT: No successors
@@ -232,7 +232,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
232232
; CHECK: middle.block:
233233
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]>
234234
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
235-
; CHECK-NEXT: Successor(s): for.cond.cleanup.loopexit, for.body
235+
; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup.loopexit>, ir-bb<for.body>
236236
; CHECK-EMPTY:
237237
; CHECK-NEXT: ir-bb<for.cond.cleanup.loopexit>
238238
; CHECK-NEXT: No successors

‎llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) {
3535
; CHECK-NEXT: middle.block:
3636
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1000>, vp<[[VTC]]>
3737
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
38-
; CHECK-NEXT: Successor(s): exit, loop
38+
; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<loop>
3939
; CHECK-EMPTY:
4040
; CHECK-NEXT: ir-bb<exit>
4141
; CHECK-NEXT: No successors
@@ -99,7 +99,7 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) {
9999
; CHECK-NEXT: middle.block:
100100
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1000>, vp<[[VTC]]>
101101
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
102-
; CHECK-NEXT: Successor(s): exit, loop
102+
; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<loop>
103103
; CHECK-EMPTY:
104104
; CHECK-NEXT: ir-bb<exit>
105105
; CHECK-NEXT: No successors

0 commit comments

Comments
 (0)