Skip to content

Commit f3e47f5

Browse files
fhahnfhahn
committed
!fixup rebase and fixup
1 parent cba8b59 commit f3e47f5

File tree

4 files changed

+72
-136
lines changed

4 files changed

+72
-136
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 0 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -9230,87 +9230,6 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
92309230
VPlanTransforms::clearReductionWrapFlags(*Plan);
92319231
}
92329232

9233-
void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
9234-
assert(IndDesc.getKind() == InductionDescriptor::IK_PtrInduction &&
9235-
"Not a pointer induction according to InductionDescriptor!");
9236-
assert(cast<PHINode>(getUnderlyingInstr())->getType()->isPointerTy() &&
9237-
"Unexpected type.");
9238-
assert(!onlyScalarsGenerated(State.VF.isScalable()) &&
9239-
"Recipe should have been replaced");
9240-
9241-
auto *IVR = getParent()->getPlan()->getCanonicalIV();
9242-
PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, 0, /*IsScalar*/ true));
9243-
unsigned CurrentPart = 0;
9244-
if (getNumOperands() == 5)
9245-
CurrentPart =
9246-
cast<ConstantInt>(getOperand(4)->getLiveInIRValue())->getZExtValue();
9247-
Type *PhiType = IndDesc.getStep()->getType();
9248-
9249-
// Build a pointer phi
9250-
Value *ScalarStartValue = getStartValue()->getLiveInIRValue();
9251-
Type *ScStValueType = ScalarStartValue->getType();
9252-
PHINode *NewPointerPhi = nullptr;
9253-
9254-
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
9255-
if (getNumOperands() == 5) {
9256-
auto *GEP = cast<GetElementPtrInst>(State.get(getOperand(3), 0));
9257-
NewPointerPhi = cast<PHINode>(GEP->getPointerOperand());
9258-
} else {
9259-
NewPointerPhi =
9260-
PHINode::Create(ScStValueType, 2, "pointer.phi", CanonicalIV);
9261-
NewPointerPhi->addIncoming(ScalarStartValue, VectorPH);
9262-
}
9263-
9264-
// A pointer induction, performed by using a gep
9265-
BasicBlock::iterator InductionLoc = State.Builder.GetInsertPoint();
9266-
unsigned UF = getNumOperands() == 2
9267-
? 1
9268-
: cast<ConstantInt>(getOperand(2)->getLiveInIRValue())
9269-
->getZExtValue();
9270-
9271-
Value *ScalarStepValue = State.get(getOperand(1), VPIteration(0, 0));
9272-
Value *RuntimeVF = getRuntimeVF(State.Builder, PhiType, State.VF);
9273-
Value *NumUnrolledElems =
9274-
State.Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, UF));
9275-
// Add induction update using an incorrect block temporarily. The phi node
9276-
// will be fixed after VPlan execution. Note that at this point the latch
9277-
// block cannot be used, as it does not exist yet.
9278-
// TODO: Model increment value in VPlan, by turning the recipe into a
9279-
// multi-def and a subclass of VPHeaderPHIRecipe.
9280-
if (getNumOperands() != 5) {
9281-
Value *InductionGEP = GetElementPtrInst::Create(
9282-
State.Builder.getInt8Ty(), NewPointerPhi,
9283-
State.Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind",
9284-
InductionLoc);
9285-
9286-
NewPointerPhi->addIncoming(InductionGEP, VectorPH);
9287-
}
9288-
9289-
// Create UF many actual address geps that use the pointer
9290-
// phi as base and a vectorized version of the step value
9291-
// (<step*0, ..., step*N>) as offset.
9292-
for (unsigned Part = 0; Part < State.UF; ++Part) {
9293-
Type *VecPhiType = VectorType::get(PhiType, State.VF);
9294-
Value *StartOffsetScalar = State.Builder.CreateMul(
9295-
RuntimeVF, ConstantInt::get(PhiType, CurrentPart));
9296-
Value *StartOffset =
9297-
State.Builder.CreateVectorSplat(State.VF, StartOffsetScalar);
9298-
// Create a vector of consecutive numbers from zero to VF.
9299-
StartOffset = State.Builder.CreateAdd(
9300-
StartOffset, State.Builder.CreateStepVector(VecPhiType));
9301-
9302-
assert(ScalarStepValue == State.get(getOperand(1), VPIteration(Part, 0)) &&
9303-
"scalar step must be the same across all parts");
9304-
Value *GEP = State.Builder.CreateGEP(
9305-
State.Builder.getInt8Ty(), NewPointerPhi,
9306-
State.Builder.CreateMul(
9307-
StartOffset,
9308-
State.Builder.CreateVectorSplat(State.VF, ScalarStepValue),
9309-
"vector.gep"));
9310-
State.set(this, GEP, Part);
9311-
}
9312-
}
9313-
93149233
void VPDerivedIVRecipe::execute(VPTransformState &State) {
93159234
assert(!State.Instance && "VPDerivedIVRecipe being replicated.");
93169235

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1682,8 +1682,7 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
16821682
bool onlyFirstPartUsed(const VPValue *Op) const override {
16831683
assert(is_contained(operands(), Op) &&
16841684
"Op must be an operand of the recipe");
1685-
assert(getNumOperands() == 1 && "must have a single operand");
1686-
return true;
1685+
return Op == getOperand(0);
16871686
}
16881687

16891688
VPVectorPointerRecipe *clone() override {

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -692,6 +692,7 @@ unsigned VPInstruction::getInterleaveCount() const {
692692
? 1
693693
: cast<ConstantInt>(getOperand(1)->getLiveInIRValue())
694694
->getZExtValue();
695+
}
695696

696697
#if !defined(NDEBUG)
697698
bool VPInstruction::isFPMathOp() const {
@@ -2531,42 +2532,59 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
25312532

25322533
auto *IVR = getParent()->getPlan()->getCanonicalIV();
25332534
PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, 0, /*IsScalar*/ true));
2535+
unsigned CurrentPart = 0;
2536+
if (getNumOperands() == 5)
2537+
CurrentPart =
2538+
cast<ConstantInt>(getOperand(4)->getLiveInIRValue())->getZExtValue();
25342539
Type *PhiType = IndDesc.getStep()->getType();
25352540

25362541
// Build a pointer phi
25372542
Value *ScalarStartValue = getStartValue()->getLiveInIRValue();
25382543
Type *ScStValueType = ScalarStartValue->getType();
2539-
PHINode *NewPointerPhi = PHINode::Create(ScStValueType, 2, "pointer.phi",
2540-
CanonicalIV->getIterator());
2544+
PHINode *NewPointerPhi = nullptr;
25412545

25422546
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
2543-
NewPointerPhi->addIncoming(ScalarStartValue, VectorPH);
2547+
if (getNumOperands() == 5) {
2548+
auto *GEP = cast<GetElementPtrInst>(State.get(getOperand(3), 0));
2549+
NewPointerPhi = cast<PHINode>(GEP->getPointerOperand());
2550+
} else {
2551+
NewPointerPhi =
2552+
PHINode::Create(ScStValueType, 2, "pointer.phi", CanonicalIV);
2553+
NewPointerPhi->addIncoming(ScalarStartValue, VectorPH);
2554+
}
25442555

25452556
// A pointer induction, performed by using a gep
25462557
BasicBlock::iterator InductionLoc = State.Builder.GetInsertPoint();
2558+
unsigned UF = getNumOperands() == 2
2559+
? 1
2560+
: cast<ConstantInt>(getOperand(2)->getLiveInIRValue())
2561+
->getZExtValue();
25472562

25482563
Value *ScalarStepValue = State.get(getOperand(1), VPIteration(0, 0));
25492564
Value *RuntimeVF = getRuntimeVF(State.Builder, PhiType, State.VF);
25502565
Value *NumUnrolledElems =
2551-
State.Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, State.UF));
2552-
Value *InductionGEP = GetElementPtrInst::Create(
2553-
State.Builder.getInt8Ty(), NewPointerPhi,
2554-
State.Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind",
2555-
InductionLoc);
2566+
State.Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, UF));
25562567
// Add induction update using an incorrect block temporarily. The phi node
25572568
// will be fixed after VPlan execution. Note that at this point the latch
25582569
// block cannot be used, as it does not exist yet.
25592570
// TODO: Model increment value in VPlan, by turning the recipe into a
25602571
// multi-def and a subclass of VPHeaderPHIRecipe.
2561-
NewPointerPhi->addIncoming(InductionGEP, VectorPH);
2572+
if (getNumOperands() != 5) {
2573+
Value *InductionGEP = GetElementPtrInst::Create(
2574+
State.Builder.getInt8Ty(), NewPointerPhi,
2575+
State.Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind",
2576+
InductionLoc);
2577+
2578+
NewPointerPhi->addIncoming(InductionGEP, VectorPH);
2579+
}
25622580

25632581
// Create UF many actual address geps that use the pointer
25642582
// phi as base and a vectorized version of the step value
25652583
// (<step*0, ..., step*N>) as offset.
25662584
for (unsigned Part = 0; Part < State.UF; ++Part) {
25672585
Type *VecPhiType = VectorType::get(PhiType, State.VF);
2568-
Value *StartOffsetScalar =
2569-
State.Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, Part));
2586+
Value *StartOffsetScalar = State.Builder.CreateMul(
2587+
RuntimeVF, ConstantInt::get(PhiType, CurrentPart));
25702588
Value *StartOffset =
25712589
State.Builder.CreateVectorSplat(State.VF, StartOffsetScalar);
25722590
// Create a vector of consecutive numbers from zero to VF.

0 commit comments

Comments
 (0)