!fixup address comments

Author: fhahn

llvm/lib/Transforms/Vectorize/VPlan.h

@@ -167,7 +167,7 @@ class VPLane {
 
   static VPLane getFirstLane() { return VPLane(0, VPLane::Kind::First); }
 
-  static VPLane getLastLaneForVF(const ElementCount &VF, unsigned Offset = 1) {
+  static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset) {
     unsigned LaneOffset = VF.getKnownMinValue() - Offset;
     Kind LaneKind;
     if (VF.isScalable())
@@ -179,6 +179,10 @@ class VPLane {
     return VPLane(LaneOffset, LaneKind);
   }
 
+  static VPLane getLastLaneForVF(const ElementCount &VF) {
+    return getLaneFromEnd(VF, 1);
+  }
+
   /// Returns a compile-time known value for the lane index and asserts if the
   /// lane can only be calculated at runtime.
   unsigned getKnownLane() const {
@@ -1182,7 +1186,7 @@ class VPInstruction : public VPRecipeWithIRFlags {
     BranchOnCount,
     BranchOnCond,
     ComputeReductionResult,
-    ExtractRecurrenceResult,
+    ExtractFromEnd,
     LogicalAnd, // Non-poison propagating logical And.
     // Add an offset in bytes (second operand) to a base pointer (first
     // operand). Only generates scalar values (either for the first lane only or
@@ -3659,7 +3663,7 @@ inline bool isUniformAfterVectorization(VPValue *VPV) {
     return all_of(GEP->operands(), isUniformAfterVectorization);
   if (auto *VPI = dyn_cast<VPInstruction>(Def))
     return VPI->getOpcode() == VPInstruction::ComputeReductionResult ||
-           VPI->getOpcode() == VPInstruction::ExtractRecurrenceResult;
+           VPI->getOpcode() == VPInstruction::ExtractFromEnd;
   return false;
 }
 } // end namespace vputils

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -137,7 +137,7 @@ bool VPRecipeBase::mayHaveSideEffects() const {
     case VPInstruction::Not:
     case VPInstruction::CalculateTripCountMinusVF:
     case VPInstruction::CanonicalIVIncrementForPart:
-    case VPInstruction::ExtractRecurrenceResult:
+    case VPInstruction::ExtractFromEnd:
     case VPInstruction::LogicalAnd:
     case VPInstruction::PtrAdd:
       return false;
@@ -301,7 +301,7 @@ bool VPInstruction::canGenerateScalarForFirstLane() const {
   case VPInstruction::CalculateTripCountMinusVF:
   case VPInstruction::CanonicalIVIncrementForPart:
   case VPInstruction::ComputeReductionResult:
-  case VPInstruction::ExtractRecurrenceResult:
+  case VPInstruction::ExtractFromEnd:
   case VPInstruction::PtrAdd:
   case VPInstruction::ExplicitVectorLength:
     return true;
@@ -560,23 +560,25 @@ Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) {
 
     return ReducedPartRdx;
   }
-  case VPInstruction::ExtractRecurrenceResult: {
+  case VPInstruction::ExtractFromEnd: {
     if (Part != 0)
       return State.get(this, 0, /*IsScalar*/ true);
 
-    // Extract the second last element in the middle block for users outside the
-    // loop.
+    auto *CI = cast<ConstantInt>(getOperand(1)->getLiveInIRValue());
+    unsigned Offset = CI->getZExtValue();
+
+    // Extract lane VF - Offset in the from the operand.
     Value *Res;
     if (State.VF.isVector()) {
       Res = State.get(
           getOperand(0),
-          VPIteration(State.UF - 1, VPLane::getLastLaneForVF(State.VF, 2)));
+          VPIteration(State.UF - 1, VPLane::getLaneFromEnd(State.VF, Offset)));
     } else {
       assert(State.UF > 1 && "VF and UF cannot both be 1");
       // When loop is unrolled without vectorizing, retrieve the value just
       // prior to the final unrolled value. This is analogous to the vectorized
       // case above: extracting the second last element when VF > 1.
-      Res = State.get(getOperand(0), State.UF - 2);
+      Res = State.get(getOperand(0), State.UF - Offset);
     }
     Res->setName(Name);
     return Res;
@@ -621,7 +623,7 @@ void VPInstruction::execute(VPTransformState &State) {
   bool GeneratesPerFirstLaneOnly =
       canGenerateScalarForFirstLane() &&
       (vputils::onlyFirstLaneUsed(this) ||
-       getOpcode() == VPInstruction::ExtractRecurrenceResult ||
+       getOpcode() == VPInstruction::ExtractFromEnd ||
        getOpcode() == VPInstruction::ComputeReductionResult);
   bool GeneratesPerAllLanes = doesGeneratePerAllLanes();
   for (unsigned Part = 0; Part < State.UF; ++Part) {
@@ -716,8 +718,8 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
   case VPInstruction::BranchOnCount:
     O << "branch-on-count";
     break;
-  case VPInstruction::ExtractRecurrenceResult:
-    O << "extract-recurrence-result";
+  case VPInstruction::ExtractFromEnd:
+    O << "extract-from-end";
     break;
   case VPInstruction::ComputeReductionResult:
     O << "compute-reduction-result";

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -802,7 +802,7 @@ sinkRecurrenceUsersAfterPrevious(VPFirstOrderRecurrencePHIRecipe *FOR,
 }
 
 bool VPlanTransforms::adjustFixedOrderRecurrences(VPlan &Plan,
-                                                  VPBuilder &Builder) {
+                                                  VPBuilder &LoopBuilder) {
   VPDominatorTree VPDT;
   VPDT.recalculate(Plan);
 
@@ -833,22 +833,26 @@ bool VPlanTransforms::adjustFixedOrderRecurrences(VPlan &Plan,
     // fixed-order recurrence.
     VPBasicBlock *InsertBlock = Previous->getParent();
     if (isa<VPHeaderPHIRecipe>(Previous))
-      Builder.setInsertPoint(InsertBlock, InsertBlock->getFirstNonPhi());
+      LoopBuilder.setInsertPoint(InsertBlock, InsertBlock->getFirstNonPhi());
     else
-      Builder.setInsertPoint(InsertBlock, std::next(Previous->getIterator()));
+      LoopBuilder.setInsertPoint(InsertBlock,
+                                 std::next(Previous->getIterator()));
 
     auto *RecurSplice = cast<VPInstruction>(
-        Builder.createNaryOp(VPInstruction::FirstOrderRecurrenceSplice,
-                             {FOR, FOR->getBackedgeValue()}));
+        LoopBuilder.createNaryOp(VPInstruction::FirstOrderRecurrenceSplice,
+                                 {FOR, FOR->getBackedgeValue()}));
 
     FOR->replaceAllUsesWith(RecurSplice);
     // Set the first operand of RecurSplice to FOR again, after replacing
     // all users.
     RecurSplice->setOperand(0, FOR);
 
+    Type *IntTy = Plan.getCanonicalIV()->getScalarType();
     auto *Result = cast<VPInstruction>(MiddleBuilder.createNaryOp(
-        VPInstruction::ExtractRecurrenceResult, {FOR->getBackedgeValue()}, {},
-        "vector.recur.extract.for.phi"));
+        VPInstruction::ExtractFromEnd,
+        {FOR->getBackedgeValue(),
+         Plan.getOrAddLiveIn(ConstantInt::get(IntTy, 2))},
+        {}, "vector.recur.extract.for.phi"));
     RecurSplice->replaceUsesWithIf(
         Result, [](VPUser &U, unsigned) { return isa<VPLiveOut>(&U); });
   }

llvm/test/Transforms/LoopVectorize/vplan-printing.ll

@@ -911,7 +911,7 @@ define i16 @print_first_order_recurrence_and_result(ptr %ptr) {
 ; CHECK-NEXT: Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
-; CHECK-NEXT:   EMIT vp<[[FOR_RESULT:%.+]]> = extract-recurrence-result ir<%for.1.next>
+; CHECK-NEXT:   EMIT vp<[[FOR_RESULT:%.+]]> = extract-from-end ir<%for.1.next>, ir<2>
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT:   Live-out i16 %for.1.lcssa = vp<[[FOR_RESULT]]>