diff --git a/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp b/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp index fb236aeb982e0..81d9334356520 100644 --- a/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp +++ b/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp @@ -221,5 +221,5 @@ void tu2(int &i) { } } -// CHECK: [[BW_LIKELY]] = !{!"branch_weights", i32 2000, i32 1} -// CHECK: [[BW_UNLIKELY]] = !{!"branch_weights", i32 1, i32 2000} +// CHECK: [[BW_LIKELY]] = !{!"branch_weights", !"expected", i32 2000, i32 1} +// CHECK: [[BW_UNLIKELY]] = !{!"branch_weights", !"expected", i32 1, i32 2000} diff --git a/llvm/docs/BranchWeightMetadata.rst b/llvm/docs/BranchWeightMetadata.rst index 522f37cdad4fc..62204753e29b0 100644 --- a/llvm/docs/BranchWeightMetadata.rst +++ b/llvm/docs/BranchWeightMetadata.rst @@ -28,11 +28,14 @@ Supported Instructions Metadata is only assigned to the conditional branches. There are two extra operands for the true and the false branch. +We optionally track if the metadata was added by ``__builtin_expect`` or +``__builtin_expect_with_probability`` with an optional field ``!"expected"``. .. code-block:: none !0 = !{ !"branch_weights", + [ !"expected", ] i32 , i32 } @@ -47,6 +50,7 @@ is always case #0). !0 = !{ !"branch_weights", + [ !"expected", ] i32 [ , i32 ... ] } @@ -60,6 +64,7 @@ Branch weights are assigned to every destination. !0 = !{ !"branch_weights", + [ !"expected", ] i32 [ , i32 ... ] } @@ -75,6 +80,7 @@ block and entry counts which may not be accurate with sampling. !0 = !{ !"branch_weights", + [ !"expected", ] i32 } @@ -95,6 +101,7 @@ is used. !0 = !{ !"branch_weights", + [ !"expected", ] i32 [ , i32 ] } diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h index 3265589b7c8df..e02ec8f5a3d8b 100644 --- a/llvm/include/llvm/IR/MDBuilder.h +++ b/llvm/include/llvm/IR/MDBuilder.h @@ -59,7 +59,11 @@ class MDBuilder { //===------------------------------------------------------------------===// /// Return metadata containing two branch weights. - MDNode *createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight); + /// @param TrueWeight the weight of the true branch + /// @param FalseWeight the weight of the false branch + /// @param Do these weights come from __builtin_expect* + MDNode *createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, + bool IsExpected = false); /// Return metadata containing two branch weights, with significant bias /// towards `true` destination. @@ -70,7 +74,10 @@ class MDBuilder { MDNode *createUnlikelyBranchWeights(); /// Return metadata containing a number of branch weights. - MDNode *createBranchWeights(ArrayRef Weights); + /// @param Weights the weights of all the branches + /// @param Do these weights come from __builtin_expect* + MDNode *createBranchWeights(ArrayRef Weights, + bool IsExpected = false); /// Return metadata specifying that a branch or switch is unpredictable. MDNode *createUnpredictable(); diff --git a/llvm/include/llvm/IR/ProfDataUtils.h b/llvm/include/llvm/IR/ProfDataUtils.h index 88fbad4d6b9d8..1d7c97d9be953 100644 --- a/llvm/include/llvm/IR/ProfDataUtils.h +++ b/llvm/include/llvm/IR/ProfDataUtils.h @@ -55,6 +55,17 @@ MDNode *getBranchWeightMDNode(const Instruction &I); /// Nullptr otherwise. MDNode *getValidBranchWeightMDNode(const Instruction &I); +/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect* +/// intrinsic +bool hasBranchWeightOrigin(const Instruction &I); + +/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect* +/// intrinsic +bool hasBranchWeightOrigin(const MDNode *ProfileData); + +/// Return the offset to the first branch weight data +unsigned getBranchWeightOffset(const MDNode *ProfileData); + /// Extract branch weights from MD_prof metadata /// /// \param ProfileData A pointer to an MDNode. @@ -111,7 +122,11 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalWeights); /// Create a new `branch_weights` metadata node and add or overwrite /// a `prof` metadata reference to instruction `I`. -void setBranchWeights(Instruction &I, ArrayRef Weights); +/// \param I the Instruction to set branch weights on. +/// \param Weights an array of weights to set on instruction I. +/// \param IsExpected were these weights added from an llvm.expect* intrinsic. +void setBranchWeights(Instruction &I, ArrayRef Weights, + bool IsExpected); /// Scaling the profile data attached to 'I' using the ratio of S/T. void scaleProfData(Instruction &I, uint64_t S, uint64_t T); diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index 339a1f1f2f002..0e01080bd75cc 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -8866,7 +8866,8 @@ bool CodeGenPrepare::splitBranchCondition(Function &F, ModifyDT &ModifiedDT) { scaleWeights(NewTrueWeight, NewFalseWeight); Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext()) - .createBranchWeights(TrueWeight, FalseWeight)); + .createBranchWeights(TrueWeight, FalseWeight, + hasBranchWeightOrigin(*Br1))); NewTrueWeight = TrueWeight; NewFalseWeight = 2 * FalseWeight; diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp index 29272e627a1d1..aec927a8cf31d 100644 --- a/llvm/lib/IR/Instruction.cpp +++ b/llvm/lib/IR/Instruction.cpp @@ -1268,12 +1268,23 @@ Instruction *Instruction::cloneImpl() const { void Instruction::swapProfMetadata() { MDNode *ProfileData = getBranchWeightMDNode(*this); - if (!ProfileData || ProfileData->getNumOperands() != 3) + if (!ProfileData) + return; + unsigned FirstIdx = getBranchWeightOffset(ProfileData); + if (ProfileData->getNumOperands() != 2 + FirstIdx) return; - // The first operand is the name. Fetch them backwards and build a new one. - Metadata *Ops[] = {ProfileData->getOperand(0), ProfileData->getOperand(2), - ProfileData->getOperand(1)}; + unsigned SecondIdx = FirstIdx + 1; + SmallVector Ops; + // If there are more weights past the second, we can't swap them + if (ProfileData->getNumOperands() > SecondIdx + 1) + return; + for (unsigned Idx = 0; Idx < FirstIdx; ++Idx) { + Ops.push_back(ProfileData->getOperand(Idx)); + } + // Switch the order of the weights + Ops.push_back(ProfileData->getOperand(SecondIdx)); + Ops.push_back(ProfileData->getOperand(FirstIdx)); setMetadata(LLVMContext::MD_prof, MDNode::get(ProfileData->getContext(), Ops)); } diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index 1213f078d05ec..de369bd62a617 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -5199,7 +5199,11 @@ void SwitchInstProfUpdateWrapper::init() { if (!ProfileData) return; - if (ProfileData->getNumOperands() != SI.getNumSuccessors() + 1) { + // FIXME: This check belongs in ProfDataUtils. Its almost equivalent to + // getValidBranchWeightMDNode(), but the need to use llvm_unreachable + // makes them slightly different. + if (ProfileData->getNumOperands() != + SI.getNumSuccessors() + getBranchWeightOffset(ProfileData)) { llvm_unreachable("number of prof branch_weights metadata operands does " "not correspond to number of succesors"); } diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp index bd68db3a6f961..000027754d13e 100644 --- a/llvm/lib/IR/MDBuilder.cpp +++ b/llvm/lib/IR/MDBuilder.cpp @@ -35,8 +35,8 @@ MDNode *MDBuilder::createFPMath(float Accuracy) { } MDNode *MDBuilder::createBranchWeights(uint32_t TrueWeight, - uint32_t FalseWeight) { - return createBranchWeights({TrueWeight, FalseWeight}); + uint32_t FalseWeight, bool IsExpected) { + return createBranchWeights({TrueWeight, FalseWeight}, IsExpected); } MDNode *MDBuilder::createLikelyBranchWeights() { @@ -49,15 +49,19 @@ MDNode *MDBuilder::createUnlikelyBranchWeights() { return createBranchWeights(1, (1U << 20) - 1); } -MDNode *MDBuilder::createBranchWeights(ArrayRef Weights) { +MDNode *MDBuilder::createBranchWeights(ArrayRef Weights, + bool IsExpected) { assert(Weights.size() >= 1 && "Need at least one branch weights!"); - SmallVector Vals(Weights.size() + 1); + unsigned int Offset = IsExpected ? 2 : 1; + SmallVector Vals(Weights.size() + Offset); Vals[0] = createString("branch_weights"); + if (IsExpected) + Vals[1] = createString("expected"); Type *Int32Ty = Type::getInt32Ty(Context); for (unsigned i = 0, e = Weights.size(); i != e; ++i) - Vals[i + 1] = createConstant(ConstantInt::get(Int32Ty, Weights[i])); + Vals[i + Offset] = createConstant(ConstantInt::get(Int32Ty, Weights[i])); return MDNode::get(Context, Vals); } diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp index b6c932495a145..5f42ce22f72fe 100644 --- a/llvm/lib/IR/Metadata.cpp +++ b/llvm/lib/IR/Metadata.cpp @@ -1196,10 +1196,10 @@ MDNode *MDNode::mergeDirectCallProfMetadata(MDNode *A, MDNode *B, StringRef AProfName = AMDS->getString(); StringRef BProfName = BMDS->getString(); if (AProfName == "branch_weights" && BProfName == "branch_weights") { - ConstantInt *AInstrWeight = - mdconst::dyn_extract(A->getOperand(1)); - ConstantInt *BInstrWeight = - mdconst::dyn_extract(B->getOperand(1)); + ConstantInt *AInstrWeight = mdconst::dyn_extract( + A->getOperand(getBranchWeightOffset(A))); + ConstantInt *BInstrWeight = mdconst::dyn_extract( + B->getOperand(getBranchWeightOffset(B))); assert(AInstrWeight && BInstrWeight && "verified by LLVM verifier"); return MDNode::get(Ctx, {MDHelper.createString("branch_weights"), diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp index 51e78dc5e6c00..c4b1ed55de8a2 100644 --- a/llvm/lib/IR/ProfDataUtils.cpp +++ b/llvm/lib/IR/ProfDataUtils.cpp @@ -40,9 +40,6 @@ namespace { // We maintain some constants here to ensure that we access the branch weights // correctly, and can change the behavior in the future if the layout changes -// The index at which the weights vector starts -constexpr unsigned WeightsIdx = 1; - // the minimum number of operands for MD_prof nodes with branch weights constexpr unsigned MinBWOps = 3; @@ -75,6 +72,7 @@ static void extractFromBranchWeightMD(const MDNode *ProfileData, assert(isBranchWeightMD(ProfileData) && "wrong metadata"); unsigned NOps = ProfileData->getNumOperands(); + unsigned WeightsIdx = getBranchWeightOffset(ProfileData); assert(WeightsIdx < NOps && "Weights Index must be less than NOps."); Weights.resize(NOps - WeightsIdx); @@ -82,8 +80,8 @@ static void extractFromBranchWeightMD(const MDNode *ProfileData, ConstantInt *Weight = mdconst::dyn_extract(ProfileData->getOperand(Idx)); assert(Weight && "Malformed branch_weight in MD_prof node"); - assert(Weight->getValue().getActiveBits() <= 32 && - "Too many bits for uint32_t"); + assert(Weight->getValue().getActiveBits() <= (sizeof(T) * 8) && + "Too many bits for MD_prof branch_weight"); Weights[Idx - WeightsIdx] = Weight->getZExtValue(); } } @@ -123,6 +121,26 @@ bool hasValidBranchWeightMD(const Instruction &I) { return getValidBranchWeightMDNode(I); } +bool hasBranchWeightOrigin(const Instruction &I) { + auto *ProfileData = I.getMetadata(LLVMContext::MD_prof); + return hasBranchWeightOrigin(ProfileData); +} + +bool hasBranchWeightOrigin(const MDNode *ProfileData) { + if (!isBranchWeightMD(ProfileData)) + return false; + auto *ProfDataName = dyn_cast(ProfileData->getOperand(1)); + // NOTE: if we ever have more types of branch weight provenance, + // we need to check the string value is "expected". For now, we + // supply a more generic API, and avoid the spurious comparisons. + assert(ProfDataName == nullptr || ProfDataName->getString() == "expected"); + return ProfDataName != nullptr; +} + +unsigned getBranchWeightOffset(const MDNode *ProfileData) { + return hasBranchWeightOrigin(ProfileData) ? 2 : 1; +} + MDNode *getBranchWeightMDNode(const Instruction &I) { auto *ProfileData = I.getMetadata(LLVMContext::MD_prof); if (!isBranchWeightMD(ProfileData)) @@ -132,7 +150,9 @@ MDNode *getBranchWeightMDNode(const Instruction &I) { MDNode *getValidBranchWeightMDNode(const Instruction &I) { auto *ProfileData = getBranchWeightMDNode(I); - if (ProfileData && ProfileData->getNumOperands() == 1 + I.getNumSuccessors()) + auto Offset = getBranchWeightOffset(ProfileData); + if (ProfileData && + ProfileData->getNumOperands() == Offset + I.getNumSuccessors()) return ProfileData; return nullptr; } @@ -191,7 +211,8 @@ bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalVal) { return false; if (ProfDataName->getString() == "branch_weights") { - for (unsigned Idx = 1; Idx < ProfileData->getNumOperands(); Idx++) { + unsigned Offset = getBranchWeightOffset(ProfileData); + for (unsigned Idx = Offset; Idx < ProfileData->getNumOperands(); ++Idx) { auto *V = mdconst::dyn_extract(ProfileData->getOperand(Idx)); assert(V && "Malformed branch_weight in MD_prof node"); TotalVal += V->getValue().getZExtValue(); @@ -212,9 +233,10 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalVal) { return extractProfTotalWeight(I.getMetadata(LLVMContext::MD_prof), TotalVal); } -void setBranchWeights(Instruction &I, ArrayRef Weights) { +void setBranchWeights(Instruction &I, ArrayRef Weights, + bool IsExpected) { MDBuilder MDB(I.getContext()); - MDNode *BranchWeights = MDB.createBranchWeights(Weights); + MDNode *BranchWeights = MDB.createBranchWeights(Weights, IsExpected); I.setMetadata(LLVMContext::MD_prof, BranchWeights); } diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 684e54444621b..e0fde2b7d90dc 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -104,6 +104,7 @@ #include "llvm/IR/Module.h" #include "llvm/IR/ModuleSlotTracker.h" #include "llvm/IR/PassManager.h" +#include "llvm/IR/ProfDataUtils.h" #include "llvm/IR/Statepoint.h" #include "llvm/IR/Type.h" #include "llvm/IR/Use.h" @@ -4808,8 +4809,10 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) { // Check consistency of !prof branch_weights metadata. if (ProfName == "branch_weights") { + unsigned int Offset = getBranchWeightOffset(MD); if (isa(&I)) { - Check(MD->getNumOperands() == 2 || MD->getNumOperands() == 3, + Check(MD->getNumOperands() == (1 + Offset) || + MD->getNumOperands() == (2 + Offset), "Wrong number of InvokeInst branch_weights operands", MD); } else { unsigned ExpectedNumOperands = 0; @@ -4829,10 +4832,10 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) { CheckFailed("!prof branch_weights are not allowed for this instruction", MD); - Check(MD->getNumOperands() == 1 + ExpectedNumOperands, + Check(MD->getNumOperands() == Offset + ExpectedNumOperands, "Wrong number of operands", MD); } - for (unsigned i = 1; i < MD->getNumOperands(); ++i) { + for (unsigned i = Offset; i < MD->getNumOperands(); ++i) { auto &MDO = MD->getOperand(i); Check(MDO, "second operand should not be null", MD); Check(mdconst::dyn_extract(MDO), diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp index 92ad4c34da6e7..7e6a8817b7a67 100644 --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -1662,7 +1662,8 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) { else if (OverwriteExistingWeights) I.setMetadata(LLVMContext::MD_prof, nullptr); } else if (!isa(&I)) { - setBranchWeights(I, {static_cast(BlockWeights[BB])}); + setBranchWeights(I, {static_cast(BlockWeights[BB])}, + /*IsExpected=*/false); } } } else if (OverwriteExistingWeights || ProfileSampleBlockAccurate) { @@ -1673,7 +1674,7 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) { if (cast(I).isIndirectCall()) { I.setMetadata(LLVMContext::MD_prof, nullptr); } else { - setBranchWeights(I, {uint32_t(0)}); + setBranchWeights(I, {uint32_t(0)}, /*IsExpected=*/false); } } } @@ -1756,7 +1757,7 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) { if (MaxWeight > 0 && (!TI->extractProfTotalWeight(TempWeight) || OverwriteExistingWeights)) { LLVM_DEBUG(dbgs() << "SUCCESS. Found non-zero weights.\n"); - setBranchWeights(*TI, Weights); + setBranchWeights(*TI, Weights, /*IsExpected=*/false); ORE->emit([&]() { return OptimizationRemark(DEBUG_TYPE, "PopularDest", MaxDestInst) << "most popular destination for conditional branches at " diff --git a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp index 0a3d8d6000cf4..731104d4fcef0 100644 --- a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp +++ b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp @@ -1878,7 +1878,7 @@ void CHR::fixupBranchesAndSelects(CHRScope *Scope, static_cast(CHRBranchBias.scale(1000)), static_cast(CHRBranchBias.getCompl().scale(1000)), }; - setBranchWeights(*MergedBR, Weights); + setBranchWeights(*MergedBR, Weights, /*IsExpected=*/false); CHR_DEBUG(dbgs() << "CHR branch bias " << Weights[0] << ":" << Weights[1] << "\n"); } diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp index 23a7c6a20aecb..6db76ca78b218 100644 --- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -259,7 +259,8 @@ CallBase &llvm::pgo::promoteIndirectCall(CallBase &CB, Function *DirectCallee, promoteCallWithIfThenElse(CB, DirectCallee, BranchWeights); if (AttachProfToDirectCall) { - setBranchWeights(NewInst, {static_cast(Count)}); + setBranchWeights(NewInst, {static_cast(Count)}, + /*IsExpected=*/false); } using namespace ore; diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index 2269c2e0fffae..ac6d3348b3db9 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -1474,7 +1474,8 @@ void PGOUseFunc::populateCoverage(IndexedInstrProfReader *PGOReader) { for (auto *Succ : successors(&BB)) Weights.push_back((Coverage[Succ] || !Coverage[&BB]) ? 1 : 0); if (Weights.size() >= 2) - llvm::setBranchWeights(*BB.getTerminator(), Weights); + llvm::setBranchWeights(*BB.getTerminator(), Weights, + /*IsExpected=*/false); } unsigned NumCorruptCoverage = 0; @@ -2260,7 +2261,7 @@ void llvm::setProfMetadata(Module *M, Instruction *TI, misexpect::checkExpectAnnotations(*TI, Weights, /*IsFrontend=*/false); - setBranchWeights(*TI, Weights); + setBranchWeights(*TI, Weights, /*IsExpected=*/false); if (EmitBranchProbability) { std::string BrCondStr = getBranchCondString(TI); if (BrCondStr.empty()) diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp index 74a8f1958dfe9..b9583836aea06 100644 --- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp +++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp @@ -231,7 +231,7 @@ static void updatePredecessorProfileMetadata(PHINode *PN, BasicBlock *BB) { Weights[0] = BP.getCompl().getNumerator(); Weights[1] = BP.getNumerator(); } - setBranchWeights(*PredBr, Weights); + setBranchWeights(*PredBr, Weights, hasBranchWeightOrigin(*PredBr)); } } @@ -2618,7 +2618,7 @@ void JumpThreadingPass::updateBlockFreqAndEdgeWeight(BasicBlock *PredBB, Weights.push_back(Prob.getNumerator()); auto TI = BB->getTerminator(); - setBranchWeights(*TI, Weights); + setBranchWeights(*TI, Weights, hasBranchWeightOrigin(*TI)); } } diff --git a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp index 6f87e4d91d2c7..17c5a4ee1fd0b 100644 --- a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp +++ b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp @@ -102,7 +102,7 @@ static bool handleSwitchExpect(SwitchInst &SI) { misexpect::checkExpectAnnotations(SI, Weights, /*IsFrontend=*/true); SI.setCondition(ArgValue); - setBranchWeights(SI, Weights); + setBranchWeights(SI, Weights, /*IsExpected=*/true); return true; } @@ -262,11 +262,13 @@ static void handlePhiDef(CallInst *Expect) { if (IsOpndComingFromSuccessor(BI->getSuccessor(1))) BI->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(LikelyBranchWeightVal, - UnlikelyBranchWeightVal)); + UnlikelyBranchWeightVal, + /*IsExpected=*/true)); else if (IsOpndComingFromSuccessor(BI->getSuccessor(0))) BI->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(UnlikelyBranchWeightVal, - LikelyBranchWeightVal)); + LikelyBranchWeightVal, + /*IsExpected=*/true)); } } @@ -331,12 +333,12 @@ template static bool handleBrSelExpect(BrSelInst &BSI) { SmallVector ExpectedWeights; if ((ExpectedValue->getZExtValue() == ValueComparedTo) == (Predicate == CmpInst::ICMP_EQ)) { - Node = - MDB.createBranchWeights(LikelyBranchWeightVal, UnlikelyBranchWeightVal); + Node = MDB.createBranchWeights( + LikelyBranchWeightVal, UnlikelyBranchWeightVal, /*IsExpected=*/true); ExpectedWeights = {LikelyBranchWeightVal, UnlikelyBranchWeightVal}; } else { - Node = - MDB.createBranchWeights(UnlikelyBranchWeightVal, LikelyBranchWeightVal); + Node = MDB.createBranchWeights(UnlikelyBranchWeightVal, + LikelyBranchWeightVal, /*IsExpected=*/true); ExpectedWeights = {UnlikelyBranchWeightVal, LikelyBranchWeightVal}; } diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp index ce0f4c7668a40..12229123675e7 100644 --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -231,7 +231,7 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions, // Remove weight for this case. std::swap(Weights[Idx + 1], Weights.back()); Weights.pop_back(); - setBranchWeights(*SI, Weights); + setBranchWeights(*SI, Weights, hasBranchWeightOrigin(MD)); } // Remove this entry. BasicBlock *ParentBB = SI->getParent(); diff --git a/llvm/lib/Transforms/Utils/LoopPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp index e2516930d251b..d517ec3d6e9f2 100644 --- a/llvm/lib/Transforms/Utils/LoopPeel.cpp +++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp @@ -680,7 +680,7 @@ struct WeightInfo { /// To avoid dealing with division rounding we can just multiple both part /// of weights to E and use weight as (F - I * E, E). static void updateBranchWeights(Instruction *Term, WeightInfo &Info) { - setBranchWeights(*Term, Info.Weights); + setBranchWeights(*Term, Info.Weights, /*IsExpected=*/false); for (auto [Idx, SubWeight] : enumerate(Info.SubWeights)) if (SubWeight != 0) // Don't set the probability of taking the edge from latch to loop header @@ -1073,7 +1073,7 @@ bool llvm::peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI, } for (const auto &[Term, Info] : Weights) { - setBranchWeights(*Term, Info.Weights); + setBranchWeights(*Term, Info.Weights, /*IsExpected=*/false); } // Update Metadata for count of peeled off iterations. diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp index 08ba65d9483e0..a02ea1c06147b 100644 --- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp @@ -390,13 +390,13 @@ static void updateBranchWeights(BranchInst &PreHeaderBI, BranchInst &LoopBI, SuccsSwapped ? LoopBackWeight : ExitWeight1, SuccsSwapped ? ExitWeight1 : LoopBackWeight, }; - setBranchWeights(LoopBI, LoopBIWeights); + setBranchWeights(LoopBI, LoopBIWeights, /*IsExpected=*/false); if (HasConditionalPreHeader) { const uint32_t PreHeaderBIWeights[] = { SuccsSwapped ? EnterWeight : ExitWeight0, SuccsSwapped ? ExitWeight0 : EnterWeight, }; - setBranchWeights(PreHeaderBI, PreHeaderBIWeights); + setBranchWeights(PreHeaderBI, PreHeaderBIWeights, /*IsExpected=*/false); } } diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index fe6ec8819ff99..4a535cf13ca87 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -861,26 +861,28 @@ static bool ValuesOverlap(std::vector &C1, // Set branch weights on SwitchInst. This sets the metadata if there is at // least one non-zero weight. -static void setBranchWeights(SwitchInst *SI, ArrayRef Weights) { +static void setBranchWeights(SwitchInst *SI, ArrayRef Weights, + bool IsExpected) { // Check that there is at least one non-zero weight. Otherwise, pass // nullptr to setMetadata which will erase the existing metadata. MDNode *N = nullptr; if (llvm::any_of(Weights, [](uint32_t W) { return W != 0; })) - N = MDBuilder(SI->getParent()->getContext()).createBranchWeights(Weights); + N = MDBuilder(SI->getParent()->getContext()) + .createBranchWeights(Weights, IsExpected); SI->setMetadata(LLVMContext::MD_prof, N); } // Similar to the above, but for branch and select instructions that take // exactly 2 weights. static void setBranchWeights(Instruction *I, uint32_t TrueWeight, - uint32_t FalseWeight) { + uint32_t FalseWeight, bool IsExpected) { assert(isa(I) || isa(I)); // Check that there is at least one non-zero weight. Otherwise, pass // nullptr to setMetadata which will erase the existing metadata. MDNode *N = nullptr; if (TrueWeight || FalseWeight) N = MDBuilder(I->getParent()->getContext()) - .createBranchWeights(TrueWeight, FalseWeight); + .createBranchWeights(TrueWeight, FalseWeight, IsExpected); I->setMetadata(LLVMContext::MD_prof, N); } @@ -1338,7 +1340,7 @@ bool SimplifyCFGOpt::PerformValueComparisonIntoPredecessorFolding( SmallVector MDWeights(Weights.begin(), Weights.end()); - setBranchWeights(NewSI, MDWeights); + setBranchWeights(NewSI, MDWeights, /*IsExpected=*/false); } EraseTerminatorAndDCECond(PTI); @@ -3831,7 +3833,7 @@ static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI, FitWeights(NewWeights); SmallVector MDWeights(NewWeights.begin(), NewWeights.end()); - setBranchWeights(PBI, MDWeights[0], MDWeights[1]); + setBranchWeights(PBI, MDWeights[0], MDWeights[1], /*IsExpected=*/false); // TODO: If BB is reachable from all paths through PredBlock, then we // could replace PBI's branch probabilities with BI's. @@ -4568,7 +4570,7 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI, // Halve the weights if any of them cannot fit in an uint32_t FitWeights(NewWeights); - setBranchWeights(PBI, NewWeights[0], NewWeights[1]); + setBranchWeights(PBI, NewWeights[0], NewWeights[1], /*IsExpected=*/false); } // OtherDest may have phi nodes. If so, add an entry from PBI's @@ -4604,7 +4606,8 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI, FitWeights(NewWeights); - setBranchWeights(NV, NewWeights[0], NewWeights[1]); + setBranchWeights(NV, NewWeights[0], NewWeights[1], + /*IsExpected=*/false); } } } @@ -4667,7 +4670,7 @@ bool SimplifyCFGOpt::SimplifyTerminatorOnSelect(Instruction *OldTerm, // Create a conditional branch sharing the condition of the select. BranchInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB); if (TrueWeight != FalseWeight) - setBranchWeights(NewBI, TrueWeight, FalseWeight); + setBranchWeights(NewBI, TrueWeight, FalseWeight, /*IsExpected=*/false); } } else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) { // Neither of the selected blocks were successors, so this @@ -5617,7 +5620,7 @@ bool SimplifyCFGOpt::TurnSwitchRangeIntoICmp(SwitchInst *SI, TrueWeight /= 2; FalseWeight /= 2; } - setBranchWeights(NewBI, TrueWeight, FalseWeight); + setBranchWeights(NewBI, TrueWeight, FalseWeight, /*IsExpected=*/false); } } diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index c7c19ef456c7c..1acecf2738b5f 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2145,7 +2145,7 @@ class GeneratedRTChecks { BranchInst &BI = *BranchInst::Create(Bypass, LoopVectorPreHeader, Cond); if (AddBranchWeights) - setBranchWeights(BI, SCEVCheckBypassWeights); + setBranchWeights(BI, SCEVCheckBypassWeights, /*IsExpected=*/false); ReplaceInstWithInst(SCEVCheckBlock->getTerminator(), &BI); return SCEVCheckBlock; } @@ -2173,7 +2173,7 @@ class GeneratedRTChecks { BranchInst &BI = *BranchInst::Create(Bypass, LoopVectorPreHeader, MemRuntimeCheckCond); if (AddBranchWeights) { - setBranchWeights(BI, MemCheckBypassWeights); + setBranchWeights(BI, MemCheckBypassWeights, /*IsExpected=*/false); } ReplaceInstWithInst(MemCheckBlock->getTerminator(), &BI); MemCheckBlock->getTerminator()->setDebugLoc( @@ -2889,7 +2889,7 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) { BranchInst &BI = *BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters); if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())) - setBranchWeights(BI, MinItersBypassWeights); + setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false); ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI); LoopBypassBlocks.push_back(TCCheckBlock); } @@ -3128,7 +3128,7 @@ BasicBlock *InnerLoopVectorizer::completeLoopSkeleton() { unsigned TripCount = UF * VF.getKnownMinValue(); assert(TripCount > 0 && "trip count should not be zero"); const uint32_t Weights[] = {1, TripCount - 1}; - setBranchWeights(BI, Weights); + setBranchWeights(BI, Weights, /*IsExpected=*/false); } } @@ -7669,7 +7669,7 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass, BranchInst &BI = *BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters); if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())) - setBranchWeights(BI, MinItersBypassWeights); + setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false); ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI); return TCCheckBlock; @@ -7826,7 +7826,7 @@ EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck( unsigned EstimatedSkipCount = std::min(MainLoopStep, EpilogueLoopStep); const uint32_t Weights[] = {EstimatedSkipCount, MainLoopStep - EstimatedSkipCount}; - setBranchWeights(BI, Weights); + setBranchWeights(BI, Weights, /*IsExpected=*/false); } ReplaceInstWithInst(Insert->getTerminator(), &BI); diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll b/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll index 0abca5b383224..8e06cd57a1001 100644 --- a/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll +++ b/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll @@ -284,7 +284,7 @@ define i32 @test10(i64 %t6) { declare i1 @llvm.expect.i1(i1, i1) nounwind readnone -; CHECK: !0 = !{!"branch_weights", i32 2000, i32 1} -; CHECK: !1 = !{!"branch_weights", i32 1, i32 2000} -; CHECK: !2 = !{!"branch_weights", i32 1, i32 1, i32 2000} -; CHECK: !3 = !{!"branch_weights", i32 2000, i32 1, i32 1} +; CHECK: !0 = !{!"branch_weights", !"expected", i32 2000, i32 1} +; CHECK: !1 = !{!"branch_weights", !"expected", i32 1, i32 2000} +; CHECK: !2 = !{!"branch_weights", !"expected", i32 1, i32 1, i32 2000} +; CHECK: !3 = !{!"branch_weights", !"expected", i32 2000, i32 1, i32 1} diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll b/llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll index 64293557050c1..40571278ca93f 100644 --- a/llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll +++ b/llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll @@ -284,7 +284,7 @@ define i32 @test10(i64 %t6) { declare i1 @llvm.expect.with.probability.i1(i1, i1, double) nounwind readnone -; CHECK: !0 = !{!"branch_weights", i32 1717986918, i32 429496731} -; CHECK: !1 = !{!"branch_weights", i32 429496731, i32 1717986918} -; CHECK: !2 = !{!"branch_weights", i32 214748366, i32 214748366, i32 1717986918} -; CHECK: !3 = !{!"branch_weights", i32 1717986918, i32 214748366, i32 214748366} +; CHECK: !0 = !{!"branch_weights", !"expected", i32 1717986918, i32 429496731} +; CHECK: !1 = !{!"branch_weights", !"expected", i32 429496731, i32 1717986918} +; CHECK: !2 = !{!"branch_weights", !"expected", i32 214748366, i32 214748366, i32 1717986918} +; CHECK: !3 = !{!"branch_weights", !"expected", i32 1717986918, i32 214748366, i32 214748366} diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll b/llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll index 2bcfb1e064be9..458a7758fa970 100644 --- a/llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll +++ b/llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll @@ -99,6 +99,5 @@ attributes #1 = { nounwind readnone } !0 = !{i32 1, !"wchar_size", i32 4} !1 = !{!"clang version 5.0.0 (trunk 304373)"} -; CHECK: [[LIKELY]] = !{!"branch_weights", i32 2000, i32 1} -; CHECK: [[UNLIKELY]] = !{!"branch_weights", i32 1, i32 2000} - +; CHECK: [[LIKELY]] = !{!"branch_weights", !"expected", i32 2000, i32 1} +; CHECK: [[UNLIKELY]] = !{!"branch_weights", !"expected", i32 1, i32 2000} diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll index 32ae9b0b2f15c..9b9d9a746dd32 100644 --- a/llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll +++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll @@ -352,5 +352,5 @@ declare i64 @llvm.expect.i64(i64, i64) !llvm.ident = !{!0} !0 = !{!"clang version 5.0.0 (trunk 302965)"} -; CHECK: [[WEIGHT]] = !{!"branch_weights", i32 2000, i32 1} -; CHECK: [[WEIGHT2]] = !{!"branch_weights", i32 1, i32 2000} +; CHECK: [[WEIGHT]] = !{!"branch_weights", !"expected", i32 2000, i32 1} +; CHECK: [[WEIGHT2]] = !{!"branch_weights", !"expected", i32 1, i32 2000} diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll index 1efa63241c2c0..e9a843225993a 100644 --- a/llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll +++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll @@ -99,5 +99,5 @@ declare i64 @llvm.expect.i64(i64, i64) !0 = !{!"clang version 5.0.0 (trunk 302965)"} -; CHECK: [[WEIGHT]] = !{!"branch_weights", i32 2000, i32 1} -; CHECK: [[WEIGHT2]] = !{!"branch_weights", i32 1, i32 2000} +; CHECK: [[WEIGHT]] = !{!"branch_weights", !"expected", i32 2000, i32 1} +; CHECK: [[WEIGHT2]] = !{!"branch_weights", !"expected", i32 1, i32 2000} diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll index 9cbaca8d13dc0..13db2c394bab2 100644 --- a/llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll +++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll @@ -53,4 +53,4 @@ declare i64 @llvm.expect.i64(i64, i64) !0 = !{!"clang version 5.0.0 (trunk 302965)"} -; CHECK: [[WEIGHT]] = !{!"branch_weights", i32 1, i32 2000} +; CHECK: [[WEIGHT]] = !{!"branch_weights", !"expected", i32 1, i32 2000} diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll index 2bad66343b761..275731d618895 100644 --- a/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll +++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll @@ -235,5 +235,5 @@ block5: ret void } -; CHECK: !0 = !{!"branch_weights", i32 2147483647, i32 1} -; CHECK: !1 = !{!"branch_weights", i32 1, i32 2147483647} +; CHECK: !0 = !{!"branch_weights", !"expected", i32 2147483647, i32 1} +; CHECK: !1 = !{!"branch_weights", !"expected", i32 1, i32 2147483647} diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll index 55dd28b70170b..a090fe20e1d93 100644 --- a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll +++ b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll @@ -293,7 +293,7 @@ declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} ; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} ; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]} -; CHECK: [[PROF4]] = !{!"branch_weights", i32 2000, i32 1} +; CHECK: [[PROF4]] = !{!"branch_weights", !"expected", i32 2000, i32 1} ; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]} ; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]} ;.