diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index e2dc0b1e38168..cdeff1e261d42 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -1627,14 +1627,31 @@ class CodeGenFunction : public CodeGenTypeCache { } void markStmtMaybeUsed(const Stmt *S) { PGO.markStmtMaybeUsed(S); } + enum CounterForIncrement { + UseExecPath = 0, + UseSkipPath, + }; + /// Increment the profiler's counter for the given statement by \p StepV. /// If \p StepV is null, the default increment is 1. void incrementProfileCounter(const Stmt *S, llvm::Value *StepV = nullptr) { + incrementProfileCounter(UseExecPath, S, false, StepV); + } + + /// Emit increment of Counter. + /// \param ExecSkip Use `Skipped` Counter if UseSkipPath is specified. + /// \param S The Stmt that Counter is associated. + /// \param UseBoth Mark both Exec/Skip as used. (for verification) + /// \param StepV The offset Value for adding to Counter. + void incrementProfileCounter(CounterForIncrement ExecSkip, const Stmt *S, + bool UseBoth = false, + llvm::Value *StepV = nullptr) { if (CGM.getCodeGenOpts().hasProfileClangInstr() && !CurFn->hasFnAttribute(llvm::Attribute::NoProfile) && !CurFn->hasFnAttribute(llvm::Attribute::SkipProfile)) { auto AL = ApplyDebugLocation::CreateArtificial(*this); - PGO.emitCounterSetOrIncrement(Builder, S, StepV); + PGO.emitCounterSetOrIncrement(Builder, S, (ExecSkip == UseSkipPath), + UseBoth, StepV); } PGO.setCurrentStmt(S); } diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp index 792373839107f..ffb1df9ec2cc6 100644 --- a/clang/lib/CodeGen/CodeGenPGO.cpp +++ b/clang/lib/CodeGen/CodeGenPGO.cpp @@ -1137,6 +1137,16 @@ void CodeGenPGO::emitCounterRegionMapping(const Decl *D) { if (CoverageMapping.empty()) return; + // Scan max(FalseCnt) and update NumRegionCounters. + unsigned MaxNumCounters = NumRegionCounters; + for (const auto [_, V] : *RegionCounterMap) { + assert((!V.Executed.hasValue() || MaxNumCounters > V.Executed) && + "TrueCnt should not be reassigned"); + if (V.Skipped.hasValue()) + MaxNumCounters = std::max(MaxNumCounters, V.Skipped + 1); + } + NumRegionCounters = MaxNumCounters; + CGM.getCoverageMapping()->addFunctionMappingRecord( FuncNameVar, FuncName, FunctionHash, CoverageMapping); } @@ -1197,11 +1207,25 @@ std::pair CodeGenPGO::getIsCounterPair(const Stmt *S) const { } void CodeGenPGO::emitCounterSetOrIncrement(CGBuilderTy &Builder, const Stmt *S, + bool UseSkipPath, bool UseBoth, llvm::Value *StepV) { - if (!RegionCounterMap || !Builder.GetInsertBlock()) + if (!RegionCounterMap) return; - unsigned Counter = (*RegionCounterMap)[S].Executed; + unsigned Counter; + auto &TheMap = (*RegionCounterMap)[S]; + if (!UseSkipPath) { + if (!TheMap.Executed.hasValue()) + return; + Counter = TheMap.Executed; + } else { + if (!TheMap.Skipped.hasValue()) + return; + Counter = TheMap.Skipped; + } + + if (!Builder.GetInsertBlock()) + return; // Make sure that pointer to global is passed in with zero addrspace // This is relevant during GPU profiling diff --git a/clang/lib/CodeGen/CodeGenPGO.h b/clang/lib/CodeGen/CodeGenPGO.h index 1944b640951d5..da386f41c0aea 100644 --- a/clang/lib/CodeGen/CodeGenPGO.h +++ b/clang/lib/CodeGen/CodeGenPGO.h @@ -112,6 +112,7 @@ class CodeGenPGO { public: std::pair getIsCounterPair(const Stmt *S) const; void emitCounterSetOrIncrement(CGBuilderTy &Builder, const Stmt *S, + bool UseFalsePath, bool UseBoth, llvm::Value *StepV); void emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, const Expr *S, Address MCDCCondBitmapAddr, diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp index f09157771d2b5..ce39b7220fc85 100644 --- a/clang/lib/CodeGen/CoverageMappingGen.cpp +++ b/clang/lib/CodeGen/CoverageMappingGen.cpp @@ -884,6 +884,9 @@ struct CounterCoverageMappingBuilder /// The map of statements to count values. llvm::DenseMap &CounterMap; + CounterExpressionBuilder::SubstMap MapToExpand; + unsigned NextCounterNum; + MCDC::State &MCDCState; /// A stack of currently live regions. @@ -958,7 +961,8 @@ struct CounterCoverageMappingBuilder BranchCounterPair getBranchCounterPair(const Stmt *S, Counter ParentCnt, std::optional SkipCntForOld = std::nullopt) { - Counter ExecCnt = getRegionCounter(S); + auto &TheMap = CounterMap[S]; + auto ExecCnt = Counter::getCounter(TheMap.Executed); // The old behavior of SingleByte is unaware of Branches. // Will be pruned after the migration of SingleByte. @@ -968,13 +972,44 @@ struct CounterCoverageMappingBuilder return {ExecCnt, *SkipCntForOld}; } - return {ExecCnt, Builder.subtract(ParentCnt, ExecCnt)}; + BranchCounterPair Counters = {ExecCnt, + Builder.subtract(ParentCnt, ExecCnt)}; + + if (!llvm::EnableSingleByteCoverage || !Counters.Skipped.isExpression()) { + assert( + !TheMap.Skipped.hasValue() && + "SkipCnt shouldn't be allocated but refer to an existing counter."); + return Counters; + } + + // Assign second if second is not assigned yet. + if (!TheMap.Skipped.hasValue()) + TheMap.Skipped = NextCounterNum++; + + // Replace an expression (ParentCnt - ExecCnt) with SkipCnt. + Counter SkipCnt = Counter::getCounter(TheMap.Skipped); + MapToExpand[SkipCnt] = Builder.subst(Counters.Skipped, MapToExpand); + Counters.Skipped = SkipCnt; + return Counters; } bool IsCounterEqual(Counter OutCount, Counter ParentCount) { if (OutCount == ParentCount) return true; + // Try comaparison with pre-replaced expressions. + // + // For example, getBranchCounterPair(#0) returns {#1, #0 - #1}. + // The sum of the pair should be equivalent to the Parent, #0. + // OTOH when (#0 - #1) is replaced with the new counter #2, + // The sum is (#1 + #2). If the reverse substitution #2 => (#0 - #1) + // can be applied, the sum can be transformed to (#1 + (#0 - #1)). + // To apply substitutions to both hand expressions, transform (LHS - RHS) + // and check isZero. + if (Builder.subst(Builder.subtract(OutCount, ParentCount), MapToExpand) + .isZero()) + return true; + return false; } @@ -1188,12 +1223,14 @@ struct CounterCoverageMappingBuilder /// and add it to the function's SourceRegions. /// Returns Counter that corresponds to SC. Counter createSwitchCaseRegion(const SwitchCase *SC, Counter ParentCount) { + Counter TrueCnt = getRegionCounter(SC); + Counter FalseCnt = (llvm::EnableSingleByteCoverage + ? Counter::getZero() // Folded + : subtractCounters(ParentCount, TrueCnt)); // Push region onto RegionStack but immediately pop it (which adds it to // the function's SourceRegions) because it doesn't apply to any other // source other than the SwitchCase. - Counter TrueCnt = getRegionCounter(SC); - popRegions(pushRegion(TrueCnt, getStart(SC), SC->getColonLoc(), - subtractCounters(ParentCount, TrueCnt))); + popRegions(pushRegion(TrueCnt, getStart(SC), SC->getColonLoc(), FalseCnt)); return TrueCnt; } @@ -1460,7 +1497,8 @@ struct CounterCoverageMappingBuilder llvm::DenseMap &CounterMap, MCDC::State &MCDCState, SourceManager &SM, const LangOptions &LangOpts) : CoverageMappingBuilder(CVM, SM, LangOpts), CounterMap(CounterMap), - MCDCState(MCDCState), MCDCBuilder(CVM.getCodeGenModule(), MCDCState) {} + NextCounterNum(CounterMap.size()), MCDCState(MCDCState), + MCDCBuilder(CVM.getCodeGenModule(), MCDCState) {} /// Write the mapping data to the output stream void write(llvm::raw_ostream &OS) { diff --git a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp index 6d6678e9e4afe..83fe5f071c62a 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp @@ -637,6 +637,9 @@ static unsigned getMaxCounterID(const CounterMappingContext &Ctx, unsigned MaxCounterID = 0; for (const auto &Region : Record.MappingRegions) { MaxCounterID = std::max(MaxCounterID, Ctx.getMaxCounterID(Region.Count)); + if (Region.isBranch()) + MaxCounterID = + std::max(MaxCounterID, Ctx.getMaxCounterID(Region.FalseCount)); } return MaxCounterID; }