diff --git a/llvm/include/llvm/Transforms/IPO/SampleProfile.h b/llvm/include/llvm/Transforms/IPO/SampleProfile.h index e94f6ba55cd0d..6f4f9701bae93 100644 --- a/llvm/include/llvm/Transforms/IPO/SampleProfile.h +++ b/llvm/include/llvm/Transforms/IPO/SampleProfile.h @@ -41,7 +41,9 @@ class SampleProfileLoaderPass : public PassInfoMixin { SampleProfileLoaderPass( std::string File = "", std::string RemappingFile = "", ThinOrFullLTOPhase LTOPhase = ThinOrFullLTOPhase::None, - IntrusiveRefCntPtr FS = nullptr); + IntrusiveRefCntPtr FS = nullptr, + bool DisableSampleProfileInlining = false, + bool UseFlattenedProfile = false); PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); @@ -50,6 +52,8 @@ class SampleProfileLoaderPass : public PassInfoMixin { std::string ProfileRemappingFileName; const ThinOrFullLTOPhase LTOPhase; IntrusiveRefCntPtr FS; + bool DisableSampleProfileInlining; + bool UseFlattenedProfile; }; } // end namespace llvm diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index af99ce5cd835b..112ec31d9729f 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -2162,6 +2162,19 @@ PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level, if (PGOOpt && PGOOpt->DebugInfoForProfiling) MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); + if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) { + // Explicitly disable sample loader inlining and use flattened profile in O0 + // pipeline. + MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile, + PGOOpt->ProfileRemappingFile, + ThinOrFullLTOPhase::None, nullptr, + /*DisableSampleProfileInlining=*/true, + /*UseFlattenedProfile=*/true)); + // Cache ProfileSummaryAnalysis once to avoid the potential need to insert + // RequireAnalysisPass for PSI before subsequent non-module passes. + MPM.addPass(RequireAnalysisPass()); + } + invokePipelineEarlySimplificationEPCallbacks(MPM, Level, Phase); // Build a minimal pipeline based on the semantics required by LLVM, diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp index 3dd06626a2d4c..b2fa66f2a6d37 100644 --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -469,7 +469,8 @@ class SampleProfileLoader final : public SampleProfileLoaderBaseImpl { std::function GetAssumptionCache, std::function GetTargetTransformInfo, std::function GetTLI, - LazyCallGraph &CG) + LazyCallGraph &CG, bool DisableSampleProfileInlining, + bool UseFlattenedProfile) : SampleProfileLoaderBaseImpl(std::string(Name), std::string(RemapName), std::move(FS)), GetAC(std::move(GetAssumptionCache)), @@ -478,7 +479,9 @@ class SampleProfileLoader final : public SampleProfileLoaderBaseImpl { AnnotatedPassName(AnnotateSampleProfileInlinePhase ? llvm::AnnotateInlinePassName(InlineContext{ LTOPhase, InlinePass::SampleProfileInliner}) - : CSINLINE_DEBUG) {} + : CSINLINE_DEBUG), + DisableSampleProfileInlining(DisableSampleProfileInlining), + UseFlattenedProfile(UseFlattenedProfile) {} bool doInitialization(Module &M, FunctionAnalysisManager *FAM = nullptr); bool runOnModule(Module &M, ModuleAnalysisManager *AM, @@ -592,6 +595,10 @@ class SampleProfileLoader final : public SampleProfileLoaderBaseImpl { // attribute. bool ProfAccForSymsInList; + bool DisableSampleProfileInlining; + + bool UseFlattenedProfile; + // External inline advisor used to replay inline decision from remarks. std::unique_ptr ExternalInlineAdvisor; @@ -919,7 +926,7 @@ bool SampleProfileLoader::tryPromoteAndInlineCandidate( Function &F, InlineCandidate &Candidate, uint64_t SumOrigin, uint64_t &Sum, SmallVector *InlinedCallSite) { // Bail out early if sample-loader inliner is disabled. - if (DisableSampleLoaderInlining) + if (DisableSampleProfileInlining) return false; // Bail out early if MaxNumPromotions is zero. @@ -1230,7 +1237,7 @@ bool SampleProfileLoader::tryInlineCandidate( InlineCandidate &Candidate, SmallVector *InlinedCallSites) { // Do not attempt to inline a candidate if // --disable-sample-loader-inlining is true. - if (DisableSampleLoaderInlining) + if (DisableSampleProfileInlining) return false; CallBase &CB = *Candidate.CallInstr; @@ -1974,6 +1981,13 @@ bool SampleProfileLoader::doInitialization(Module &M, PSL = Reader->getProfileSymbolList(); + if (DisableSampleLoaderInlining.getNumOccurrences()) + DisableSampleProfileInlining = DisableSampleLoaderInlining; + + if (UseFlattenedProfile) + ProfileConverter::flattenProfile(Reader->getProfiles(), + Reader->profileIsCS()); + // While profile-sample-accurate is on, ignore symbol list. ProfAccForSymsInList = ProfileAccurateForSymsInList && PSL && !ProfileSampleAccurate; @@ -2304,9 +2318,12 @@ bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM) } SampleProfileLoaderPass::SampleProfileLoaderPass( std::string File, std::string RemappingFile, ThinOrFullLTOPhase LTOPhase, - IntrusiveRefCntPtr FS) + IntrusiveRefCntPtr FS, bool DisableSampleProfileInlining, + bool UseFlattenedProfile) : ProfileFileName(File), ProfileRemappingFileName(RemappingFile), - LTOPhase(LTOPhase), FS(std::move(FS)) {} + LTOPhase(LTOPhase), FS(std::move(FS)), + DisableSampleProfileInlining(DisableSampleProfileInlining), + UseFlattenedProfile(UseFlattenedProfile) {} PreservedAnalyses SampleProfileLoaderPass::run(Module &M, ModuleAnalysisManager &AM) { @@ -2331,7 +2348,8 @@ PreservedAnalyses SampleProfileLoaderPass::run(Module &M, ProfileFileName.empty() ? SampleProfileFile : ProfileFileName, ProfileRemappingFileName.empty() ? SampleProfileRemappingFile : ProfileRemappingFileName, - LTOPhase, FS, GetAssumptionCache, GetTTI, GetTLI, CG); + LTOPhase, FS, GetAssumptionCache, GetTTI, GetTLI, CG, + DisableSampleProfileInlining, UseFlattenedProfile); if (!SampleLoader.doInitialization(M, &FAM)) return PreservedAnalyses::all(); diff --git a/llvm/test/Other/new-pm-pgo-O0.ll b/llvm/test/Other/new-pm-pgo-O0.ll index d7a6a03b8e44e..d4f662fb25ace 100644 --- a/llvm/test/Other/new-pm-pgo-O0.ll +++ b/llvm/test/Other/new-pm-pgo-O0.ll @@ -9,8 +9,9 @@ ; RUN: |FileCheck %s --check-prefixes=USE_POST_LINK,USE ; RUN: opt -debug-pass-manager -passes='lto' -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' %s 2>&1 \ ; RUN: |FileCheck %s --check-prefixes=USE_POST_LINK,USE +; RUN: opt -debug-pass-manager -passes='default' -pgo-kind=pgo-sample-use-pipeline -profile-file='%S/Inputs/new-pm-pgo.prof' %s 2>&1 \ +; RUN: |FileCheck %s --check-prefixes=SAMPLE_USE -; ; GEN: Running pass: PGOInstrumentationGen ; USE_DEFAULT: Running pass: PGOInstrumentationUse ; USE_PRE_LINK: Running pass: PGOInstrumentationUse @@ -18,6 +19,9 @@ ; USE-NOT: Running pass: PGOIndirectCallPromotion ; USE-NOT: Running pass: PGOMemOPSizeOpt +; SAMPLE_USE: Running pass: AddDiscriminatorsPass +; SAMPLE_USE: Running pass: SampleProfileLoaderPass + define void @foo() { ret void }