diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h index a1382a5e8e403..7f82c5fea7a11 100644 --- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h +++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h @@ -28,6 +28,7 @@ #include "llvm/CodeGen/ExpandMemCmp.h" #include "llvm/CodeGen/ExpandReductions.h" #include "llvm/CodeGen/GCMetadata.h" +#include "llvm/CodeGen/GlobalMerge.h" #include "llvm/CodeGen/IndirectBrExpand.h" #include "llvm/CodeGen/InterleavedAccess.h" #include "llvm/CodeGen/InterleavedLoadCombine.h" @@ -290,6 +291,9 @@ template class CodeGenPassBuilder { inconvertibleErrorCode()); } + /// Target can override this to add GlobalMergePass before all IR passes. + void addGlobalMergePass(AddIRPass &) const {} + /// Add passes that optimize instruction level parallelism for out-of-order /// targets. These passes are run while the machine code is still in SSA /// form, so they can use MachineTraceMetrics to control their heuristics. @@ -603,6 +607,7 @@ CodeGenPassBuilder::getPassNameFromLegacyName(StringRef Name) const { template void CodeGenPassBuilder::addISelPasses(AddIRPass &addPass) const { + derived().addGlobalMergePass(addPass); if (TM.useEmulatedTLS()) addPass(LowerEmuTLSPass()); diff --git a/llvm/include/llvm/CodeGen/GlobalMerge.h b/llvm/include/llvm/CodeGen/GlobalMerge.h new file mode 100644 index 0000000000000..6b3766ab9e024 --- /dev/null +++ b/llvm/include/llvm/CodeGen/GlobalMerge.h @@ -0,0 +1,50 @@ +//===- llvm/CodeGen/GlobalMerge.h -------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_GLOBALMERGE_H +#define LLVM_CODEGEN_GLOBALMERGE_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class TargetMachine; + +struct GlobalMergeOptions { + // FIXME: Infer the maximum possible offset depending on the actual users + // (these max offsets are different for the users inside Thumb or ARM + // functions), see the code that passes in the offset in the ARM backend + // for more information. + unsigned MaxOffset = 0; + bool GroupByUse = true; + bool IgnoreSingleUse = true; + bool MergeConst = false; + /// Whether we should merge global variables that have external linkage. + bool MergeExternal = true; + /// Whether we should try to optimize for size only. + /// Currently, this applies a dead simple heuristic: only consider globals + /// used in minsize functions for merging. + /// FIXME: This could learn about optsize, and be used in the cost model. + bool SizeOnly = false; +}; + +// FIXME: This pass must run before AsmPrinterPass::doInitialization! +class GlobalMergePass : public PassInfoMixin { + const TargetMachine *TM; + GlobalMergeOptions Options; + +public: + GlobalMergePass(const TargetMachine *TM, GlobalMergeOptions Options) + : TM(TM), Options(Options) {} + + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); +}; + +} // namespace llvm + +#endif // LLVM_CODEGEN_GLOBALMERGE_H diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def index cbfd4327da6e6..ebc3ac20ffb03 100644 --- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def +++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def @@ -23,9 +23,10 @@ MODULE_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis, (PIC)) #ifndef MODULE_PASS #define MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) #endif -MODULE_PASS("pre-isel-intrinsic-lowering", PreISelIntrinsicLoweringPass, ()) +MODULE_PASS("global-merge", GlobalMergePass, (TM, GlobalMergeOptions())) MODULE_PASS("jmc-instrumenter", JMCInstrumenterPass, ()) MODULE_PASS("lower-emutls", LowerEmuTLSPass, ()) +MODULE_PASS("pre-isel-intrinsic-lowering", PreISelIntrinsicLoweringPass, ()) MODULE_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ()) #undef MODULE_PASS diff --git a/llvm/lib/CodeGen/GlobalMerge.cpp b/llvm/lib/CodeGen/GlobalMerge.cpp index 22b6d31d06349..a2b5cbf7bad9f 100644 --- a/llvm/lib/CodeGen/GlobalMerge.cpp +++ b/llvm/lib/CodeGen/GlobalMerge.cpp @@ -60,6 +60,7 @@ // // ===---------------------------------------------------------------------===// +#include "llvm/CodeGen/GlobalMerge.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SetVector.h" @@ -137,88 +138,99 @@ STATISTIC(NumMerged, "Number of globals merged"); namespace { - class GlobalMerge : public FunctionPass { - const TargetMachine *TM = nullptr; - - // FIXME: Infer the maximum possible offset depending on the actual users - // (these max offsets are different for the users inside Thumb or ARM - // functions), see the code that passes in the offset in the ARM backend - // for more information. - unsigned MaxOffset; - - /// Whether we should try to optimize for size only. - /// Currently, this applies a dead simple heuristic: only consider globals - /// used in minsize functions for merging. - /// FIXME: This could learn about optsize, and be used in the cost model. - bool OnlyOptimizeForSize = false; - - /// Whether we should merge global variables that have external linkage. - bool MergeExternalGlobals = false; +class GlobalMergeImpl { + const TargetMachine *TM = nullptr; + GlobalMergeOptions Opt; + bool IsMachO = false; + +private: + bool doMerge(SmallVectorImpl &Globals, Module &M, + bool isConst, unsigned AddrSpace) const; + + /// Merge everything in \p Globals for which the corresponding bit + /// in \p GlobalSet is set. + bool doMerge(const SmallVectorImpl &Globals, + const BitVector &GlobalSet, Module &M, bool isConst, + unsigned AddrSpace) const; + + /// Check if the given variable has been identified as must keep + /// \pre setMustKeepGlobalVariables must have been called on the Module that + /// contains GV + bool isMustKeepGlobalVariable(const GlobalVariable *GV) const { + return MustKeepGlobalVariables.count(GV); + } - bool IsMachO = false; + /// Collect every variables marked as "used" or used in a landing pad + /// instruction for this Module. + void setMustKeepGlobalVariables(Module &M); - bool doMerge(SmallVectorImpl &Globals, - Module &M, bool isConst, unsigned AddrSpace) const; + /// Collect every variables marked as "used" + void collectUsedGlobalVariables(Module &M, StringRef Name); - /// Merge everything in \p Globals for which the corresponding bit - /// in \p GlobalSet is set. - bool doMerge(const SmallVectorImpl &Globals, - const BitVector &GlobalSet, Module &M, bool isConst, - unsigned AddrSpace) const; + /// Keep track of the GlobalVariable that must not be merged away + SmallSetVector MustKeepGlobalVariables; - /// Check if the given variable has been identified as must keep - /// \pre setMustKeepGlobalVariables must have been called on the Module that - /// contains GV - bool isMustKeepGlobalVariable(const GlobalVariable *GV) const { - return MustKeepGlobalVariables.count(GV); - } +public: + GlobalMergeImpl(const TargetMachine *TM, GlobalMergeOptions Opt) + : TM(TM), Opt(Opt) {} + bool run(Module &M); +}; - /// Collect every variables marked as "used" or used in a landing pad - /// instruction for this Module. - void setMustKeepGlobalVariables(Module &M); +class GlobalMerge : public FunctionPass { + const TargetMachine *TM = nullptr; + GlobalMergeOptions Opt; - /// Collect every variables marked as "used" - void collectUsedGlobalVariables(Module &M, StringRef Name); +public: + static char ID; // Pass identification, replacement for typeid. - /// Keep track of the GlobalVariable that must not be merged away - SmallSetVector MustKeepGlobalVariables; + explicit GlobalMerge() : FunctionPass(ID) { + Opt.MaxOffset = GlobalMergeMaxOffset; + initializeGlobalMergePass(*PassRegistry::getPassRegistry()); + } - public: - static char ID; // Pass identification, replacement for typeid. + explicit GlobalMerge(const TargetMachine *TM, unsigned MaximalOffset, + bool OnlyOptimizeForSize, bool MergeExternalGlobals) + : FunctionPass(ID), TM(TM) { + Opt.MaxOffset = MaximalOffset; + Opt.SizeOnly = OnlyOptimizeForSize; + Opt.MergeExternal = MergeExternalGlobals; + initializeGlobalMergePass(*PassRegistry::getPassRegistry()); + } - explicit GlobalMerge() - : FunctionPass(ID), MaxOffset(GlobalMergeMaxOffset) { - initializeGlobalMergePass(*PassRegistry::getPassRegistry()); - } + bool doInitialization(Module &M) override { + GlobalMergeImpl P(TM, Opt); + return P.run(M); + } + bool runOnFunction(Function &F) override { return false; } - explicit GlobalMerge(const TargetMachine *TM, unsigned MaximalOffset, - bool OnlyOptimizeForSize, bool MergeExternalGlobals) - : FunctionPass(ID), TM(TM), MaxOffset(MaximalOffset), - OnlyOptimizeForSize(OnlyOptimizeForSize), - MergeExternalGlobals(MergeExternalGlobals) { - initializeGlobalMergePass(*PassRegistry::getPassRegistry()); - } + StringRef getPassName() const override { return "Merge internal globals"; } - bool doInitialization(Module &M) override; - bool runOnFunction(Function &F) override; - bool doFinalization(Module &M) override; + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + FunctionPass::getAnalysisUsage(AU); + } +}; - StringRef getPassName() const override { return "Merge internal globals"; } +} // end anonymous namespace - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.setPreservesCFG(); - FunctionPass::getAnalysisUsage(AU); - } - }; +PreservedAnalyses GlobalMergePass::run(Module &M, ModuleAnalysisManager &) { + GlobalMergeImpl P(TM, Options); + bool Changed = P.run(M); + if (!Changed) + return PreservedAnalyses::all(); -} // end anonymous namespace + PreservedAnalyses PA; + PA.preserveSet(); + return PA; +} char GlobalMerge::ID = 0; INITIALIZE_PASS(GlobalMerge, DEBUG_TYPE, "Merge global variables", false, false) -bool GlobalMerge::doMerge(SmallVectorImpl &Globals, - Module &M, bool isConst, unsigned AddrSpace) const { +bool GlobalMergeImpl::doMerge(SmallVectorImpl &Globals, + Module &M, bool isConst, + unsigned AddrSpace) const { auto &DL = M.getDataLayout(); // FIXME: Find better heuristics llvm::stable_sort( @@ -333,7 +345,7 @@ bool GlobalMerge::doMerge(SmallVectorImpl &Globals, Function *ParentFn = I->getParent()->getParent(); // If we're only optimizing for size, ignore non-minsize functions. - if (OnlyOptimizeForSize && !ParentFn->hasMinSize()) + if (Opt.SizeOnly && !ParentFn->hasMinSize()) continue; size_t UGSIdx = GlobalUsesByFunction[ParentFn]; @@ -434,9 +446,9 @@ bool GlobalMerge::doMerge(SmallVectorImpl &Globals, return Changed; } -bool GlobalMerge::doMerge(const SmallVectorImpl &Globals, - const BitVector &GlobalSet, Module &M, bool isConst, - unsigned AddrSpace) const { +bool GlobalMergeImpl::doMerge(const SmallVectorImpl &Globals, + const BitVector &GlobalSet, Module &M, + bool isConst, unsigned AddrSpace) const { assert(Globals.size() > 1); Type *Int32Ty = Type::getInt32Ty(M.getContext()); @@ -467,7 +479,7 @@ bool GlobalMerge::doMerge(const SmallVectorImpl &Globals, unsigned Padding = alignTo(MergedSize, Alignment) - MergedSize; MergedSize += Padding; MergedSize += DL.getTypeAllocSize(Ty); - if (MergedSize > MaxOffset) { + if (MergedSize > Opt.MaxOffset) { break; } if (Padding) { @@ -563,7 +575,7 @@ bool GlobalMerge::doMerge(const SmallVectorImpl &Globals, return Changed; } -void GlobalMerge::collectUsedGlobalVariables(Module &M, StringRef Name) { +void GlobalMergeImpl::collectUsedGlobalVariables(Module &M, StringRef Name) { // Extract global variables from llvm.used array const GlobalVariable *GV = M.getGlobalVariable(Name); if (!GV || !GV->hasInitializer()) return; @@ -577,7 +589,7 @@ void GlobalMerge::collectUsedGlobalVariables(Module &M, StringRef Name) { MustKeepGlobalVariables.insert(G); } -void GlobalMerge::setMustKeepGlobalVariables(Module &M) { +void GlobalMergeImpl::setMustKeepGlobalVariables(Module &M) { collectUsedGlobalVariables(M, "llvm.used"); collectUsedGlobalVariables(M, "llvm.compiler.used"); @@ -604,7 +616,7 @@ void GlobalMerge::setMustKeepGlobalVariables(Module &M) { } } -bool GlobalMerge::doInitialization(Module &M) { +bool GlobalMergeImpl::run(Module &M) { if (!EnableGlobalMerge) return false; @@ -632,7 +644,7 @@ bool GlobalMerge::doInitialization(Module &M) { if (TM && !TM->shouldAssumeDSOLocal(M, &GV)) continue; - if (!(MergeExternalGlobals && GV.hasExternalLinkage()) && + if (!(Opt.MergeExternal && GV.hasExternalLinkage()) && !GV.hasInternalLinkage()) continue; @@ -659,7 +671,7 @@ bool GlobalMerge::doInitialization(Module &M) { continue; Type *Ty = GV.getValueType(); - if (DL.getTypeAllocSize(Ty) < MaxOffset) { + if (DL.getTypeAllocSize(Ty) < Opt.MaxOffset) { if (TM && TargetLoweringObjectFile::getKindForGlobal(&GV, *TM).isBSS()) BSSGlobals[{AddressSpace, Section}].push_back(&GV); @@ -686,15 +698,6 @@ bool GlobalMerge::doInitialization(Module &M) { return Changed; } -bool GlobalMerge::runOnFunction(Function &F) { - return false; -} - -bool GlobalMerge::doFinalization(Module &M) { - MustKeepGlobalVariables.clear(); - return false; -} - Pass *llvm::createGlobalMergePass(const TargetMachine *TM, unsigned Offset, bool OnlyOptimizeForSize, bool MergeExternalByDefault) { diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 27bfe12127cc2..c3acfcdfe5ab6 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -80,6 +80,7 @@ #include "llvm/CodeGen/ExpandLargeFpConvert.h" #include "llvm/CodeGen/ExpandMemCmp.h" #include "llvm/CodeGen/GCMetadata.h" +#include "llvm/CodeGen/GlobalMerge.h" #include "llvm/CodeGen/HardwareLoops.h" #include "llvm/CodeGen/IndirectBrExpand.h" #include "llvm/CodeGen/InterleavedAccess.h" @@ -1160,6 +1161,32 @@ Expected parseWinEHPrepareOptions(StringRef Params) { "WinEHPreparePass"); } +Expected parseGlobalMergeOptions(StringRef Params) { + GlobalMergeOptions Result; + while (!Params.empty()) { + StringRef ParamName; + std::tie(ParamName, Params) = Params.split(';'); + + bool Enable = !ParamName.consume_front("no-"); + if (ParamName == "group-by-use") + Result.GroupByUse = Enable; + else if (ParamName == "ignore-single-use") + Result.IgnoreSingleUse = Enable; + else if (ParamName == "merge-const") + Result.MergeConst = Enable; + else if (ParamName == "merge-external") + Result.MergeExternal = Enable; + else if (ParamName.consume_front("max-offset=")) { + if (ParamName.getAsInteger(0, Result.MaxOffset)) + return make_error( + formatv("invalid GlobalMergePass parameter '{0}' ", ParamName) + .str(), + inconvertibleErrorCode()); + } + } + return Result; +} + } // namespace /// Tests whether a pass name starts with a valid prefix for a default pipeline diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index bda36bd8c107c..a5d986a637947 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -151,6 +151,13 @@ MODULE_PASS_WITH_PARAMS( "asan", "AddressSanitizerPass", [](AddressSanitizerOptions Opts) { return AddressSanitizerPass(Opts); }, parseASanPassOptions, "kernel") +MODULE_PASS_WITH_PARAMS( + "global-merge", "GlobalMergePass", + [TM = TM](GlobalMergeOptions Opts) { return GlobalMergePass(TM, Opts); }, + parseGlobalMergeOptions, + "group-by-use;ignore-single-use;max-offset=N;merge-const;merge-external;" + "no-group-by-use;no-ignore-single-use;no-merge-const;no-merge-external;" + "size-only") MODULE_PASS_WITH_PARAMS( "globaldce", "GlobalDCEPass", [](bool InLTOPostLink) { return GlobalDCEPass(InLTOPostLink); }, @@ -421,7 +428,7 @@ FUNCTION_PASS("structurizecfg", StructurizeCFGPass()) FUNCTION_PASS("tailcallelim", TailCallElimPass()) FUNCTION_PASS("tlshoist", TLSVariableHoistPass()) FUNCTION_PASS("transform-warning", WarnMissedTransformationsPass()) -FUNCTION_PASS("trigger-verifier-error", TriggerVerifierErrorPass()) +FUNCTION_PASS("trigger-verifier-error", TriggerVerifierErrorPass()) FUNCTION_PASS("tsan", ThreadSanitizerPass()) FUNCTION_PASS("typepromotion", TypePromotionPass(TM)) FUNCTION_PASS("unify-loop-exits", UnifyLoopExitsPass()) diff --git a/llvm/test/Transforms/GlobalMerge/alignment-2.ll b/llvm/test/Transforms/GlobalMerge/alignment-2.ll index 194421613e81e..99657d8f6ba7d 100644 --- a/llvm/test/Transforms/GlobalMerge/alignment-2.ll +++ b/llvm/test/Transforms/GlobalMerge/alignment-2.ll @@ -1,4 +1,5 @@ ; RUN: opt -global-merge -global-merge-max-offset=100 -S -o - %s | FileCheck %s +; RUN: opt -passes='global-merge' -S -o - %s | FileCheck %s target datalayout = "e-p:64:64" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/GlobalMerge/alignment.ll b/llvm/test/Transforms/GlobalMerge/alignment.ll index e3c3eb3ce065b..a787791369c62 100644 --- a/llvm/test/Transforms/GlobalMerge/alignment.ll +++ b/llvm/test/Transforms/GlobalMerge/alignment.ll @@ -1,4 +1,5 @@ ; RUN: opt -global-merge -global-merge-max-offset=100 -S -o - %s | FileCheck %s +; RUN: opt -passes='global-merge' -S -o - %s | FileCheck %s target datalayout = "e-p:64:64" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/GlobalMerge/basic.ll b/llvm/test/Transforms/GlobalMerge/basic.ll index 69d897d04a1a0..67e2b70220a78 100644 --- a/llvm/test/Transforms/GlobalMerge/basic.ll +++ b/llvm/test/Transforms/GlobalMerge/basic.ll @@ -1,4 +1,5 @@ ; RUN: opt -global-merge -global-merge-max-offset=100 -S -o - %s | FileCheck %s +; RUN: opt -passes='global-merge' -S -o - %s | FileCheck %s target datalayout = "e-p:64:64" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/GlobalMerge/debug-info.ll b/llvm/test/Transforms/GlobalMerge/debug-info.ll index b20e59007716e..8a535b5d3873a 100644 --- a/llvm/test/Transforms/GlobalMerge/debug-info.ll +++ b/llvm/test/Transforms/GlobalMerge/debug-info.ll @@ -1,4 +1,5 @@ ; RUN: opt -global-merge -global-merge-max-offset=100 -S -o - %s | FileCheck %s +; RUN: opt -passes='global-merge' -S -o - %s | FileCheck %s source_filename = "test/Transforms/GlobalMerge/debug-info.ll" target datalayout = "e-p:64:64" diff --git a/llvm/test/Transforms/GlobalMerge/eh-filter.ll b/llvm/test/Transforms/GlobalMerge/eh-filter.ll index 3500865331aab..28f7905c37057 100644 --- a/llvm/test/Transforms/GlobalMerge/eh-filter.ll +++ b/llvm/test/Transforms/GlobalMerge/eh-filter.ll @@ -1,5 +1,6 @@ ; REQUIRES: asserts ; RUN: opt -global-merge -debug-only=global-merge -S -o - %s 2>&1 | FileCheck %s +; RUN: opt -passes='global-merge' -debug-only=global-merge -S -o - %s 2>&1 | FileCheck %s ;; Checks from the debug info. ; CHECK: Number of GV that must be kept: 5 diff --git a/llvm/test/Transforms/GlobalMerge/used.ll b/llvm/test/Transforms/GlobalMerge/used.ll index 18a68d1a737a9..fbc93858eb47e 100644 --- a/llvm/test/Transforms/GlobalMerge/used.ll +++ b/llvm/test/Transforms/GlobalMerge/used.ll @@ -1,4 +1,5 @@ ; RUN: opt -global-merge -global-merge-max-offset=100 -S -o - %s | FileCheck %s +; RUN: opt -passes='global-merge' -S -o - %s | FileCheck %s target datalayout = "e-p:64:64" target triple = "x86_64-unknown-linux-gnu"