diff --git a/llvm/include/llvm/Transforms/Instrumentation.h b/llvm/include/llvm/Transforms/Instrumentation.h index ea97ab2562a5b..667505ded16a0 100644 --- a/llvm/include/llvm/Transforms/Instrumentation.h +++ b/llvm/include/llvm/Transforms/Instrumentation.h @@ -49,10 +49,25 @@ GlobalVariable *createPrivateGlobalForString(Module &M, StringRef Str, // Returns nullptr on failure. Comdat *getOrCreateFunctionComdat(Function &F, Triple &T); -// Place global in a large section for x86-64 ELF binaries to mitigate -// relocation overflow pressure. This can be be used for metadata globals that -// aren't directly accessed by code, which has no performance impact. -void setGlobalVariableLargeSection(const Triple &TargetTriple, +// Place global in a large section for x86-64 ELF binaries in code models with +// split small/large data sections to mitigate relocation overflow pressure. +// This can be be used for metadata globals that aren't directly accessed by +// code, which has no performance impact. +void setGlobalVariableLargeCodeModel(const Triple &TargetTriple, + GlobalVariable &GV); + +// Place global in a small or large section for x86-64 ELF binaries in code +// models with split small/large data sections. This can be be used for metadata +// globals that vary in size that are put into an explicit section, which can +// cause inconsistent small/large section flags on the explicit section. This +// should only be used for globals that are frequently accessed, otherwise +// setGlobalVariableLargeCodeModel() should be used to mitigate relocation +// pressure. +// +// Currently this places the variables in a small section for the medium code +// model, where performance is still a concern, but in a large section for the +// large code model, where we try to avoid all relocation pressure. +void setHotGlobalVariableCodeModel(const Triple &TargetTriple, GlobalVariable &GV); // Insert GCOV profiling instrumentation diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp index fb5838bb7941a..3ab8a21e340ce 100644 --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -2181,7 +2181,7 @@ ModuleAddressSanitizer::CreateMetadataGlobal(Module &M, Constant *Initializer, Metadata->setSection(getGlobalMetadataSection()); // Place metadata in a large section for x86-64 ELF binaries to mitigate // relocation pressure. - setGlobalVariableLargeSection(TargetTriple, *Metadata); + setGlobalVariableLargeCodeModel(TargetTriple, *Metadata); return Metadata; } diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index a19b140872544..1a5b16f411899 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -1285,6 +1285,7 @@ GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc, // Put the counters and bitmaps in their own sections so linkers can // remove unneeded sections. Ptr->setSection(getInstrProfSectionName(IPSK, TT.getObjectFormat())); + setHotGlobalVariableCodeModel(TT, *Ptr); Ptr->setLinkage(Linkage); maybeSetComdat(Ptr, Fn, VarName); return Ptr; @@ -1450,7 +1451,7 @@ void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) { M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy), getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed)); ValuesVar->setVisibility(Visibility); - setGlobalVariableLargeSection(TT, *ValuesVar); + setGlobalVariableLargeCodeModel(TT, *ValuesVar); ValuesVar->setSection( getInstrProfSectionName(IPSK_vals, TT.getObjectFormat())); ValuesVar->setAlignment(Align(8)); @@ -1588,7 +1589,7 @@ void InstrLowerer::emitVNodes() { auto *VNodesVar = new GlobalVariable( M, VNodesTy, false, GlobalValue::PrivateLinkage, Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName()); - setGlobalVariableLargeSection(TT, *VNodesVar); + setGlobalVariableLargeCodeModel(TT, *VNodesVar); VNodesVar->setSection( getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat())); VNodesVar->setAlignment(M.getDataLayout().getABITypeAlign(VNodesTy)); @@ -1616,7 +1617,7 @@ void InstrLowerer::emitNameData() { GlobalValue::PrivateLinkage, NamesVal, getInstrProfNamesVarName()); NamesSize = CompressedNameStr.size(); - setGlobalVariableLargeSection(TT, *NamesVar); + setGlobalVariableLargeCodeModel(TT, *NamesVar); NamesVar->setSection( ProfileCorrelate == InstrProfCorrelator::BINARY ? getInstrProfSectionName(IPSK_covname, TT.getObjectFormat()) diff --git a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp index b842d9eef407c..09fff5f2d7954 100644 --- a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp @@ -85,8 +85,8 @@ Comdat *llvm::getOrCreateFunctionComdat(Function &F, Triple &T) { return C; } -void llvm::setGlobalVariableLargeSection(const Triple &TargetTriple, - GlobalVariable &GV) { +void llvm::setGlobalVariableLargeCodeModel(const Triple &TargetTriple, + GlobalVariable &GV) { // Limit to x86-64 ELF. if (TargetTriple.getArch() != Triple::x86_64 || TargetTriple.getObjectFormat() != Triple::ELF) @@ -97,3 +97,19 @@ void llvm::setGlobalVariableLargeSection(const Triple &TargetTriple, return; GV.setCodeModel(CodeModel::Large); } + +void llvm::setHotGlobalVariableCodeModel(const Triple &TargetTriple, + GlobalVariable &GV) { + // Limit to x86-64 ELF. + if (TargetTriple.getArch() != Triple::x86_64 || + TargetTriple.getObjectFormat() != Triple::ELF) + return; + // Limit to medium/large code models. + std::optional CM = GV.getParent()->getCodeModel(); + if (!CM) + return; + if (*CM == CodeModel::Medium) + GV.setCodeModel(CodeModel::Small); + else if (*CM == CodeModel::Large) + GV.setCodeModel(CodeModel::Large); +} diff --git a/llvm/test/Instrumentation/InstrProfiling/section-code-model-large.ll b/llvm/test/Instrumentation/InstrProfiling/section-code-model-large.ll index 5ebce67bab86a..dff86542d26d7 100644 --- a/llvm/test/Instrumentation/InstrProfiling/section-code-model-large.ll +++ b/llvm/test/Instrumentation/InstrProfiling/section-code-model-large.ll @@ -24,7 +24,7 @@ attributes #0 = { nounwind } !0 = !{i32 1, !"Code Model", i32 4} ; CHECK: @__profc_foo = -; CHECK-NOT: code_model "large" +; CHECK-SAME: code_model "large" ; CHECK: @__profvp_foo = ; CHECK-SAME: code_model "large" ; CHECK: @__profd_foo = diff --git a/llvm/test/Instrumentation/InstrProfiling/section-code-model-medium.ll b/llvm/test/Instrumentation/InstrProfiling/section-code-model-medium.ll index 0b269a87a6444..e575d8e7198b7 100644 --- a/llvm/test/Instrumentation/InstrProfiling/section-code-model-medium.ll +++ b/llvm/test/Instrumentation/InstrProfiling/section-code-model-medium.ll @@ -25,7 +25,8 @@ attributes #0 = { nounwind } !0 = !{i32 1, !"Code Model", i32 3} ; CHECK: @__profc_foo = -; CHECK-NOT: code_model "large" +; X8664-SAME: code_model "small" +; PPC-NOT: code_model ; CHECK: @__profvp_foo = ; X8664-SAME: code_model "large" ; PPC-NOT: code_model "large" diff --git a/llvm/test/Instrumentation/InstrProfiling/section-code-model-small.ll b/llvm/test/Instrumentation/InstrProfiling/section-code-model-small.ll index 11cc187512951..8c1d595cab3cb 100644 --- a/llvm/test/Instrumentation/InstrProfiling/section-code-model-small.ll +++ b/llvm/test/Instrumentation/InstrProfiling/section-code-model-small.ll @@ -24,7 +24,7 @@ attributes #0 = { nounwind } !0 = !{i32 1, !"Code Model", i32 1} ; CHECK: @__profc_foo = -; CHECK-NOT: code_model "large" +; CHECK-NOT: code_model ; CHECK: @__profvp_foo = ; CHECK-NOT: code_model "large" ; CHECK: @__profd_foo =