Skip to content

Commit be8d199

Browse files
elevianteleviant
committed
[ThinLTO] Internalize readonly globals
This patch allows internalising globals if all accesses to them (from live functions) are from non-volatile load instructions Differential revision: https://reviews.llvm.org/D49362 llvm-svn: 346584
1 parent 825f9d3 commit be8d199

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+774
-89
lines changed

‎llvm/include/llvm/IR/ModuleSummaryIndex.h

+38-12
Original file line numberDiff line numberDiff line change
@@ -163,13 +163,13 @@ using GlobalValueSummaryMapTy =
163163
/// Struct that holds a reference to a particular GUID in a global value
164164
/// summary.
165165
struct ValueInfo {
166-
PointerIntPair<const GlobalValueSummaryMapTy::value_type *, 1, bool>
167-
RefAndFlag;
166+
PointerIntPair<const GlobalValueSummaryMapTy::value_type *, 2, int>
167+
RefAndFlags;
168168

169169
ValueInfo() = default;
170170
ValueInfo(bool HaveGVs, const GlobalValueSummaryMapTy::value_type *R) {
171-
RefAndFlag.setPointer(R);
172-
RefAndFlag.setInt(HaveGVs);
171+
RefAndFlags.setPointer(R);
172+
RefAndFlags.setInt(HaveGVs);
173173
}
174174

175175
operator bool() const { return getRef(); }
@@ -189,10 +189,12 @@ struct ValueInfo {
189189
: getRef()->second.U.Name;
190190
}
191191

192-
bool haveGVs() const { return RefAndFlag.getInt(); }
192+
bool haveGVs() const { return RefAndFlags.getInt() & 0x1; }
193+
bool isReadOnly() const { return RefAndFlags.getInt() & 0x2; }
194+
void setReadOnly() { RefAndFlags.setInt(RefAndFlags.getInt() | 0x2); }
193195

194196
const GlobalValueSummaryMapTy::value_type *getRef() const {
195-
return RefAndFlag.getPointer();
197+
return RefAndFlags.getPointer();
196198
}
197199

198200
bool isDSOLocal() const;
@@ -543,6 +545,8 @@ class FunctionSummary : public GlobalValueSummary {
543545
std::move(TypeTestAssumeConstVCalls),
544546
std::move(TypeCheckedLoadConstVCalls)});
545547
}
548+
// Gets the number of immutable refs in RefEdgeList
549+
unsigned immutableRefCount() const;
546550

547551
/// Check if this is a function summary.
548552
static bool classof(const GlobalValueSummary *GVS) {
@@ -652,19 +656,30 @@ template <> struct DenseMapInfo<FunctionSummary::ConstVCall> {
652656
/// Global variable summary information to aid decisions and
653657
/// implementation of importing.
654658
///
655-
/// Currently this doesn't add anything to the base \p GlobalValueSummary,
656-
/// but is a placeholder as additional info may be added to the summary
657-
/// for variables.
659+
/// Global variable summary has extra flag, telling if it is
660+
/// modified during the program run or not. This affects ThinLTO
661+
/// internalization
658662
class GlobalVarSummary : public GlobalValueSummary {
659-
660663
public:
661-
GlobalVarSummary(GVFlags Flags, std::vector<ValueInfo> Refs)
662-
: GlobalValueSummary(GlobalVarKind, Flags, std::move(Refs)) {}
664+
struct GVarFlags {
665+
GVarFlags(bool ReadOnly = false) : ReadOnly(ReadOnly) {}
666+
667+
unsigned ReadOnly : 1;
668+
} VarFlags;
669+
670+
GlobalVarSummary(GVFlags Flags, GVarFlags VarFlags,
671+
std::vector<ValueInfo> Refs)
672+
: GlobalValueSummary(GlobalVarKind, Flags, std::move(Refs)),
673+
VarFlags(VarFlags) {}
663674

664675
/// Check if this is a global variable summary.
665676
static bool classof(const GlobalValueSummary *GVS) {
666677
return GVS->getSummaryKind() == GlobalVarKind;
667678
}
679+
680+
GVarFlags varflags() const { return VarFlags; }
681+
void setReadOnly(bool RO) { VarFlags.ReadOnly = RO; }
682+
bool isReadOnly() const { return VarFlags.ReadOnly; }
668683
};
669684

670685
struct TypeTestResolution {
@@ -1135,6 +1150,9 @@ class ModuleSummaryIndex {
11351150

11361151
/// Print out strongly connected components for debugging.
11371152
void dumpSCCs(raw_ostream &OS);
1153+
1154+
/// Analyze index and detect unmodified globals
1155+
void propagateConstants(const DenseSet<GlobalValue::GUID> &PreservedSymbols);
11381156
};
11391157

11401158
/// GraphTraits definition to build SCC for the index
@@ -1184,6 +1202,14 @@ struct GraphTraits<ModuleSummaryIndex *> : public GraphTraits<ValueInfo> {
11841202
}
11851203
};
11861204

1205+
static inline bool canImportGlobalVar(GlobalValueSummary *S) {
1206+
assert(isa<GlobalVarSummary>(S->getBaseObject()));
1207+
1208+
// We don't import GV with references, because it can result
1209+
// in promotion of local variables in the source module.
1210+
return !GlobalValue::isInterposableLinkage(S->linkage()) &&
1211+
!S->notEligibleToImport() && S->refs().empty();
1212+
}
11871213
} // end namespace llvm
11881214

11891215
#endif // LLVM_IR_MODULESUMMARYINDEX_H

‎llvm/include/llvm/Transforms/IPO/FunctionImport.h

+8
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,14 @@ void computeDeadSymbols(
176176
const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
177177
function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing);
178178

179+
/// Compute dead symbols and run constant propagation in combined index
180+
/// after that.
181+
void computeDeadSymbolsWithConstProp(
182+
ModuleSummaryIndex &Index,
183+
const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
184+
function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing,
185+
bool ImportEnabled);
186+
179187
/// Converts value \p GV to declaration, or replaces with a declaration if
180188
/// it is an alias. Returns true if converted, false if replaced.
181189
bool convertToDeclaration(GlobalValue &GV);

‎llvm/include/llvm/Transforms/Utils/FunctionImportUtils.h

-1
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,6 @@ class FunctionImportGlobalProcessing {
113113
bool renameModuleForThinLTO(
114114
Module &M, const ModuleSummaryIndex &Index,
115115
SetVector<GlobalValue *> *GlobalsToImport = nullptr);
116-
117116
} // End llvm namespace
118117

119118
#endif

‎llvm/lib/Analysis/ModuleSummaryAnalysis.cpp

+58-17
Original file line numberDiff line numberDiff line change
@@ -220,10 +220,19 @@ static void addIntrinsicToSummary(
220220
}
221221
}
222222

223-
static void computeFunctionSummary(
224-
ModuleSummaryIndex &Index, const Module &M, const Function &F,
225-
BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, DominatorTree &DT,
226-
bool HasLocalsInUsedOrAsm, DenseSet<GlobalValue::GUID> &CantBePromoted) {
223+
static bool isNonVolatileLoad(const Instruction *I) {
224+
if (const auto *LI = dyn_cast<LoadInst>(I))
225+
return !LI->isVolatile();
226+
227+
return false;
228+
}
229+
230+
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
231+
const Function &F, BlockFrequencyInfo *BFI,
232+
ProfileSummaryInfo *PSI, DominatorTree &DT,
233+
bool HasLocalsInUsedOrAsm,
234+
DenseSet<GlobalValue::GUID> &CantBePromoted,
235+
bool IsThinLTO) {
227236
// Summary not currently supported for anonymous functions, they should
228237
// have been named.
229238
assert(F.hasName());
@@ -244,13 +253,21 @@ static void computeFunctionSummary(
244253
// Add personality function, prefix data and prologue data to function's ref
245254
// list.
246255
findRefEdges(Index, &F, RefEdges, Visited);
256+
std::vector<const Instruction *> NonVolatileLoads;
247257

248258
bool HasInlineAsmMaybeReferencingInternal = false;
249259
for (const BasicBlock &BB : F)
250260
for (const Instruction &I : BB) {
251261
if (isa<DbgInfoIntrinsic>(I))
252262
continue;
253263
++NumInsts;
264+
if (isNonVolatileLoad(&I)) {
265+
// Postpone processing of non-volatile load instructions
266+
// See comments below
267+
Visited.insert(&I);
268+
NonVolatileLoads.push_back(&I);
269+
continue;
270+
}
254271
findRefEdges(Index, &I, RefEdges, Visited);
255272
auto CS = ImmutableCallSite(&I);
256273
if (!CS)
@@ -340,6 +357,24 @@ static void computeFunctionSummary(
340357
}
341358
}
342359

360+
// By now we processed all instructions in a function, except
361+
// non-volatile loads. All new refs we add in a loop below
362+
// are obviously constant. All constant refs are grouped in the
363+
// end of RefEdges vector, so we can use a single integer value
364+
// to identify them.
365+
unsigned RefCnt = RefEdges.size();
366+
for (const Instruction *I : NonVolatileLoads) {
367+
Visited.erase(I);
368+
findRefEdges(Index, I, RefEdges, Visited);
369+
}
370+
std::vector<ValueInfo> Refs = RefEdges.takeVector();
371+
// Regular LTO module doesn't participate in ThinLTO import,
372+
// so no reference from it can be readonly, since this would
373+
// require importing variable as local copy
374+
if (IsThinLTO)
375+
for (; RefCnt < Refs.size(); ++RefCnt)
376+
Refs[RefCnt].setReadOnly();
377+
343378
// Explicit add hot edges to enforce importing for designated GUIDs for
344379
// sample PGO, to enable the same inlines as the profiled optimized binary.
345380
for (auto &I : F.getImportGUIDs())
@@ -363,9 +398,9 @@ static void computeFunctionSummary(
363398
// Don't try to import functions with noinline attribute.
364399
F.getAttributes().hasFnAttribute(Attribute::NoInline)};
365400
auto FuncSummary = llvm::make_unique<FunctionSummary>(
366-
Flags, NumInsts, FunFlags, RefEdges.takeVector(),
367-
CallGraphEdges.takeVector(), TypeTests.takeVector(),
368-
TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(),
401+
Flags, NumInsts, FunFlags, std::move(Refs), CallGraphEdges.takeVector(),
402+
TypeTests.takeVector(), TypeTestAssumeVCalls.takeVector(),
403+
TypeCheckedLoadVCalls.takeVector(),
369404
TypeTestAssumeConstVCalls.takeVector(),
370405
TypeCheckedLoadConstVCalls.takeVector());
371406
if (NonRenamableLocal)
@@ -382,8 +417,13 @@ computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V,
382417
bool NonRenamableLocal = isNonRenamableLocal(V);
383418
GlobalValueSummary::GVFlags Flags(V.getLinkage(), NonRenamableLocal,
384419
/* Live = */ false, V.isDSOLocal());
385-
auto GVarSummary =
386-
llvm::make_unique<GlobalVarSummary>(Flags, RefEdges.takeVector());
420+
421+
// Don't mark variables we won't be able to internalize as read-only.
422+
GlobalVarSummary::GVarFlags VarFlags(
423+
!V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() &&
424+
!V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass());
425+
auto GVarSummary = llvm::make_unique<GlobalVarSummary>(Flags, VarFlags,
426+
RefEdges.takeVector());
387427
if (NonRenamableLocal)
388428
CantBePromoted.insert(V.getGUID());
389429
if (HasBlockAddress)
@@ -487,13 +527,19 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
487527
Index.addGlobalValueSummary(*GV, std::move(Summary));
488528
} else {
489529
std::unique_ptr<GlobalVarSummary> Summary =
490-
llvm::make_unique<GlobalVarSummary>(GVFlags,
491-
ArrayRef<ValueInfo>{});
530+
llvm::make_unique<GlobalVarSummary>(
531+
GVFlags, GlobalVarSummary::GVarFlags(),
532+
ArrayRef<ValueInfo>{});
492533
Index.addGlobalValueSummary(*GV, std::move(Summary));
493534
}
494535
});
495536
}
496537

538+
bool IsThinLTO = true;
539+
if (auto *MD =
540+
mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO")))
541+
IsThinLTO = MD->getZExtValue();
542+
497543
// Compute summaries for all functions defined in module, and save in the
498544
// index.
499545
for (auto &F : M) {
@@ -514,7 +560,7 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
514560

515561
computeFunctionSummary(Index, M, F, BFI, PSI, DT,
516562
!LocalsUsed.empty() || HasLocalInlineAsmSymbol,
517-
CantBePromoted);
563+
CantBePromoted, IsThinLTO);
518564
}
519565

520566
// Compute summaries for all variables defined in module, and save in the
@@ -545,11 +591,6 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
545591
setLiveRoot(Index, "llvm.global_dtors");
546592
setLiveRoot(Index, "llvm.global.annotations");
547593

548-
bool IsThinLTO = true;
549-
if (auto *MD =
550-
mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO")))
551-
IsThinLTO = MD->getZExtValue();
552-
553594
for (auto &GlobalList : Index) {
554595
// Ignore entries for references that are undefined in the current module.
555596
if (GlobalList.second.SummaryList.empty())

‎llvm/lib/AsmParser/LLParser.cpp

+2-1
Original file line numberDiff line numberDiff line change
@@ -7642,7 +7642,8 @@ bool LLParser::ParseVariableSummary(std::string Name, GlobalValue::GUID GUID,
76427642
if (ParseToken(lltok::rparen, "expected ')' here"))
76437643
return true;
76447644

7645-
auto GS = llvm::make_unique<GlobalVarSummary>(GVFlags, std::move(Refs));
7645+
auto GS = llvm::make_unique<GlobalVarSummary>(
7646+
GVFlags, GlobalVarSummary::GVarFlags(), std::move(Refs));
76467647

76477648
GS->setModulePath(ModulePath);
76487649

0 commit comments

Comments
 (0)