From b2ef3f3d57a5e12e266f897c172966dff6fdbb4c Mon Sep 17 00:00:00 2001 From: Artem Pianykh Date: Thu, 12 Sep 2024 15:23:43 -0700 Subject: [PATCH 1/8] [NFC][Utils] Extract BuildDebugInfoMDMap from CloneFunctionInto Summary: Extract the logic to build up a metadata map to use in metadata cloning into a separate function. Test Plan: ninja check-llvm-unit check-llvm stack-info: PR: https://github.com/llvm/llvm-project/pull/118622, branch: users/artempyanykh/fast-coro-upstream/3 --- llvm/include/llvm/Transforms/Utils/Cloning.h | 8 ++ llvm/lib/Transforms/Utils/CloneFunction.cpp | 89 +++++++++++--------- 2 files changed, 58 insertions(+), 39 deletions(-) diff --git a/llvm/include/llvm/Transforms/Utils/Cloning.h b/llvm/include/llvm/Transforms/Utils/Cloning.h index 3c8f2cbfaa9b8..7858c9d9def0d 100644 --- a/llvm/include/llvm/Transforms/Utils/Cloning.h +++ b/llvm/include/llvm/Transforms/Utils/Cloning.h @@ -220,6 +220,14 @@ DISubprogram *CollectDebugInfoForCloning(const Function &F, CloneFunctionChangeType Changes, DebugInfoFinder &DIFinder); +/// Build a map of debug info to use during Metadata cloning. +/// Returns true if cloning would need module level changes and false if there +/// would only be local changes. +bool BuildDebugInfoMDMap(DenseMap &MD, + CloneFunctionChangeType Changes, + DebugInfoFinder &DIFinder, + DISubprogram *SPClonedWithinModule); + /// This class captures the data input to the InlineFunction call, and records /// the auxiliary results produced by it. class InlineFunctionInfo { diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp index d038117090e4c..6dc5f601b7fca 100644 --- a/llvm/lib/Transforms/Utils/CloneFunction.cpp +++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp @@ -152,6 +152,54 @@ DISubprogram *llvm::CollectDebugInfoForCloning(const Function &F, return SPClonedWithinModule; } +bool llvm::BuildDebugInfoMDMap(DenseMap &MD, + CloneFunctionChangeType Changes, + DebugInfoFinder &DIFinder, + DISubprogram *SPClonedWithinModule) { + bool ModuleLevelChanges = Changes > CloneFunctionChangeType::LocalChangesOnly; + if (Changes < CloneFunctionChangeType::DifferentModule && + DIFinder.subprogram_count() > 0) { + // Turn on module-level changes, since we need to clone (some of) the + // debug info metadata. + // + // FIXME: Metadata effectively owned by a function should be made + // local, and only that local metadata should be cloned. + ModuleLevelChanges = true; + + auto mapToSelfIfNew = [&MD](MDNode *N) { + // Avoid clobbering an existing mapping. + (void)MD.try_emplace(N, N); + }; + + // Avoid cloning types, compile units, and (other) subprograms. + SmallPtrSet MappedToSelfSPs; + for (DISubprogram *ISP : DIFinder.subprograms()) { + if (ISP != SPClonedWithinModule) { + mapToSelfIfNew(ISP); + MappedToSelfSPs.insert(ISP); + } + } + + // If a subprogram isn't going to be cloned skip its lexical blocks as well. + for (DIScope *S : DIFinder.scopes()) { + auto *LScope = dyn_cast(S); + if (LScope && MappedToSelfSPs.count(LScope->getSubprogram())) + mapToSelfIfNew(S); + } + + for (DICompileUnit *CU : DIFinder.compile_units()) + mapToSelfIfNew(CU); + + for (DIType *Type : DIFinder.types()) + mapToSelfIfNew(Type); + } else { + assert(!SPClonedWithinModule && + "Subprogram should be in DIFinder->subprogram_count()..."); + } + + return ModuleLevelChanges; +} + // Clone OldFunc into NewFunc, transforming the old arguments into references to // VMap values. void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, @@ -210,45 +258,8 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, DISubprogram *SPClonedWithinModule = CollectDebugInfoForCloning(*OldFunc, Changes, DIFinder); - if (Changes < CloneFunctionChangeType::DifferentModule && - DIFinder.subprogram_count() > 0) { - // Turn on module-level changes, since we need to clone (some of) the - // debug info metadata. - // - // FIXME: Metadata effectively owned by a function should be made - // local, and only that local metadata should be cloned. - ModuleLevelChanges = true; - - auto mapToSelfIfNew = [&VMap](MDNode *N) { - // Avoid clobbering an existing mapping. - (void)VMap.MD().try_emplace(N, N); - }; - - // Avoid cloning types, compile units, and (other) subprograms. - SmallPtrSet MappedToSelfSPs; - for (DISubprogram *ISP : DIFinder.subprograms()) { - if (ISP != SPClonedWithinModule) { - mapToSelfIfNew(ISP); - MappedToSelfSPs.insert(ISP); - } - } - - // If a subprogram isn't going to be cloned skip its lexical blocks as well. - for (DIScope *S : DIFinder.scopes()) { - auto *LScope = dyn_cast(S); - if (LScope && MappedToSelfSPs.count(LScope->getSubprogram())) - mapToSelfIfNew(S); - } - - for (DICompileUnit *CU : DIFinder.compile_units()) - mapToSelfIfNew(CU); - - for (DIType *Type : DIFinder.types()) - mapToSelfIfNew(Type); - } else { - assert(!SPClonedWithinModule && - "Subprogram should be in DIFinder->subprogram_count()..."); - } + ModuleLevelChanges = + BuildDebugInfoMDMap(VMap.MD(), Changes, DIFinder, SPClonedWithinModule); const auto RemapFlag = ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges; // Duplicate the metadata that is attached to the cloned function. From 9dcfc56cf669e9b8b571469ba5ba850b7565faa7 Mon Sep 17 00:00:00 2001 From: Artem Pianykh Date: Thu, 12 Sep 2024 15:35:38 -0700 Subject: [PATCH 2/8] [NFC][Utils] Extract CloneFunctionMetadataInto from CloneFunctionInto Summary: The new API expects the caller to populate the VMap. We need it this way for a subsequent change around coroutine cloning. Test Plan: ninja check-llvm-unit check-llvm stack-info: PR: https://github.com/llvm/llvm-project/pull/118623, branch: users/artempyanykh/fast-coro-upstream/4 --- llvm/include/llvm/Transforms/Utils/Cloning.h | 12 +++++++++ llvm/lib/Transforms/Utils/CloneFunction.cpp | 28 +++++++++++++------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/llvm/include/llvm/Transforms/Utils/Cloning.h b/llvm/include/llvm/Transforms/Utils/Cloning.h index 7858c9d9def0d..9a574fc4e4c08 100644 --- a/llvm/include/llvm/Transforms/Utils/Cloning.h +++ b/llvm/include/llvm/Transforms/Utils/Cloning.h @@ -182,6 +182,18 @@ void CloneFunctionAttributesInto(Function *NewFunc, const Function *OldFunc, ValueMapTypeRemapper *TypeMapper = nullptr, ValueMaterializer *Materializer = nullptr); +/// Clone OldFunc's metadata into NewFunc. +/// +/// The caller is expected to populate \p VMap beforehand and set an appropriate +/// \p RemapFlag. +/// +/// NOTE: This function doesn't clone !llvm.dbg.cu when cloning into a different +/// module. Use CloneFunctionInto for that behavior. +void CloneFunctionMetadataInto(Function *NewFunc, const Function *OldFunc, + ValueToValueMapTy &VMap, RemapFlags RemapFlag, + ValueMapTypeRemapper *TypeMapper = nullptr, + ValueMaterializer *Materializer = nullptr); + void CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc, const Instruction *StartingInst, ValueToValueMapTy &VMap, bool ModuleLevelChanges, diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp index 6dc5f601b7fca..c967e78123af1 100644 --- a/llvm/lib/Transforms/Utils/CloneFunction.cpp +++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp @@ -200,6 +200,22 @@ bool llvm::BuildDebugInfoMDMap(DenseMap &MD, return ModuleLevelChanges; } +void llvm::CloneFunctionMetadataInto(Function *NewFunc, const Function *OldFunc, + ValueToValueMapTy &VMap, + RemapFlags RemapFlag, + ValueMapTypeRemapper *TypeMapper, + ValueMaterializer *Materializer) { + // Duplicate the metadata that is attached to the cloned function. + // Subprograms/CUs/types that were already mapped to themselves won't be + // duplicated. + SmallVector, 1> MDs; + OldFunc->getAllMetadata(MDs); + for (auto MD : MDs) { + NewFunc->addMetadata(MD.first, *MapMetadata(MD.second, VMap, RemapFlag, + TypeMapper, Materializer)); + } +} + // Clone OldFunc into NewFunc, transforming the old arguments into references to // VMap values. void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, @@ -262,15 +278,9 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, BuildDebugInfoMDMap(VMap.MD(), Changes, DIFinder, SPClonedWithinModule); const auto RemapFlag = ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges; - // Duplicate the metadata that is attached to the cloned function. - // Subprograms/CUs/types that were already mapped to themselves won't be - // duplicated. - SmallVector, 1> MDs; - OldFunc->getAllMetadata(MDs); - for (auto MD : MDs) { - NewFunc->addMetadata(MD.first, *MapMetadata(MD.second, VMap, RemapFlag, - TypeMapper, Materializer)); - } + + CloneFunctionMetadataInto(NewFunc, OldFunc, VMap, RemapFlag, TypeMapper, + Materializer); // Loop over all of the basic blocks in the function, cloning them as // appropriate. Note that we save BE this way in order to handle cloning of From 6845960eea8fe58cc5f6e05611ef2920c49a34b9 Mon Sep 17 00:00:00 2001 From: Artem Pianykh Date: Thu, 12 Sep 2024 15:50:25 -0700 Subject: [PATCH 3/8] [NFC][Utils] Extract CloneFunctionBodyInto from CloneFunctionInto Summary: This and previously extracted `CloneFunction*Into` functions will be used in later diffs. Test Plan: ninja check-llvm-unit check-llvm stack-info: PR: https://github.com/llvm/llvm-project/pull/118624, branch: users/artempyanykh/fast-coro-upstream/5 --- llvm/include/llvm/Transforms/Utils/Cloning.h | 34 ++++--- llvm/lib/Transforms/Utils/CloneFunction.cpp | 96 +++++++++++--------- 2 files changed, 76 insertions(+), 54 deletions(-) diff --git a/llvm/include/llvm/Transforms/Utils/Cloning.h b/llvm/include/llvm/Transforms/Utils/Cloning.h index 9a574fc4e4c08..50518c746d11c 100644 --- a/llvm/include/llvm/Transforms/Utils/Cloning.h +++ b/llvm/include/llvm/Transforms/Utils/Cloning.h @@ -194,6 +194,15 @@ void CloneFunctionMetadataInto(Function *NewFunc, const Function *OldFunc, ValueMapTypeRemapper *TypeMapper = nullptr, ValueMaterializer *Materializer = nullptr); +/// Clone OldFunc's body into NewFunc. +void CloneFunctionBodyInto(Function *NewFunc, const Function *OldFunc, + ValueToValueMapTy &VMap, RemapFlags RemapFlag, + SmallVectorImpl &Returns, + const char *NameSuffix = "", + ClonedCodeInfo *CodeInfo = nullptr, + ValueMapTypeRemapper *TypeMapper = nullptr, + ValueMaterializer *Materializer = nullptr); + void CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc, const Instruction *StartingInst, ValueToValueMapTy &VMap, bool ModuleLevelChanges, @@ -214,7 +223,7 @@ void CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc, /// void CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, bool ModuleLevelChanges, - SmallVectorImpl &Returns, + SmallVectorImpl &Returns, const char *NameSuffix = "", ClonedCodeInfo *CodeInfo = nullptr); @@ -361,32 +370,31 @@ void updateProfileCallee( /// Find the 'llvm.experimental.noalias.scope.decl' intrinsics in the specified /// basic blocks and extract their scope. These are candidates for duplication /// when cloning. -void identifyNoAliasScopesToClone( - ArrayRef BBs, SmallVectorImpl &NoAliasDeclScopes); +void identifyNoAliasScopesToClone(ArrayRef BBs, + SmallVectorImpl &NoAliasDeclScopes); /// Find the 'llvm.experimental.noalias.scope.decl' intrinsics in the specified /// instruction range and extract their scope. These are candidates for /// duplication when cloning. -void identifyNoAliasScopesToClone( - BasicBlock::iterator Start, BasicBlock::iterator End, - SmallVectorImpl &NoAliasDeclScopes); +void identifyNoAliasScopesToClone(BasicBlock::iterator Start, + BasicBlock::iterator End, + SmallVectorImpl &NoAliasDeclScopes); /// Duplicate the specified list of noalias decl scopes. /// The 'Ext' string is added as an extension to the name. /// Afterwards, the ClonedScopes contains the mapping of the original scope /// MDNode onto the cloned scope. /// Be aware that the cloned scopes are still part of the original scope domain. -void cloneNoAliasScopes( - ArrayRef NoAliasDeclScopes, - DenseMap &ClonedScopes, - StringRef Ext, LLVMContext &Context); +void cloneNoAliasScopes(ArrayRef NoAliasDeclScopes, + DenseMap &ClonedScopes, + StringRef Ext, LLVMContext &Context); /// Adapt the metadata for the specified instruction according to the /// provided mapping. This is normally used after cloning an instruction, when /// some noalias scopes needed to be cloned. -void adaptNoAliasScopes( - llvm::Instruction *I, const DenseMap &ClonedScopes, - LLVMContext &Context); +void adaptNoAliasScopes(llvm::Instruction *I, + const DenseMap &ClonedScopes, + LLVMContext &Context); /// Clone the specified noalias decl scopes. Then adapt all instructions in the /// NewBlocks basicblocks to the cloned versions. diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp index c967e78123af1..cf4b1c7a045e0 100644 --- a/llvm/lib/Transforms/Utils/CloneFunction.cpp +++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp @@ -216,6 +216,59 @@ void llvm::CloneFunctionMetadataInto(Function *NewFunc, const Function *OldFunc, } } +void llvm::CloneFunctionBodyInto(Function *NewFunc, const Function *OldFunc, + ValueToValueMapTy &VMap, RemapFlags RemapFlag, + SmallVectorImpl &Returns, + const char *NameSuffix, + ClonedCodeInfo *CodeInfo, + ValueMapTypeRemapper *TypeMapper, + ValueMaterializer *Materializer) { + if (OldFunc->isDeclaration()) + return; + + // Loop over all of the basic blocks in the function, cloning them as + // appropriate. Note that we save BE this way in order to handle cloning of + // recursive functions into themselves. + for (const BasicBlock &BB : *OldFunc) { + + // Create a new basic block and copy instructions into it! + BasicBlock *CBB = CloneBasicBlock(&BB, VMap, NameSuffix, NewFunc, CodeInfo); + + // Add basic block mapping. + VMap[&BB] = CBB; + + // It is only legal to clone a function if a block address within that + // function is never referenced outside of the function. Given that, we + // want to map block addresses from the old function to block addresses in + // the clone. (This is different from the generic ValueMapper + // implementation, which generates an invalid blockaddress when + // cloning a function.) + if (BB.hasAddressTaken()) { + Constant *OldBBAddr = BlockAddress::get(const_cast(OldFunc), + const_cast(&BB)); + VMap[OldBBAddr] = BlockAddress::get(NewFunc, CBB); + } + + // Note return instructions for the caller. + if (ReturnInst *RI = dyn_cast(CBB->getTerminator())) + Returns.push_back(RI); + } + + // Loop over all of the instructions in the new function, fixing up operand + // references as we go. This uses VMap to do all the hard work. + for (Function::iterator + BB = cast(VMap[&OldFunc->front()])->getIterator(), + BE = NewFunc->end(); + BB != BE; ++BB) + // Loop over all instructions, fixing each one as we find it, and any + // attached debug-info records. + for (Instruction &II : *BB) { + RemapInstruction(&II, VMap, RemapFlag, TypeMapper, Materializer); + RemapDbgRecordRange(II.getModule(), II.getDbgRecordRange(), VMap, + RemapFlag, TypeMapper, Materializer); + } +} + // Clone OldFunc into NewFunc, transforming the old arguments into references to // VMap values. void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, @@ -282,47 +335,8 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, CloneFunctionMetadataInto(NewFunc, OldFunc, VMap, RemapFlag, TypeMapper, Materializer); - // Loop over all of the basic blocks in the function, cloning them as - // appropriate. Note that we save BE this way in order to handle cloning of - // recursive functions into themselves. - for (const BasicBlock &BB : *OldFunc) { - - // Create a new basic block and copy instructions into it! - BasicBlock *CBB = CloneBasicBlock(&BB, VMap, NameSuffix, NewFunc, CodeInfo); - - // Add basic block mapping. - VMap[&BB] = CBB; - - // It is only legal to clone a function if a block address within that - // function is never referenced outside of the function. Given that, we - // want to map block addresses from the old function to block addresses in - // the clone. (This is different from the generic ValueMapper - // implementation, which generates an invalid blockaddress when - // cloning a function.) - if (BB.hasAddressTaken()) { - Constant *OldBBAddr = BlockAddress::get(const_cast(OldFunc), - const_cast(&BB)); - VMap[OldBBAddr] = BlockAddress::get(NewFunc, CBB); - } - - // Note return instructions for the caller. - if (ReturnInst *RI = dyn_cast(CBB->getTerminator())) - Returns.push_back(RI); - } - - // Loop over all of the instructions in the new function, fixing up operand - // references as we go. This uses VMap to do all the hard work. - for (Function::iterator - BB = cast(VMap[&OldFunc->front()])->getIterator(), - BE = NewFunc->end(); - BB != BE; ++BB) - // Loop over all instructions, fixing each one as we find it, and any - // attached debug-info records. - for (Instruction &II : *BB) { - RemapInstruction(&II, VMap, RemapFlag, TypeMapper, Materializer); - RemapDbgRecordRange(II.getModule(), II.getDbgRecordRange(), VMap, - RemapFlag, TypeMapper, Materializer); - } + CloneFunctionBodyInto(NewFunc, OldFunc, VMap, RemapFlag, Returns, NameSuffix, + CodeInfo, TypeMapper, Materializer); // Only update !llvm.dbg.cu for DifferentModule (not CloneModule). In the // same module, the compile unit will already be listed (or not). When From a6885e92b192c398d51f3723baccf628d3bdcb26 Mon Sep 17 00:00:00 2001 From: Artem Pianykh Date: Sat, 14 Sep 2024 16:02:51 -0700 Subject: [PATCH 4/8] [NFC][Utils] Eliminate DISubprogram set from BuildDebugInfoMDMap Summary: Previously, we'd add all SPs distinct from the cloned one into a set. Then when cloning a local scope we'd check if it's from one of those 'distinct' SPs by checking if it's in the set. We don't need to do that. We can just check against the cloned SP directly and drop the set. Test Plan: ninja check-llvm-unit check-llvm stack-info: PR: https://github.com/llvm/llvm-project/pull/118625, branch: users/artempyanykh/fast-coro-upstream/6 --- llvm/lib/Transforms/Utils/CloneFunction.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp index cf4b1c7a045e0..34400d45aa6e7 100644 --- a/llvm/lib/Transforms/Utils/CloneFunction.cpp +++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp @@ -172,18 +172,15 @@ bool llvm::BuildDebugInfoMDMap(DenseMap &MD, }; // Avoid cloning types, compile units, and (other) subprograms. - SmallPtrSet MappedToSelfSPs; for (DISubprogram *ISP : DIFinder.subprograms()) { - if (ISP != SPClonedWithinModule) { + if (ISP != SPClonedWithinModule) mapToSelfIfNew(ISP); - MappedToSelfSPs.insert(ISP); - } } // If a subprogram isn't going to be cloned skip its lexical blocks as well. for (DIScope *S : DIFinder.scopes()) { auto *LScope = dyn_cast(S); - if (LScope && MappedToSelfSPs.count(LScope->getSubprogram())) + if (LScope && LScope->getSubprogram() != SPClonedWithinModule) mapToSelfIfNew(S); } From 210eee4c7cd822b6399317ad2b6f4d5c0301c63a Mon Sep 17 00:00:00 2001 From: Artem Pianykh Date: Sun, 15 Sep 2024 04:39:20 -0700 Subject: [PATCH 5/8] [Utils] Identity map module-level debug info on first use in CloneFunction* Summary: To avoid cloning module-level debug info (owned by the module rather than the function), CloneFunction implementation used to eagerly identity map such debug info into ValueMap's MD map. In larger modules with meaningful volume of debug info this gets very expensive. By passing such debug info metadata via an IdentityMD set for the ValueMapper to map on first use, we get several benefits: 1. Mapping metadata is not cheap, particularly because of tracking. When cloning a Function we identity map lots of global module-level metadata to avoid cloning it, while only a fraction of it is actually used by the function. Mapping on first use is a lot faster for modules with meaningful amount of debug info. 2. Eagerly identity mapping metadata makes it harder to cache module-level data (e.g. a set of metadata nodes in a \a DICompileUnit). With this patch we can cache certain module-level metadata calculations to speed things up further. Anecdata from compiling a sample cpp file with full debug info shows that this moderately speeds up CoroSplitPass which is one of the heavier users of cloning: | | Baseline | IdentityMD set | |-----------------|----------|----------------| | CoroSplitPass | 306ms | 221ms | | CoroCloner | 101ms | 72ms | | Speed up | 1x | 1.4x | Test Plan: ninja check-llvm-unit ninja check-llvm stack-info: PR: https://github.com/llvm/llvm-project/pull/118627, branch: users/artempyanykh/fast-coro-upstream/8 --- llvm/include/llvm/Transforms/Utils/Cloning.h | 19 +++--- .../llvm/Transforms/Utils/ValueMapper.h | 67 ++++++++++++++----- llvm/lib/Transforms/Utils/CloneFunction.cpp | 59 ++++++++-------- llvm/lib/Transforms/Utils/ValueMapper.cpp | 19 ++++-- 4 files changed, 103 insertions(+), 61 deletions(-) diff --git a/llvm/include/llvm/Transforms/Utils/Cloning.h b/llvm/include/llvm/Transforms/Utils/Cloning.h index 50518c746d11c..9b256f9b4d689 100644 --- a/llvm/include/llvm/Transforms/Utils/Cloning.h +++ b/llvm/include/llvm/Transforms/Utils/Cloning.h @@ -192,7 +192,8 @@ void CloneFunctionAttributesInto(Function *NewFunc, const Function *OldFunc, void CloneFunctionMetadataInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, RemapFlags RemapFlag, ValueMapTypeRemapper *TypeMapper = nullptr, - ValueMaterializer *Materializer = nullptr); + ValueMaterializer *Materializer = nullptr, + const MetadataSetTy *IdentityMD = nullptr); /// Clone OldFunc's body into NewFunc. void CloneFunctionBodyInto(Function *NewFunc, const Function *OldFunc, @@ -201,7 +202,8 @@ void CloneFunctionBodyInto(Function *NewFunc, const Function *OldFunc, const char *NameSuffix = "", ClonedCodeInfo *CodeInfo = nullptr, ValueMapTypeRemapper *TypeMapper = nullptr, - ValueMaterializer *Materializer = nullptr); + ValueMaterializer *Materializer = nullptr, + const MetadataSetTy *IdentityMD = nullptr); void CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc, const Instruction *StartingInst, @@ -241,13 +243,12 @@ DISubprogram *CollectDebugInfoForCloning(const Function &F, CloneFunctionChangeType Changes, DebugInfoFinder &DIFinder); -/// Build a map of debug info to use during Metadata cloning. -/// Returns true if cloning would need module level changes and false if there -/// would only be local changes. -bool BuildDebugInfoMDMap(DenseMap &MD, - CloneFunctionChangeType Changes, - DebugInfoFinder &DIFinder, - DISubprogram *SPClonedWithinModule); +/// Based on \p Changes and \p DIFinder populate \p MD with debug info that +/// needs to be identity mapped during Metadata cloning. +void FindDebugInfoToIdentityMap(MetadataSetTy &MD, + CloneFunctionChangeType Changes, + DebugInfoFinder &DIFinder, + DISubprogram *SPClonedWithinModule); /// This class captures the data input to the InlineFunction call, and records /// the auxiliary results produced by it. diff --git a/llvm/include/llvm/Transforms/Utils/ValueMapper.h b/llvm/include/llvm/Transforms/Utils/ValueMapper.h index 743cfeb7ef3f0..b8d612f11d519 100644 --- a/llvm/include/llvm/Transforms/Utils/ValueMapper.h +++ b/llvm/include/llvm/Transforms/Utils/ValueMapper.h @@ -15,6 +15,7 @@ #define LLVM_TRANSFORMS_UTILS_VALUEMAPPER_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/simple_ilist.h" #include "llvm/IR/ValueHandle.h" #include "llvm/IR/ValueMap.h" @@ -35,6 +36,7 @@ class Value; using ValueToValueMapTy = ValueMap; using DbgRecordIterator = simple_ilist::iterator; +using MetadataSetTy = SmallPtrSet; /// This is a class that can be implemented by clients to remap types when /// cloning constants and instructions. @@ -136,6 +138,18 @@ inline RemapFlags operator|(RemapFlags LHS, RemapFlags RHS) { /// alternate \a ValueToValueMapTy and \a ValueMaterializer and returns a ID to /// pass into the schedule*() functions. /// +/// NOTE: \c IdentityMD is used by CloneFunction* to directly specify metadata +/// that should be identity mapped (and hence not cloned). The metadata will be +/// identity mapped in \c VM on first use. There are several reasons for doing +/// it this way rather than eagerly identity mapping metadata nodes in \c VM: +/// 1. Mapping metadata is not cheap, particularly because of tracking. +/// 2. When cloning a Function we identity map lots of global module-level +/// metadata to avoid cloning it, while only a fraction of it is actually +/// used by the function. Mapping on first use is a lot faster for modules +/// with meaningful amount of debug info. +/// 3. Eagerly identity mapping metadata makes it harder to cache module-level +/// data (e.g. a set of metadata nodes in a \a DICompileUnit). +/// /// TODO: lib/Linker really doesn't need the \a ValueHandle in the \a /// ValueToValueMapTy. We should template \a ValueMapper (and its /// implementation classes), and explicitly instantiate on two concrete @@ -152,7 +166,8 @@ class ValueMapper { public: ValueMapper(ValueToValueMapTy &VM, RemapFlags Flags = RF_None, ValueMapTypeRemapper *TypeMapper = nullptr, - ValueMaterializer *Materializer = nullptr); + ValueMaterializer *Materializer = nullptr, + const MetadataSetTy *IdentityMD = nullptr); ValueMapper(ValueMapper &&) = delete; ValueMapper(const ValueMapper &) = delete; ValueMapper &operator=(ValueMapper &&) = delete; @@ -218,8 +233,10 @@ class ValueMapper { inline Value *MapValue(const Value *V, ValueToValueMapTy &VM, RemapFlags Flags = RF_None, ValueMapTypeRemapper *TypeMapper = nullptr, - ValueMaterializer *Materializer = nullptr) { - return ValueMapper(VM, Flags, TypeMapper, Materializer).mapValue(*V); + ValueMaterializer *Materializer = nullptr, + const MetadataSetTy *IdentityMD = nullptr) { + return ValueMapper(VM, Flags, TypeMapper, Materializer, IdentityMD) + .mapValue(*V); } /// Lookup or compute a mapping for a piece of metadata. @@ -231,7 +248,9 @@ inline Value *MapValue(const Value *V, ValueToValueMapTy &VM, /// \c MD. /// 3. Else if \c MD is a \a ConstantAsMetadata, call \a MapValue() and /// re-wrap its return (returning nullptr on nullptr). -/// 4. Else, \c MD is an \a MDNode. These are remapped, along with their +/// 4. Else if \c MD is in \c IdentityMD then add an identity mapping for it +/// and return it. +/// 5. Else, \c MD is an \a MDNode. These are remapped, along with their /// transitive operands. Distinct nodes are duplicated or moved depending /// on \a RF_MoveDistinctNodes. Uniqued nodes are remapped like constants. /// @@ -240,16 +259,20 @@ inline Value *MapValue(const Value *V, ValueToValueMapTy &VM, inline Metadata *MapMetadata(const Metadata *MD, ValueToValueMapTy &VM, RemapFlags Flags = RF_None, ValueMapTypeRemapper *TypeMapper = nullptr, - ValueMaterializer *Materializer = nullptr) { - return ValueMapper(VM, Flags, TypeMapper, Materializer).mapMetadata(*MD); + ValueMaterializer *Materializer = nullptr, + const MetadataSetTy *IdentityMD = nullptr) { + return ValueMapper(VM, Flags, TypeMapper, Materializer, IdentityMD) + .mapMetadata(*MD); } /// Version of MapMetadata with type safety for MDNode. inline MDNode *MapMetadata(const MDNode *MD, ValueToValueMapTy &VM, RemapFlags Flags = RF_None, ValueMapTypeRemapper *TypeMapper = nullptr, - ValueMaterializer *Materializer = nullptr) { - return ValueMapper(VM, Flags, TypeMapper, Materializer).mapMDNode(*MD); + ValueMaterializer *Materializer = nullptr, + const MetadataSetTy *IdentityMD = nullptr) { + return ValueMapper(VM, Flags, TypeMapper, Materializer, IdentityMD) + .mapMDNode(*MD); } /// Convert the instruction operands from referencing the current values into @@ -263,8 +286,10 @@ inline MDNode *MapMetadata(const MDNode *MD, ValueToValueMapTy &VM, inline void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, RemapFlags Flags = RF_None, ValueMapTypeRemapper *TypeMapper = nullptr, - ValueMaterializer *Materializer = nullptr) { - ValueMapper(VM, Flags, TypeMapper, Materializer).remapInstruction(*I); + ValueMaterializer *Materializer = nullptr, + const MetadataSetTy *IdentityMD = nullptr) { + ValueMapper(VM, Flags, TypeMapper, Materializer, IdentityMD) + .remapInstruction(*I); } /// Remap the Values used in the DbgRecord \a DR using the value map \a @@ -272,8 +297,10 @@ inline void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, inline void RemapDbgRecord(Module *M, DbgRecord *DR, ValueToValueMapTy &VM, RemapFlags Flags = RF_None, ValueMapTypeRemapper *TypeMapper = nullptr, - ValueMaterializer *Materializer = nullptr) { - ValueMapper(VM, Flags, TypeMapper, Materializer).remapDbgRecord(M, *DR); + ValueMaterializer *Materializer = nullptr, + const MetadataSetTy *IdentityMD = nullptr) { + ValueMapper(VM, Flags, TypeMapper, Materializer, IdentityMD) + .remapDbgRecord(M, *DR); } /// Remap the Values used in the DbgRecords \a Range using the value map \a @@ -283,8 +310,9 @@ inline void RemapDbgRecordRange(Module *M, ValueToValueMapTy &VM, RemapFlags Flags = RF_None, ValueMapTypeRemapper *TypeMapper = nullptr, - ValueMaterializer *Materializer = nullptr) { - ValueMapper(VM, Flags, TypeMapper, Materializer) + ValueMaterializer *Materializer = nullptr, + const MetadataSetTy *IdentityMD = nullptr) { + ValueMapper(VM, Flags, TypeMapper, Materializer, IdentityMD) .remapDbgRecordRange(M, Range); } @@ -297,16 +325,19 @@ inline void RemapDbgRecordRange(Module *M, inline void RemapFunction(Function &F, ValueToValueMapTy &VM, RemapFlags Flags = RF_None, ValueMapTypeRemapper *TypeMapper = nullptr, - ValueMaterializer *Materializer = nullptr) { - ValueMapper(VM, Flags, TypeMapper, Materializer).remapFunction(F); + ValueMaterializer *Materializer = nullptr, + const MetadataSetTy *IdentityMD = nullptr) { + ValueMapper(VM, Flags, TypeMapper, Materializer, IdentityMD).remapFunction(F); } /// Version of MapValue with type safety for Constant. inline Constant *MapValue(const Constant *V, ValueToValueMapTy &VM, RemapFlags Flags = RF_None, ValueMapTypeRemapper *TypeMapper = nullptr, - ValueMaterializer *Materializer = nullptr) { - return ValueMapper(VM, Flags, TypeMapper, Materializer).mapConstant(*V); + ValueMaterializer *Materializer = nullptr, + const MetadataSetTy *IdentityMD = nullptr) { + return ValueMapper(VM, Flags, TypeMapper, Materializer, IdentityMD) + .mapConstant(*V); } } // end namespace llvm diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp index 34400d45aa6e7..057bd4fae3353 100644 --- a/llvm/lib/Transforms/Utils/CloneFunction.cpp +++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp @@ -152,64 +152,57 @@ DISubprogram *llvm::CollectDebugInfoForCloning(const Function &F, return SPClonedWithinModule; } -bool llvm::BuildDebugInfoMDMap(DenseMap &MD, - CloneFunctionChangeType Changes, - DebugInfoFinder &DIFinder, - DISubprogram *SPClonedWithinModule) { - bool ModuleLevelChanges = Changes > CloneFunctionChangeType::LocalChangesOnly; +void llvm::FindDebugInfoToIdentityMap(MetadataSetTy &MD, + CloneFunctionChangeType Changes, + DebugInfoFinder &DIFinder, + DISubprogram *SPClonedWithinModule) { if (Changes < CloneFunctionChangeType::DifferentModule && DIFinder.subprogram_count() > 0) { - // Turn on module-level changes, since we need to clone (some of) the - // debug info metadata. + // Even if Changes are local only, we turn on module-level changes, since we + // need to clone (some of) the debug info metadata. // // FIXME: Metadata effectively owned by a function should be made // local, and only that local metadata should be cloned. - ModuleLevelChanges = true; - - auto mapToSelfIfNew = [&MD](MDNode *N) { - // Avoid clobbering an existing mapping. - (void)MD.try_emplace(N, N); - }; // Avoid cloning types, compile units, and (other) subprograms. for (DISubprogram *ISP : DIFinder.subprograms()) { if (ISP != SPClonedWithinModule) - mapToSelfIfNew(ISP); + MD.insert(ISP); } // If a subprogram isn't going to be cloned skip its lexical blocks as well. for (DIScope *S : DIFinder.scopes()) { auto *LScope = dyn_cast(S); if (LScope && LScope->getSubprogram() != SPClonedWithinModule) - mapToSelfIfNew(S); + MD.insert(S); } for (DICompileUnit *CU : DIFinder.compile_units()) - mapToSelfIfNew(CU); + MD.insert(CU); for (DIType *Type : DIFinder.types()) - mapToSelfIfNew(Type); + MD.insert(Type); } else { assert(!SPClonedWithinModule && "Subprogram should be in DIFinder->subprogram_count()..."); } - - return ModuleLevelChanges; } void llvm::CloneFunctionMetadataInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, RemapFlags RemapFlag, ValueMapTypeRemapper *TypeMapper, - ValueMaterializer *Materializer) { + ValueMaterializer *Materializer, + const MetadataSetTy *IdentityMD) { // Duplicate the metadata that is attached to the cloned function. // Subprograms/CUs/types that were already mapped to themselves won't be // duplicated. SmallVector, 1> MDs; OldFunc->getAllMetadata(MDs); for (auto MD : MDs) { - NewFunc->addMetadata(MD.first, *MapMetadata(MD.second, VMap, RemapFlag, - TypeMapper, Materializer)); + NewFunc->addMetadata(MD.first, + *MapMetadata(MD.second, VMap, RemapFlag, TypeMapper, + Materializer, IdentityMD)); } } @@ -219,7 +212,8 @@ void llvm::CloneFunctionBodyInto(Function *NewFunc, const Function *OldFunc, const char *NameSuffix, ClonedCodeInfo *CodeInfo, ValueMapTypeRemapper *TypeMapper, - ValueMaterializer *Materializer) { + ValueMaterializer *Materializer, + const MetadataSetTy *IdentityMD) { if (OldFunc->isDeclaration()) return; @@ -260,9 +254,10 @@ void llvm::CloneFunctionBodyInto(Function *NewFunc, const Function *OldFunc, // Loop over all instructions, fixing each one as we find it, and any // attached debug-info records. for (Instruction &II : *BB) { - RemapInstruction(&II, VMap, RemapFlag, TypeMapper, Materializer); + RemapInstruction(&II, VMap, RemapFlag, TypeMapper, Materializer, + IdentityMD); RemapDbgRecordRange(II.getModule(), II.getDbgRecordRange(), VMap, - RemapFlag, TypeMapper, Materializer); + RemapFlag, TypeMapper, Materializer, IdentityMD); } } @@ -324,16 +319,20 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, DISubprogram *SPClonedWithinModule = CollectDebugInfoForCloning(*OldFunc, Changes, DIFinder); - ModuleLevelChanges = - BuildDebugInfoMDMap(VMap.MD(), Changes, DIFinder, SPClonedWithinModule); + MetadataSetTy IdentityMD; + FindDebugInfoToIdentityMap(IdentityMD, Changes, DIFinder, + SPClonedWithinModule); - const auto RemapFlag = ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges; + // Current implementation always upgrades from local changes to module level + // changes due to the way metadata cloning is done. See + // BuildDebugInfoToIdentityMap for more details. + const auto RemapFlag = RF_None; CloneFunctionMetadataInto(NewFunc, OldFunc, VMap, RemapFlag, TypeMapper, - Materializer); + Materializer, &IdentityMD); CloneFunctionBodyInto(NewFunc, OldFunc, VMap, RemapFlag, Returns, NameSuffix, - CodeInfo, TypeMapper, Materializer); + CodeInfo, TypeMapper, Materializer, &IdentityMD); // Only update !llvm.dbg.cu for DifferentModule (not CloneModule). In the // same module, the compile unit will already be listed (or not). When diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp index 3faea48466ba9..f6ab5bd284597 100644 --- a/llvm/lib/Transforms/Utils/ValueMapper.cpp +++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp @@ -120,12 +120,14 @@ class Mapper { SmallVector Worklist; SmallVector DelayedBBs; SmallVector AppendingInits; + const MetadataSetTy *IdentityMD; public: Mapper(ValueToValueMapTy &VM, RemapFlags Flags, - ValueMapTypeRemapper *TypeMapper, ValueMaterializer *Materializer) + ValueMapTypeRemapper *TypeMapper, ValueMaterializer *Materializer, + const MetadataSetTy *IdentityMD) : Flags(Flags), TypeMapper(TypeMapper), - MCs(1, MappingContext(VM, Materializer)) {} + MCs(1, MappingContext(VM, Materializer)), IdentityMD(IdentityMD) {} /// ValueMapper should explicitly call \a flush() before destruction. ~Mapper() { assert(!hasWorkToDo() && "Expected to be flushed"); } @@ -899,6 +901,14 @@ std::optional Mapper::mapSimpleMetadata(const Metadata *MD) { return wrapConstantAsMetadata(*CMD, mapValue(CMD->getValue())); } + // Map metadata from IdentityMD on first use. We need to add these nodes to + // the mapping as otherwise metadata nodes numbering gets messed up. This is + // still economical because the amount of data in IdentityMD may be a lot + // larger than what will actually get used. + if (IdentityMD && IdentityMD->contains(MD)) { + return getVM().MD()[MD] = TrackingMDRef(const_cast(MD)); + } + assert(isa(MD) && "Expected a metadata node"); return std::nullopt; @@ -1198,8 +1208,9 @@ class FlushingMapper { ValueMapper::ValueMapper(ValueToValueMapTy &VM, RemapFlags Flags, ValueMapTypeRemapper *TypeMapper, - ValueMaterializer *Materializer) - : pImpl(new Mapper(VM, Flags, TypeMapper, Materializer)) {} + ValueMaterializer *Materializer, + const MetadataSetTy *IdentityMD) + : pImpl(new Mapper(VM, Flags, TypeMapper, Materializer, IdentityMD)) {} ValueMapper::~ValueMapper() { delete getAsMapper(pImpl); } From a52204ccce9180681008f3761db6b4da948c78a9 Mon Sep 17 00:00:00 2001 From: Artem Pianykh Date: Tue, 19 Nov 2024 17:19:27 -0700 Subject: [PATCH 6/8] [Coro] Prebuild a module-level debug info set and share it between all coroutine clones Summary: CoroCloner, by calling into CloneFunctionInto, does a lot of repeated work priming DIFinder and building a list of common module-level debug info metadata. For programs compiled with full debug info this can get very expensive. This diff builds the data once and shares it between all clones. Anecdata for a sample cpp source file compiled with full debug info: | | Baseline | IdentityMD set | Prebuilt CommonDI (cur.) | |-----------------|----------|----------------|--------------------------| | CoroSplitPass | 306ms | 221ms | 68ms | | CoroCloner | 101ms | 72ms | 0.5ms | | CollectGlobalDI | - | - | 63ms | | Speed up | 1x | 1.4x | 4.5x | Note that CollectCommonDebugInfo happens once *per coroutine* rather than per clone. Test Plan: ninja check-llvm-unit ninja check-llvm Compiled a sample internal source file, checked time trace output for scope timings. stack-info: PR: https://github.com/llvm/llvm-project/pull/118628, branch: users/artempyanykh/fast-coro-upstream/9 --- llvm/lib/Transforms/Coroutines/CoroCloner.h | 31 +++++++----- llvm/lib/Transforms/Coroutines/CoroSplit.cpp | 50 +++++++++++++++++--- 2 files changed, 63 insertions(+), 18 deletions(-) diff --git a/llvm/lib/Transforms/Coroutines/CoroCloner.h b/llvm/lib/Transforms/Coroutines/CoroCloner.h index d1887980fb3bc..b817e55cad9fc 100644 --- a/llvm/lib/Transforms/Coroutines/CoroCloner.h +++ b/llvm/lib/Transforms/Coroutines/CoroCloner.h @@ -48,6 +48,9 @@ class BaseCloner { CloneKind FKind; IRBuilder<> Builder; TargetTransformInfo &TTI; + // Common module-level metadata that's shared between all coroutine clones and + // doesn't need to be cloned itself. + const MetadataSetTy &CommonDebugInfo; ValueToValueMapTy VMap; Function *NewF = nullptr; @@ -60,12 +63,12 @@ class BaseCloner { /// Create a cloner for a continuation lowering. BaseCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, Function *NewF, AnyCoroSuspendInst *ActiveSuspend, - TargetTransformInfo &TTI) + TargetTransformInfo &TTI, const MetadataSetTy &CommonDebugInfo) : OrigF(OrigF), Suffix(Suffix), Shape(Shape), FKind(Shape.ABI == ABI::Async ? CloneKind::Async : CloneKind::Continuation), - Builder(OrigF.getContext()), TTI(TTI), NewF(NewF), - ActiveSuspend(ActiveSuspend) { + Builder(OrigF.getContext()), TTI(TTI), CommonDebugInfo(CommonDebugInfo), + NewF(NewF), ActiveSuspend(ActiveSuspend) { assert(Shape.ABI == ABI::Retcon || Shape.ABI == ABI::RetconOnce || Shape.ABI == ABI::Async); assert(NewF && "need existing function for continuation"); @@ -74,9 +77,11 @@ class BaseCloner { public: BaseCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, - CloneKind FKind, TargetTransformInfo &TTI) + CloneKind FKind, TargetTransformInfo &TTI, + const MetadataSetTy &CommonDebugInfo) : OrigF(OrigF), Suffix(Suffix), Shape(Shape), FKind(FKind), - Builder(OrigF.getContext()), TTI(TTI) {} + Builder(OrigF.getContext()), TTI(TTI), + CommonDebugInfo(CommonDebugInfo) {} virtual ~BaseCloner() {} @@ -84,12 +89,14 @@ class BaseCloner { static Function *createClone(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, Function *NewF, AnyCoroSuspendInst *ActiveSuspend, - TargetTransformInfo &TTI) { + TargetTransformInfo &TTI, + const MetadataSetTy &CommonDebugInfo) { assert(Shape.ABI == ABI::Retcon || Shape.ABI == ABI::RetconOnce || Shape.ABI == ABI::Async); TimeTraceScope FunctionScope("BaseCloner"); - BaseCloner Cloner(OrigF, Suffix, Shape, NewF, ActiveSuspend, TTI); + BaseCloner Cloner(OrigF, Suffix, Shape, NewF, ActiveSuspend, TTI, + CommonDebugInfo); Cloner.create(); return Cloner.getFunction(); } @@ -129,8 +136,9 @@ class SwitchCloner : public BaseCloner { protected: /// Create a cloner for a switch lowering. SwitchCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, - CloneKind FKind, TargetTransformInfo &TTI) - : BaseCloner(OrigF, Suffix, Shape, FKind, TTI) {} + CloneKind FKind, TargetTransformInfo &TTI, + const MetadataSetTy &CommonDebugInfo) + : BaseCloner(OrigF, Suffix, Shape, FKind, TTI, CommonDebugInfo) {} void create() override; @@ -138,11 +146,12 @@ class SwitchCloner : public BaseCloner { /// Create a clone for a switch lowering. static Function *createClone(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, CloneKind FKind, - TargetTransformInfo &TTI) { + TargetTransformInfo &TTI, + const MetadataSetTy &CommonDebugInfo) { assert(Shape.ABI == ABI::Switch); TimeTraceScope FunctionScope("SwitchCloner"); - SwitchCloner Cloner(OrigF, Suffix, Shape, FKind, TTI); + SwitchCloner Cloner(OrigF, Suffix, Shape, FKind, TTI, CommonDebugInfo); Cloner.create(); return Cloner.getFunction(); } diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp index 3808147fc2600..377d96bdfae00 100644 --- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp @@ -43,6 +43,7 @@ #include "llvm/IR/CallingConv.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/DebugInfo.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/GlobalValue.h" @@ -77,6 +78,27 @@ using namespace llvm; #define DEBUG_TYPE "coro-split" +namespace { +/// Collect (a known) subset of global debug info metadata potentially used by +/// the function \p F. +/// +/// This metadata set can be used to avoid cloning debug info not owned by \p F +/// and is shared among all potential clones \p F. +MetadataSetTy collectCommonDebugInfo(Function &F) { + TimeTraceScope FunctionScope("CollectCommonDebugInfo"); + + MetadataSetTy CommonDebugInfo; + DebugInfoFinder DIFinder; + DISubprogram *SPClonedWithinModule = CollectDebugInfoForCloning( + F, CloneFunctionChangeType::LocalChangesOnly, DIFinder); + + FindDebugInfoToIdentityMap(CommonDebugInfo, + CloneFunctionChangeType::LocalChangesOnly, + DIFinder, SPClonedWithinModule); + return CommonDebugInfo; +} +} // end anonymous namespace + // FIXME: // Lower the intrinisc in CoroEarly phase if coroutine frame doesn't escape // and it is known that other transformations, for example, sanitizers @@ -891,8 +913,11 @@ void coro::BaseCloner::create() { auto savedLinkage = NewF->getLinkage(); NewF->setLinkage(llvm::GlobalValue::ExternalLinkage); - CloneFunctionInto(NewF, &OrigF, VMap, - CloneFunctionChangeType::LocalChangesOnly, Returns); + CloneFunctionAttributesInto(NewF, &OrigF, VMap, false); + CloneFunctionMetadataInto(NewF, &OrigF, VMap, RF_None, nullptr, nullptr, + &CommonDebugInfo); + CloneFunctionBodyInto(NewF, &OrigF, VMap, RF_None, Returns, "", nullptr, + nullptr, nullptr, &CommonDebugInfo); auto &Context = NewF->getContext(); @@ -1374,16 +1399,21 @@ struct SwitchCoroutineSplitter { TargetTransformInfo &TTI) { assert(Shape.ABI == coro::ABI::Switch); + MetadataSetTy CommonDebugInfo{collectCommonDebugInfo(F)}; + // Create a resume clone by cloning the body of the original function, // setting new entry block and replacing coro.suspend an appropriate value // to force resume or cleanup pass for every suspend point. createResumeEntryBlock(F, Shape); auto *ResumeClone = coro::SwitchCloner::createClone( - F, ".resume", Shape, coro::CloneKind::SwitchResume, TTI); + F, ".resume", Shape, coro::CloneKind::SwitchResume, TTI, + CommonDebugInfo); auto *DestroyClone = coro::SwitchCloner::createClone( - F, ".destroy", Shape, coro::CloneKind::SwitchUnwind, TTI); + F, ".destroy", Shape, coro::CloneKind::SwitchUnwind, TTI, + CommonDebugInfo); auto *CleanupClone = coro::SwitchCloner::createClone( - F, ".cleanup", Shape, coro::CloneKind::SwitchCleanup, TTI); + F, ".cleanup", Shape, coro::CloneKind::SwitchCleanup, TTI, + CommonDebugInfo); postSplitCleanup(*ResumeClone); postSplitCleanup(*DestroyClone); @@ -1768,12 +1798,15 @@ void coro::AsyncABI::splitCoroutine(Function &F, coro::Shape &Shape, } assert(Clones.size() == Shape.CoroSuspends.size()); + + MetadataSetTy CommonDebugInfo{collectCommonDebugInfo(F)}; + for (auto [Idx, CS] : llvm::enumerate(Shape.CoroSuspends)) { auto *Suspend = CS; auto *Clone = Clones[Idx]; coro::BaseCloner::createClone(F, "resume." + Twine(Idx), Shape, Clone, - Suspend, TTI); + Suspend, TTI, CommonDebugInfo); } } @@ -1899,12 +1932,15 @@ void coro::AnyRetconABI::splitCoroutine(Function &F, coro::Shape &Shape, } assert(Clones.size() == Shape.CoroSuspends.size()); + + MetadataSetTy CommonDebugInfo{collectCommonDebugInfo(F)}; + for (auto [Idx, CS] : llvm::enumerate(Shape.CoroSuspends)) { auto Suspend = CS; auto Clone = Clones[Idx]; coro::BaseCloner::createClone(F, "resume." + Twine(Idx), Shape, Clone, - Suspend, TTI); + Suspend, TTI, CommonDebugInfo); } } From 0407e73d2d49bc210d6b927ea84dd14879dcaa77 Mon Sep 17 00:00:00 2001 From: Artem Pianykh Date: Sun, 15 Sep 2024 10:51:38 -0700 Subject: [PATCH 7/8] [Analysis] Add DebugInfoCache analysis Summary: The analysis simply primes and caches DebugInfoFinders for each DICompileUnit in a module. This allows (future) callers like CoroSplitPass to compute global debug info metadata (required for coroutine function cloning) much faster. Specifically, pay the price of DICompileUnit processing only once per compile unit, rather than once per coroutine. Test Plan: Added a smoke test for the new analysis ninja check-llvm-unit check-llvm stack-info: PR: https://github.com/llvm/llvm-project/pull/118629, branch: users/artempyanykh/fast-coro-upstream/10 --- llvm/include/llvm/Analysis/DebugInfoCache.h | 50 +++++ llvm/include/llvm/IR/DebugInfo.h | 4 +- llvm/lib/Analysis/CMakeLists.txt | 1 + llvm/lib/Analysis/DebugInfoCache.cpp | 47 ++++ llvm/lib/Passes/PassBuilder.cpp | 1 + llvm/lib/Passes/PassRegistry.def | 1 + llvm/unittests/Analysis/CMakeLists.txt | 1 + .../unittests/Analysis/DebugInfoCacheTest.cpp | 211 ++++++++++++++++++ 8 files changed, 315 insertions(+), 1 deletion(-) create mode 100644 llvm/include/llvm/Analysis/DebugInfoCache.h create mode 100644 llvm/lib/Analysis/DebugInfoCache.cpp create mode 100644 llvm/unittests/Analysis/DebugInfoCacheTest.cpp diff --git a/llvm/include/llvm/Analysis/DebugInfoCache.h b/llvm/include/llvm/Analysis/DebugInfoCache.h new file mode 100644 index 0000000000000..dbd6802c99ea0 --- /dev/null +++ b/llvm/include/llvm/Analysis/DebugInfoCache.h @@ -0,0 +1,50 @@ +//===- llvm/Analysis/DebugInfoCache.h - debug info cache --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains an analysis that builds a cache of debug info for each +// DICompileUnit in a module. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_DEBUGINFOCACHE_H +#define LLVM_ANALYSIS_DEBUGINFOCACHE_H + +#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/PassManager.h" + +namespace llvm { + +/// Processes and caches debug info for each DICompileUnit in a module. +/// +/// The result of the analysis is a set of DebugInfoFinders primed on their +/// respective DICompileUnit. Such DebugInfoFinders can be used to speed up +/// function cloning which otherwise requires an expensive traversal of +/// DICompileUnit-level debug info. See an example usage in CoroSplit. +class DebugInfoCache { +public: + using DIFinderCache = SmallDenseMap; + DIFinderCache Result; + + DebugInfoCache(const Module &M); + + bool invalidate(Module &, const PreservedAnalyses &, + ModuleAnalysisManager::Invalidator &); +}; + +class DebugInfoCacheAnalysis + : public AnalysisInfoMixin { + friend AnalysisInfoMixin; + static AnalysisKey Key; + +public: + using Result = DebugInfoCache; + Result run(Module &M, ModuleAnalysisManager &); +}; +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/IR/DebugInfo.h b/llvm/include/llvm/IR/DebugInfo.h index 73f45c3769be4..11907fbb7f20b 100644 --- a/llvm/include/llvm/IR/DebugInfo.h +++ b/llvm/include/llvm/IR/DebugInfo.h @@ -120,11 +120,13 @@ class DebugInfoFinder { /// Process subprogram. void processSubprogram(DISubprogram *SP); + /// Process a compile unit. + void processCompileUnit(DICompileUnit *CU); + /// Clear all lists. void reset(); private: - void processCompileUnit(DICompileUnit *CU); void processScope(DIScope *Scope); void processType(DIType *DT); bool addCompileUnit(DICompileUnit *CU); diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt index 0db5b80f336cb..db9a569e30156 100644 --- a/llvm/lib/Analysis/CMakeLists.txt +++ b/llvm/lib/Analysis/CMakeLists.txt @@ -52,6 +52,7 @@ add_llvm_component_library(LLVMAnalysis DDGPrinter.cpp ConstraintSystem.cpp Delinearization.cpp + DebugInfoCache.cpp DemandedBits.cpp DependenceAnalysis.cpp DependenceGraphBuilder.cpp diff --git a/llvm/lib/Analysis/DebugInfoCache.cpp b/llvm/lib/Analysis/DebugInfoCache.cpp new file mode 100644 index 0000000000000..c1a3e89f0a6cc --- /dev/null +++ b/llvm/lib/Analysis/DebugInfoCache.cpp @@ -0,0 +1,47 @@ +//===- llvm/Analysis/DebugInfoCache.cpp - debug info cache ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains an analysis that builds a cache of debug info for each +// DICompileUnit in a module. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Analysis/DebugInfoCache.h" +#include "llvm/IR/Module.h" + +using namespace llvm; + +namespace { +DebugInfoFinder processCompileUnit(DICompileUnit *CU) { + DebugInfoFinder DIFinder; + DIFinder.processCompileUnit(CU); + + return DIFinder; +} +} // namespace + +DebugInfoCache::DebugInfoCache(const Module &M) { + for (const auto CU : M.debug_compile_units()) { + auto DIFinder = processCompileUnit(CU); + Result[CU] = std::move(DIFinder); + } +} + +bool DebugInfoCache::invalidate(Module &M, const PreservedAnalyses &PA, + ModuleAnalysisManager::Invalidator &) { + // Check whether the analysis has been explicitly invalidated. Otherwise, it's + // stateless and remains preserved. + auto PAC = PA.getChecker(); + return !PAC.preservedWhenStateless(); +} + +AnalysisKey DebugInfoCacheAnalysis::Key; + +DebugInfoCache DebugInfoCacheAnalysis::run(Module &M, ModuleAnalysisManager &) { + return DebugInfoCache(M); +} diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 260a34f2e060d..8046edf078c7a 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -34,6 +34,7 @@ #include "llvm/Analysis/DDGPrinter.h" #include "llvm/Analysis/DXILMetadataAnalysis.h" #include "llvm/Analysis/DXILResource.h" +#include "llvm/Analysis/DebugInfoCache.h" #include "llvm/Analysis/Delinearization.h" #include "llvm/Analysis/DemandedBits.h" #include "llvm/Analysis/DependenceAnalysis.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 2ddebb07017c2..3e47365ed80a4 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -21,6 +21,7 @@ MODULE_ANALYSIS("callgraph", CallGraphAnalysis()) MODULE_ANALYSIS("collector-metadata", CollectorMetadataAnalysis()) MODULE_ANALYSIS("ctx-prof-analysis", CtxProfAnalysis()) +MODULE_ANALYSIS("debug-info-cache", DebugInfoCacheAnalysis()) MODULE_ANALYSIS("dxil-metadata", DXILMetadataAnalysis()) MODULE_ANALYSIS("dxil-resource", DXILResourceAnalysis()) MODULE_ANALYSIS("inline-advisor", InlineAdvisorAnalysis()) diff --git a/llvm/unittests/Analysis/CMakeLists.txt b/llvm/unittests/Analysis/CMakeLists.txt index 76d16513d9341..73694a1d6ba29 100644 --- a/llvm/unittests/Analysis/CMakeLists.txt +++ b/llvm/unittests/Analysis/CMakeLists.txt @@ -25,6 +25,7 @@ set(ANALYSIS_TEST_SOURCES ConstraintSystemTest.cpp CtxProfAnalysisTest.cpp DDGTest.cpp + DebugInfoCacheTest.cpp DomTreeUpdaterTest.cpp DXILResourceTest.cpp GraphWriterTest.cpp diff --git a/llvm/unittests/Analysis/DebugInfoCacheTest.cpp b/llvm/unittests/Analysis/DebugInfoCacheTest.cpp new file mode 100644 index 0000000000000..b32e4cb543158 --- /dev/null +++ b/llvm/unittests/Analysis/DebugInfoCacheTest.cpp @@ -0,0 +1,211 @@ +//===- DebugInfoCacheTest.cpp - DebugInfoCache unit tests -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Analysis/DebugInfoCache.h" +#include "llvm/AsmParser/Parser.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Verifier.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/raw_ostream.h" +#include "gtest/gtest.h" + +namespace llvm { +namespace { + +// Forward declare the assembly +extern StringRef MultiCUModule; + +const DICompileUnit *findCU(const Module &M, StringRef FileName) { + for (const auto CU : M.debug_compile_units()) { + if (CU->getFilename() == FileName) + return CU; + } + + return nullptr; +} + +class DebugInfoCacheTest : public testing::Test { +protected: + LLVMContext C; + + std::unique_ptr makeModule(StringRef Assembly) { + SMDiagnostic Err; + auto M = parseAssemblyString(Assembly, Err, C); + if (!M) + Err.print("DebugInfoCacheTest", errs()); + + verifyModule(*M, &errs()); + return M; + } +}; + +TEST_F(DebugInfoCacheTest, TestEmpty) { + auto M = makeModule(""); + DebugInfoCache DIC{*M}; + EXPECT_EQ(DIC.Result.size(), 0u); +} + +TEST_F(DebugInfoCacheTest, TestMultiCU) { + auto M = makeModule(MultiCUModule); + DebugInfoCache DIC{*M}; + EXPECT_EQ(DIC.Result.size(), 2u); + + auto *File1CU = findCU(*M, "file1.cpp"); + EXPECT_NE(File1CU, nullptr); + + auto File1DIFinder = DIC.Result.find(File1CU); + EXPECT_NE(File1DIFinder, DIC.Result.end()); + + EXPECT_EQ(File1DIFinder->getSecond().compile_unit_count(), 1u); + EXPECT_EQ(File1DIFinder->getSecond().type_count(), 6u); + EXPECT_EQ(File1DIFinder->getSecond().subprogram_count(), 0u); + EXPECT_EQ(File1DIFinder->getSecond().scope_count(), 1u); + + auto *File2CU = findCU(*M, "file2.cpp"); + EXPECT_NE(File1CU, nullptr); + + auto File2DIFinder = DIC.Result.find(File2CU); + EXPECT_NE(File2DIFinder, DIC.Result.end()); + + EXPECT_EQ(File2DIFinder->getSecond().compile_unit_count(), 1u); + EXPECT_EQ(File2DIFinder->getSecond().type_count(), 2u); + EXPECT_EQ(File2DIFinder->getSecond().subprogram_count(), 0u); + EXPECT_EQ(File2DIFinder->getSecond().scope_count(), 2u); +} + +/* Generated roughly by +file1.cpp: +struct file1_extern_type1; +struct file1_extern_type2; + +namespace file1 { +typedef struct file1_type1 { int x; float y; } file1_type1; +file1_type1 global{0, 1.}; +} // file1 + +extern struct file1_extern_type1 *file1_extern_func1(struct +file1_extern_type2*); + +file1::file1_type1 file1_func1(file1::file1_type1 x) { return x; } +-------- +file2.cpp: +struct file2_extern_type1; +struct file2_extern_type2; + +namespace file2 { +typedef struct file2_type1 { float x; float y; } file2_type1; +enum class file2_type2 { opt1, opt2 }; + +namespace inner { +file2_type2 inner_global{file2_type2::opt2}; +} // inner +} // file2 + +extern struct file2_extern_type1 *file2_extern_func1(struct +file2_extern_type2*); + +file2::file2_type1 file2_func1(file2::file2_type1 x, file2::file2_type2 y) { +return x; } +-------- +$ clang -S -emit-llvm file*.cpp +$ llvm-link -S -o single.ll file*.ll +*/ +StringRef MultiCUModule = R"""( +%"struct.file1::file1_type1" = type { i32, float } +%"struct.file2::file2_type1" = type { float, float } + +@_ZN5file16globalE = dso_local global %"struct.file1::file1_type1" { i32 0, float 1.000000e+00 }, align 4, !dbg !0 +@_ZN5file25inner12inner_globalE = dso_local global i32 1, align 4, !dbg !11 + +define dso_local i64 @_Z11file1_func1N5file111file1_type1E(i64 %0) !dbg !33 { + %2 = alloca %"struct.file1::file1_type1", align 4 + %3 = alloca %"struct.file1::file1_type1", align 4 + store i64 %0, ptr %3, align 4 + #dbg_declare(ptr %3, !37, !DIExpression(), !38) + call void @llvm.memcpy.p0.p0.i64(ptr align 4 %2, ptr align 4 %3, i64 8, i1 false), !dbg !39 + %4 = load i64, ptr %2, align 4, !dbg !40 + ret i64 %4, !dbg !40 +} + +declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) + +define dso_local <2 x float> @_Z11file2_func1N5file211file2_type1ENS_11file2_type2E(<2 x float> %0, i32 noundef %1) !dbg !41 { + %3 = alloca %"struct.file2::file2_type1", align 4 + %4 = alloca %"struct.file2::file2_type1", align 4 + %5 = alloca i32, align 4 + store <2 x float> %0, ptr %4, align 4 + #dbg_declare(ptr %4, !49, !DIExpression(), !50) + store i32 %1, ptr %5, align 4 + #dbg_declare(ptr %5, !51, !DIExpression(), !52) + call void @llvm.memcpy.p0.p0.i64(ptr align 4 %3, ptr align 4 %4, i64 8, i1 false), !dbg !53 + %6 = load <2 x float>, ptr %3, align 4, !dbg !54 + ret <2 x float> %6, !dbg !54 +} + +!llvm.dbg.cu = !{!20, !22} +!llvm.ident = !{!25, !25} +!llvm.module.flags = !{!26, !27, !28, !29, !30, !31, !32} + +!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) +!1 = distinct !DIGlobalVariable(name: "global", linkageName: "_ZN5file16globalE", scope: !2, file: !3, line: 6, type: !4, isLocal: false, isDefinition: true) +!2 = !DINamespace(name: "file1", scope: null) +!3 = !DIFile(filename: "file1.cpp", directory: "") +!4 = !DIDerivedType(tag: DW_TAG_typedef, name: "file1_type1", scope: !2, file: !3, line: 5, baseType: !5) +!5 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "file1_type1", scope: !2, file: !3, line: 5, size: 64, flags: DIFlagTypePassByValue, elements: !6, identifier: "_ZTSN5file111file1_type1E") +!6 = !{!7, !9} +!7 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !5, file: !3, line: 5, baseType: !8, size: 32) +!8 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!9 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !5, file: !3, line: 5, baseType: !10, size: 32, offset: 32) +!10 = !DIBasicType(name: "float", size: 32, encoding: DW_ATE_float) +!11 = !DIGlobalVariableExpression(var: !12, expr: !DIExpression()) +!12 = distinct !DIGlobalVariable(name: "inner_global", linkageName: "_ZN5file25inner12inner_globalE", scope: !13, file: !15, line: 9, type: !16, isLocal: false, isDefinition: true) +!13 = !DINamespace(name: "inner", scope: !14) +!14 = !DINamespace(name: "file2", scope: null) +!15 = !DIFile(filename: "file2.cpp", directory: "") +!16 = distinct !DICompositeType(tag: DW_TAG_enumeration_type, name: "file2_type2", scope: !14, file: !15, line: 6, baseType: !8, size: 32, flags: DIFlagEnumClass, elements: !17, identifier: "_ZTSN5file211file2_type2E") +!17 = !{!18, !19} +!18 = !DIEnumerator(name: "opt1", value: 0) +!19 = !DIEnumerator(name: "opt2", value: 1) +!20 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !3, isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, globals: !21, splitDebugInlining: false, nameTableKind: None) +!21 = !{!0} +!22 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !15, isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !23, globals: !24, splitDebugInlining: false, nameTableKind: None) +!23 = !{!16} +!24 = !{!11} +!25 = !{!"clang"} +!26 = !{i32 7, !"Dwarf Version", i32 5} +!27 = !{i32 2, !"Debug Info Version", i32 3} +!28 = !{i32 1, !"wchar_size", i32 4} +!29 = !{i32 8, !"PIC Level", i32 2} +!30 = !{i32 7, !"PIE Level", i32 2} +!31 = !{i32 7, !"uwtable", i32 2} +!32 = !{i32 7, !"frame-pointer", i32 2} +!33 = distinct !DISubprogram(name: "file1_func1", linkageName: "_Z11file1_func1N5file111file1_type1E", scope: !3, file: !3, line: 11, type: !34, scopeLine: 11, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !20, retainedNodes: !36) +!34 = !DISubroutineType(types: !35) +!35 = !{!4, !4} +!36 = !{} +!37 = !DILocalVariable(name: "x", arg: 1, scope: !33, file: !3, line: 11, type: !4) +!38 = !DILocation(line: 11, column: 51, scope: !33) +!39 = !DILocation(line: 11, column: 63, scope: !33) +!40 = !DILocation(line: 11, column: 56, scope: !33) +!41 = distinct !DISubprogram(name: "file2_func1", linkageName: "_Z11file2_func1N5file211file2_type1ENS_11file2_type2E", scope: !15, file: !15, line: 15, type: !42, scopeLine: 15, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !22, retainedNodes: !36) +!42 = !DISubroutineType(types: !43) +!43 = !{!44, !44, !16} +!44 = !DIDerivedType(tag: DW_TAG_typedef, name: "file2_type1", scope: !14, file: !15, line: 5, baseType: !45) +!45 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "file2_type1", scope: !14, file: !15, line: 5, size: 64, flags: DIFlagTypePassByValue, elements: !46, identifier: "_ZTSN5file211file2_type1E") +!46 = !{!47, !48} +!47 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !45, file: !15, line: 5, baseType: !10, size: 32) +!48 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !45, file: !15, line: 5, baseType: !10, size: 32, offset: 32) +!49 = !DILocalVariable(name: "x", arg: 1, scope: !41, file: !15, line: 15, type: !44) +!50 = !DILocation(line: 15, column: 51, scope: !41) +!51 = !DILocalVariable(name: "y", arg: 2, scope: !41, file: !15, line: 15, type: !16) +!52 = !DILocation(line: 15, column: 73, scope: !41) +!53 = !DILocation(line: 15, column: 85, scope: !41) +!54 = !DILocation(line: 15, column: 78, scope: !41) +)"""; +} // namespace +} // namespace llvm From fa73123f506fbac7e7554384c48c31210187ad77 Mon Sep 17 00:00:00 2001 From: Artem Pianykh Date: Sun, 15 Sep 2024 11:00:00 -0700 Subject: [PATCH 8/8] [Coro] Use DebugInfoCache to speed up cloning in CoroSplitPass Summary: We can use a DebugInfoFinder from DebugInfoCache which is already primed on a compile unit to speed up collection of module-level debug info. The pass could likely be another 2x+ faster if we avoid rebuilding the set of global debug info. This needs further massaging of CloneFunction and ValueMapper, though, and can be done incrementally on top of this. Comparing performance of CoroSplitPass at various points in this stack, this is anecdata from a sample cpp file compiled with full debug info: | | Baseline | IdentityMD set | Prebuilt CommonDI | Cached CU DIFinder (cur.) | |-----------------|----------|----------------|-------------------|---------------------------| | CoroSplitPass | 306ms | 221ms | 68ms | 17ms | | CoroCloner | 101ms | 72ms | 0.5ms | 0.5ms | | CollectGlobalDI | - | - | 63ms | 13ms | | Speed up | 1x | 1.4x | 4.5x | 18x | Test Plan: ninja check-llvm-unit ninja check-llvm Compiled a sample cpp file with time trace to get the avg. duration of the pass and inner scopes. stack-info: PR: https://github.com/llvm/llvm-project/pull/118630, branch: users/artempyanykh/fast-coro-upstream/11 --- llvm/include/llvm/Transforms/Coroutines/ABI.h | 13 +++-- llvm/lib/Analysis/CGSCCPassManager.cpp | 7 +++ llvm/lib/Transforms/Coroutines/CoroSplit.cpp | 55 +++++++++++++++---- llvm/test/Other/new-pass-manager.ll | 1 + llvm/test/Other/new-pm-defaults.ll | 1 + llvm/test/Other/new-pm-lto-defaults.ll | 1 + llvm/test/Other/new-pm-pgo-preinline.ll | 1 + .../Other/new-pm-thinlto-postlink-defaults.ll | 1 + .../new-pm-thinlto-postlink-pgo-defaults.ll | 1 + ...-pm-thinlto-postlink-samplepgo-defaults.ll | 1 + .../Other/new-pm-thinlto-prelink-defaults.ll | 1 + .../new-pm-thinlto-prelink-pgo-defaults.ll | 1 + ...w-pm-thinlto-prelink-samplepgo-defaults.ll | 1 + .../Analysis/CGSCCPassManagerTest.cpp | 4 +- 14 files changed, 72 insertions(+), 17 deletions(-) diff --git a/llvm/include/llvm/Transforms/Coroutines/ABI.h b/llvm/include/llvm/Transforms/Coroutines/ABI.h index 0b2d405f3caec..2cf614b6bb1e2 100644 --- a/llvm/include/llvm/Transforms/Coroutines/ABI.h +++ b/llvm/include/llvm/Transforms/Coroutines/ABI.h @@ -15,6 +15,7 @@ #ifndef LLVM_TRANSFORMS_COROUTINES_ABI_H #define LLVM_TRANSFORMS_COROUTINES_ABI_H +#include "llvm/Analysis/DebugInfoCache.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Transforms/Coroutines/CoroShape.h" #include "llvm/Transforms/Coroutines/MaterializationUtils.h" @@ -53,7 +54,8 @@ class BaseABI { // Perform the function splitting according to the ABI. virtual void splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl &Clones, - TargetTransformInfo &TTI) = 0; + TargetTransformInfo &TTI, + const DebugInfoCache *DICache) = 0; Function &F; coro::Shape &Shape; @@ -73,7 +75,8 @@ class SwitchABI : public BaseABI { void splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl &Clones, - TargetTransformInfo &TTI) override; + TargetTransformInfo &TTI, + const DebugInfoCache *DICache) override; }; class AsyncABI : public BaseABI { @@ -86,7 +89,8 @@ class AsyncABI : public BaseABI { void splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl &Clones, - TargetTransformInfo &TTI) override; + TargetTransformInfo &TTI, + const DebugInfoCache *DICache) override; }; class AnyRetconABI : public BaseABI { @@ -99,7 +103,8 @@ class AnyRetconABI : public BaseABI { void splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl &Clones, - TargetTransformInfo &TTI) override; + TargetTransformInfo &TTI, + const DebugInfoCache *DICache) override; }; } // end namespace coro diff --git a/llvm/lib/Analysis/CGSCCPassManager.cpp b/llvm/lib/Analysis/CGSCCPassManager.cpp index 948bc2435ab27..3ba085cdb0be8 100644 --- a/llvm/lib/Analysis/CGSCCPassManager.cpp +++ b/llvm/lib/Analysis/CGSCCPassManager.cpp @@ -14,6 +14,7 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator_range.h" +#include "llvm/Analysis/DebugInfoCache.h" #include "llvm/Analysis/LazyCallGraph.h" #include "llvm/IR/Constant.h" #include "llvm/IR/InstIterator.h" @@ -139,6 +140,11 @@ ModuleToPostOrderCGSCCPassAdaptor::run(Module &M, ModuleAnalysisManager &AM) { // Get the call graph for this module. LazyCallGraph &CG = AM.getResult(M); + // Prime DebugInfoCache. + // TODO: Currently, the only user is CoroSplitPass. Consider running + // conditionally. + AM.getResult(M); + // Get Function analysis manager from its proxy. FunctionAnalysisManager &FAM = AM.getCachedResult(M)->getManager(); @@ -350,6 +356,7 @@ ModuleToPostOrderCGSCCPassAdaptor::run(Module &M, ModuleAnalysisManager &AM) { // analysis proxies by handling them above and in any nested pass managers. PA.preserveSet>(); PA.preserve(); + PA.preserve(); PA.preserve(); PA.preserve(); return PA; diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp index 377d96bdfae00..32651fe6fbd92 100644 --- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp @@ -32,6 +32,7 @@ #include "llvm/Analysis/CFG.h" #include "llvm/Analysis/CallGraph.h" #include "llvm/Analysis/ConstantFolding.h" +#include "llvm/Analysis/DebugInfoCache.h" #include "llvm/Analysis/LazyCallGraph.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/TargetTransformInfo.h" @@ -79,16 +80,39 @@ using namespace llvm; #define DEBUG_TYPE "coro-split" namespace { +const DebugInfoFinder *cachedDIFinder(Function &F, + const DebugInfoCache *DICache) { + if (!DICache) + return nullptr; + + auto *SP = F.getSubprogram(); + auto *CU = SP ? SP->getUnit() : nullptr; + if (!CU) + return nullptr; + + if (auto Found = DICache->Result.find(CU); Found != DICache->Result.end()) + return &Found->getSecond(); + + return nullptr; +} + /// Collect (a known) subset of global debug info metadata potentially used by /// the function \p F. /// /// This metadata set can be used to avoid cloning debug info not owned by \p F /// and is shared among all potential clones \p F. -MetadataSetTy collectCommonDebugInfo(Function &F) { +MetadataSetTy collectCommonDebugInfo(Function &F, + const DebugInfoCache *DICache) { TimeTraceScope FunctionScope("CollectCommonDebugInfo"); MetadataSetTy CommonDebugInfo; DebugInfoFinder DIFinder; + + // Copy DIFinder from cache which is primed on F's compile unit when available + auto *PrimedDIFinder = cachedDIFinder(F, DICache); + if (PrimedDIFinder) + DIFinder = *PrimedDIFinder; + DISubprogram *SPClonedWithinModule = CollectDebugInfoForCloning( F, CloneFunctionChangeType::LocalChangesOnly, DIFinder); @@ -1396,10 +1420,10 @@ namespace { struct SwitchCoroutineSplitter { static void split(Function &F, coro::Shape &Shape, SmallVectorImpl &Clones, - TargetTransformInfo &TTI) { + TargetTransformInfo &TTI, const DebugInfoCache *DICache) { assert(Shape.ABI == coro::ABI::Switch); - MetadataSetTy CommonDebugInfo{collectCommonDebugInfo(F)}; + MetadataSetTy CommonDebugInfo{collectCommonDebugInfo(F, DICache)}; // Create a resume clone by cloning the body of the original function, // setting new entry block and replacing coro.suspend an appropriate value @@ -1713,7 +1737,8 @@ CallInst *coro::createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, void coro::AsyncABI::splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl &Clones, - TargetTransformInfo &TTI) { + TargetTransformInfo &TTI, + const DebugInfoCache *DICache) { assert(Shape.ABI == coro::ABI::Async); assert(Clones.empty()); // Reset various things that the optimizer might have decided it @@ -1799,7 +1824,7 @@ void coro::AsyncABI::splitCoroutine(Function &F, coro::Shape &Shape, assert(Clones.size() == Shape.CoroSuspends.size()); - MetadataSetTy CommonDebugInfo{collectCommonDebugInfo(F)}; + MetadataSetTy CommonDebugInfo{collectCommonDebugInfo(F, DICache)}; for (auto [Idx, CS] : llvm::enumerate(Shape.CoroSuspends)) { auto *Suspend = CS; @@ -1812,7 +1837,8 @@ void coro::AsyncABI::splitCoroutine(Function &F, coro::Shape &Shape, void coro::AnyRetconABI::splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl &Clones, - TargetTransformInfo &TTI) { + TargetTransformInfo &TTI, + const DebugInfoCache *DICache) { assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce); assert(Clones.empty()); @@ -1933,7 +1959,7 @@ void coro::AnyRetconABI::splitCoroutine(Function &F, coro::Shape &Shape, assert(Clones.size() == Shape.CoroSuspends.size()); - MetadataSetTy CommonDebugInfo{collectCommonDebugInfo(F)}; + MetadataSetTy CommonDebugInfo{collectCommonDebugInfo(F, DICache)}; for (auto [Idx, CS] : llvm::enumerate(Shape.CoroSuspends)) { auto Suspend = CS; @@ -1987,13 +2013,15 @@ static bool hasSafeElideCaller(Function &F) { void coro::SwitchABI::splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl &Clones, - TargetTransformInfo &TTI) { - SwitchCoroutineSplitter::split(F, Shape, Clones, TTI); + TargetTransformInfo &TTI, + const DebugInfoCache *DICache) { + SwitchCoroutineSplitter::split(F, Shape, Clones, TTI, DICache); } static void doSplitCoroutine(Function &F, SmallVectorImpl &Clones, coro::BaseABI &ABI, TargetTransformInfo &TTI, - bool OptimizeFrame) { + bool OptimizeFrame, + const DebugInfoCache *DICache) { PrettyStackTraceFunction prettyStackTrace(F); auto &Shape = ABI.Shape; @@ -2018,7 +2046,7 @@ static void doSplitCoroutine(Function &F, SmallVectorImpl &Clones, if (isNoSuspendCoroutine) { handleNoSuspendCoroutine(Shape); } else { - ABI.splitCoroutine(F, Shape, Clones, TTI); + ABI.splitCoroutine(F, Shape, Clones, TTI, DICache); } // Replace all the swifterror operations in the original function. @@ -2215,6 +2243,9 @@ PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C, auto &FAM = AM.getResult(C, CG).getManager(); + const auto &MAMProxy = AM.getResult(C, CG); + const auto *DICache = MAMProxy.getCachedResult(M); + // Check for uses of llvm.coro.prepare.retcon/async. SmallVector PrepareFns; addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon"); @@ -2251,7 +2282,7 @@ PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C, SmallVector Clones; auto &TTI = FAM.getResult(F); - doSplitCoroutine(F, Clones, *ABI, TTI, OptimizeFrame); + doSplitCoroutine(F, Clones, *ABI, TTI, OptimizeFrame, DICache); CurrentSCC = &updateCallGraphAfterCoroutineSplit( *N, Shape, Clones, *CurrentSCC, CG, AM, UR, FAM); diff --git a/llvm/test/Other/new-pass-manager.ll b/llvm/test/Other/new-pass-manager.ll index f0fe708806f1b..53fd6fe2a317e 100644 --- a/llvm/test/Other/new-pass-manager.ll +++ b/llvm/test/Other/new-pass-manager.ll @@ -23,6 +23,7 @@ ; CHECK-CGSCC-PASS-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*(FunctionAnalysisManager|AnalysisManager<.*Function.*>).*}},{{.*}}Module> ; CHECK-CGSCC-PASS-NEXT: Running analysis: LazyCallGraphAnalysis ; CHECK-CGSCC-PASS-NEXT: Running analysis: TargetLibraryAnalysis +; CHECK-CGSCC-PASS-NEXT: Running analysis: DebugInfoCacheAnalysis ; CHECK-CGSCC-PASS-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy ; CHECK-CGSCC-PASS-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}> ; CHECK-CGSCC-PASS-NEXT: Running pass: NoOpCGSCCPass diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll index 7cf035b0c6f37..19bfec3ab718e 100644 --- a/llvm/test/Other/new-pm-defaults.ll +++ b/llvm/test/Other/new-pm-defaults.ll @@ -139,6 +139,7 @@ ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis +; CHECK-O-NEXT: Running analysis: DebugInfoCacheAnalysis ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}> ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll index f788db1e338a1..8f4fa763b5209 100644 --- a/llvm/test/Other/new-pm-lto-defaults.ll +++ b/llvm/test/Other/new-pm-lto-defaults.ll @@ -43,6 +43,7 @@ ; CHECK-O23SZ-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis ; CHECK-O23SZ-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}SCC ; CHECK-O23SZ-NEXT: Running analysis: LazyCallGraphAnalysis +; CHECK-O23SZ-NEXT: Running analysis: DebugInfoCacheAnalysis ; CHECK-O23SZ-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy ; CHECK-O23SZ-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph{{.*}}> ; CHECK-O23SZ-NEXT: Running pass: PostOrderFunctionAttrsPass diff --git a/llvm/test/Other/new-pm-pgo-preinline.ll b/llvm/test/Other/new-pm-pgo-preinline.ll index f07a3728ba3d4..97813bb243364 100644 --- a/llvm/test/Other/new-pm-pgo-preinline.ll +++ b/llvm/test/Other/new-pm-pgo-preinline.ll @@ -5,6 +5,7 @@ ; CHECK-Osz-NEXT: Running analysis: InlineAdvisorAnalysis ; CHECK-Osz-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-Osz-NEXT: Running analysis: LazyCallGraphAnalysis +; CHECK-Osz-NEXT: Running analysis: DebugInfoCacheAnalysis ; CHECK-Osz-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy on (foo) ; CHECK-Osz-NEXT: Running analysis: OuterAnalysisManagerProxy ; CHECK-Osz-NEXT: Running pass: InlinerPass on (foo) diff --git a/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll index ed13402e1c4b1..e1ad86015fda9 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll @@ -74,6 +74,7 @@ ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis +; CHECK-O-NEXT: Running analysis: DebugInfoCacheAnalysis ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass diff --git a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll index c82c34f7ff01e..3f6c5351e8e8e 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll @@ -62,6 +62,7 @@ ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis +; CHECK-O-NEXT: Running analysis: DebugInfoCacheAnalysis ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}> ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass diff --git a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll index d375747547d61..371dde305b099 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll @@ -71,6 +71,7 @@ ; CHECK-O-NEXT: Invalidating analysis: AAManager ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy +; CHECK-O-NEXT: Running analysis: DebugInfoCacheAnalysis ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass diff --git a/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll index 5aacd26def2be..860fb99525030 100644 --- a/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll @@ -106,6 +106,7 @@ ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis +; CHECK-O-NEXT: Running analysis: DebugInfoCacheAnalysis ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass diff --git a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll index f6a9406596803..d97cc97169b56 100644 --- a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll @@ -62,6 +62,7 @@ ; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis +; CHECK-O-NEXT: Running analysis: DebugInfoCacheAnalysis ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy on (foo) ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy ; CHECK-O-NEXT: Running pass: InlinerPass on (foo) diff --git a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll index 48a9433d24999..d338817d07646 100644 --- a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll @@ -76,6 +76,7 @@ ; CHECK-O-NEXT: Invalidating analysis: AAManager ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy +; CHECK-O-NEXT: Running analysis: DebugInfoCacheAnalysis ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}> ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass diff --git a/llvm/unittests/Analysis/CGSCCPassManagerTest.cpp b/llvm/unittests/Analysis/CGSCCPassManagerTest.cpp index 5c71bc8063d6c..7212107d99263 100644 --- a/llvm/unittests/Analysis/CGSCCPassManagerTest.cpp +++ b/llvm/unittests/Analysis/CGSCCPassManagerTest.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/CGSCCPassManager.h" +#include "llvm/Analysis/DebugInfoCache.h" #include "llvm/Analysis/LazyCallGraph.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/AsmParser/Parser.h" @@ -16,8 +17,8 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" -#include "llvm/IR/PassManager.h" #include "llvm/IR/PassInstrumentation.h" +#include "llvm/IR/PassManager.h" #include "llvm/IR/Verifier.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Transforms/Utils/CallGraphUpdater.h" @@ -255,6 +256,7 @@ class CGSCCPassManagerTest : public ::testing::Test { "}\n")) { FAM.registerPass([&] { return TargetLibraryAnalysis(); }); MAM.registerPass([&] { return LazyCallGraphAnalysis(); }); + MAM.registerPass([&] { return DebugInfoCacheAnalysis(); }); MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); }); // Register required pass instrumentation analysis.