From 9170b6418fb6fd9446cdca03eddb15653dad943b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= Date: Fri, 10 Jan 2025 13:43:25 +0100 Subject: [PATCH] [SPIR-V] Add pass to fixup global variable AS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SPIR-V has storage classes, which behave similarly to LLVM's address spaces. At the HLSL language level, this concept is not well defined, and some operations with no SPIR-V equivalent might be valid, for example: ``` static int global; int& get_ref(int x, int &local) { return x ? global : local; } void main() { int local; int& ref = get_ref(0, local); } ``` In this example, the function `get_ref` cannot be emitted in SPIR-V because the return value is a pointer with either the Function or the Private storage class. A solution for this is to force inlining the function, and then fixup the pointer address spaces, which are then lowered into the SPIR-V storage class. This is however impossible when building a SPIR-V library. The real solution is to make sure we use a single address space for both globals and locals, meaning we must make all variable local, or vis-versa. Moving all variables to the local scope is not possible, once again because of the library target. The last solution is to move all variables to the global scope. This is possible only because Vulkan disallow static recursion. This commit adds a new pass to handle those fixups in the backend. It works at the IR level, and moves all local variables to the global scope. It also rewrite all pointer to the default AS to `ptr addrspace(10)`. Function address spaces are left untouched: we have no indirect jump, hence we have no pointer to code, only data. Signed-off-by: Nathan Gauër --- llvm/lib/Target/SPIRV/CMakeLists.txt | 1 + llvm/lib/Target/SPIRV/SPIRV.h | 1 + .../lib/Target/SPIRV/SPIRVFixAddressSpace.cpp | 593 ++++++++++++++++++ llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp | 8 + .../SPIRV/hlsl-intrinsics/WaveGetLaneIndex.ll | 4 +- .../CodeGen/SPIRV/logical-access-chain.ll | 15 +- .../CodeGen/SPIRV/logical-struct-access.ll | 12 +- .../SPIRV/passes/SPIRVFixAddressSpace-call.ll | 31 + .../SPIRVFixAddressSpace-initialization.ll | 25 + .../passes/SPIRVFixAddressSpace-object.ll | 60 ++ .../passes/SPIRVFixAddressSpace-ptr-ptr.ll | 31 + .../SPIRVFixAddressSpace-simple-local.ll | 42 ++ .../passes/SPIRVFixAddressSpace-simple.ll | 30 + .../CodeGen/SPIRV/structurizer/basic-phi.ll | 2 +- .../CodeGen/SPIRV/structurizer/cf.cond-op.ll | 38 +- .../SPIRV/structurizer/cf.for.plain.ll | 14 +- .../SPIRV/structurizer/condition-linear.ll | 4 +- .../CodeGen/SPIRV/structurizer/logical-or.ll | 9 +- .../SPIRV/structurizer/loop-continue-split.ll | 9 +- .../SPIRV/structurizer/merge-exit-break.ll | 5 +- .../merge-exit-convergence-in-break.ll | 4 +- .../structurizer/merge-exit-multiple-break.ll | 6 +- .../SPIRV/structurizer/return-early.ll | 4 +- .../SPIRV/transcoding/GlobalFunAnnotate.ll | 8 +- 24 files changed, 903 insertions(+), 53 deletions(-) create mode 100644 llvm/lib/Target/SPIRV/SPIRVFixAddressSpace.cpp create mode 100644 llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-call.ll create mode 100644 llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-initialization.ll create mode 100644 llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-object.ll create mode 100644 llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-ptr-ptr.ll create mode 100644 llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-simple-local.ll create mode 100644 llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-simple.ll diff --git a/llvm/lib/Target/SPIRV/CMakeLists.txt b/llvm/lib/Target/SPIRV/CMakeLists.txt index efdd8c8d24fbd..2f6870195581f 100644 --- a/llvm/lib/Target/SPIRV/CMakeLists.txt +++ b/llvm/lib/Target/SPIRV/CMakeLists.txt @@ -34,6 +34,7 @@ add_llvm_target(SPIRVCodeGen SPIRVMetadata.cpp SPIRVModuleAnalysis.cpp SPIRVStructurizer.cpp + SPIRVFixAddressSpace.cpp SPIRVPreLegalizer.cpp SPIRVPreLegalizerCombiner.cpp SPIRVPostLegalizer.cpp diff --git a/llvm/lib/Target/SPIRV/SPIRV.h b/llvm/lib/Target/SPIRV/SPIRV.h index 6d00a046ff7ca..2353d04b3df08 100644 --- a/llvm/lib/Target/SPIRV/SPIRV.h +++ b/llvm/lib/Target/SPIRV/SPIRV.h @@ -20,6 +20,7 @@ class InstructionSelector; class RegisterBankInfo; ModulePass *createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM); +ModulePass *createSPIRVFixAddressSpacePass(); FunctionPass *createSPIRVStructurizerPass(); FunctionPass *createSPIRVMergeRegionExitTargetsPass(); FunctionPass *createSPIRVStripConvergenceIntrinsicsPass(); diff --git a/llvm/lib/Target/SPIRV/SPIRVFixAddressSpace.cpp b/llvm/lib/Target/SPIRV/SPIRVFixAddressSpace.cpp new file mode 100644 index 0000000000000..80bc60d917ab9 --- /dev/null +++ b/llvm/lib/Target/SPIRV/SPIRVFixAddressSpace.cpp @@ -0,0 +1,593 @@ +//===-- SPIRVFixAddressSpace.cpp ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// This pass is Vulkan specific as Logical SPIR-V doesn't support pointer +// cast. +// +// In LLVM IR, global and local variables are by default in the same address +// space. In SPIR-V, global and local variables live in a different address +// space (storage class in the SPIR-V parlance). +// +// This means the following function cannot be lowered to SPIR-V: +// static int global = 0; +// int& return_ref(int& ref, bool select) { +// return select ? ref : global; +// } +// +// A solution is to force inline, but this would prevent us from emitting SPIR-V +// libraries. Another solution is to move all globals to local variables, but +// this also blocks libraries. The last solution is to replace all local +// variables with global variables. This is possible because Vulkan SPIR-V +// completely forbids static recursion. +// +// This pass replace all alloca instruction with a new global variable. +// In addition, it moves all such allocations into the `Private` address space. +// +// After this pass, no variable or pointer should reference the default address +// space. +// +// Note: +// LLVM IR has address spaces for functions, but SPIR-V doesn't. In addition, +// Vulkan disallow function pointers and indirect jump, meaning we could never +// have a pointer storing the function address. For this reason, functions are +// left in the default address space, but all pointer operands to the default +// AS are rewritten to point to the AS `Private`. This kind of blind rewrite +// simplifies the code, but can only work with those assumptions. +//===----------------------------------------------------------------------===// + +#include "Analysis/SPIRVConvergenceRegionAnalysis.h" +#include "SPIRV.h" +#include "SPIRVSubtarget.h" +#include "SPIRVTargetMachine.h" +#include "SPIRVUtils.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/TypeSwitch.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/CodeGen/IntrinsicLowering.h" +#include "llvm/IR/CFG.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/InstVisitor.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsSPIRV.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/NoFolder.h" +#include "llvm/InitializePasses.h" +#include "llvm/PassRegistry.h" +#include "llvm/Transforms/Utils.h" +#include "llvm/Transforms/Utils/Cloning.h" +#include "llvm/Transforms/Utils/LoopSimplify.h" +#include "llvm/Transforms/Utils/LowerMemIntrinsics.h" +#include +#include +#include +#include + +using namespace llvm; +using namespace SPIRV; + +namespace llvm { +void initializeSPIRVFixAddressSpacePass(PassRegistry &); +} // namespace llvm + +namespace { + +constexpr unsigned GlobalAddressSpace = + storageClassToAddressSpace(SPIRV::StorageClass::Private); + +// Returns true if the given type or any subtype contains a pointer to the +// default address space. +bool typeRequiresConversion(Type *T) { + if (T->isPointerTy()) + return T->getPointerAddressSpace() == 0; + + if (T->isArrayTy()) + return typeRequiresConversion(T->getArrayElementType()); + if (T->isStructTy()) { + for (unsigned I = 0; I < T->getStructNumElements(); ++I) + if (typeRequiresConversion(T->getStructElementType(I))) + return true; + return false; + } + if (FunctionType *FT = dyn_cast(T)) { + if (typeRequiresConversion(FT->getReturnType())) + return true; + for (Type *TP : FT->params()) + if (typeRequiresConversion(TP)) + return true; + return false; + } + + return false; +} + +class SPIRVFixAddressSpace : public InstVisitor, + public ModulePass { + + // Types are supposed to be unique. When using Type::get, there is a lookup to + // only create types on demand. StructType::get only allows creating literal + // structs, meaning we would lose the type name. This forces us to use + // StructType::create, which doesn't deduplicates. This means we must bring + // our own old-type/new-type map to prevent creating distinct types when we + // shouldn't. + std::unordered_map ConvertedTypes; + +private: + // If the passed type or subtype contains a pointer to AS 0, get or create a + // new type with all pointer changed to address space `Private`. The functions + // below are overloads depending on the input type. + PointerType *convertPointerType(PointerType *PT); + ArrayType *convertArrayType(ArrayType *AT); + StructType *convertStructType(StructType *ST); + VectorType *convertVectorType(VectorType *VT); + FunctionType *convertFunctionType(FunctionType *FT); + + // Get or create a new type if `T` or any of its subtype is a pointer to the + // address space 0. All pointers in the returned type points to the address + // space `Private`. If `T` was already converted once, the cached converted + // type is returned and no additional type is created. + Type *convertType(Type *T); + + // If `C` type or subtype contains any pointer to the address space 0, returns + // a new constant with a fixed type. + Constant *convertConstant(Constant *C); + + // See `convertConstant`. Those functions are overloads to handle specific + // constant types. + Constant *convertConstantAggregate(ConstantAggregate *CA); + ConstantData *convertConstantData(ConstantData *CD); + + // If the passes global variable is in the default address space, replace it + // with a global in the `Private` address space (=SPIR-V storage class). Does + // not modify globals in a different address space (resources for ex). Returns + // true if the global was replaced. + bool rewriteGlobalVariable(Module &M, GlobalVariable *GV); + + // Modifies the given function by replacing all alloca by a global variable. + // This function requires the alloca allocation size to be static: + // - Vulkan doesn't support VLA in local variables. (See + // VUID-StandaloneSpirv-OpTypeRuntimeArray-04680). + // - HLSL doesn't allow VLA. + // Returns true if the function was modified. + bool replaceAlloca(Function &F); + + // Mutate the types and operands of `F` to make sure no referenced type has a + // pointer to the default address space. This function does not propagate the + // type changes, hence if not used carefully, this could generate invalid IR. + // Returns true if the function was modified. + bool blindlyMutateTypes(Function &F); + + // Modifies any GEP instruction in the given function to only use + // ptr addrspace(10) instead of pointers to the default address space. + // Returns true if the function was modified. + bool rewriteGEP(Function &F); + + // Checks all instructions in F, and make sure no pointer to the default + // address space or local variable remains. This function assumes all other + // functions/globals undergo the same treatment. Not calling this function on + // all the module functions could yield to invalid IR. Returns true if the + // function has been modified. + bool fixInstructions(Function &F); + + // Checks if any type/subtype in the return value or a parameter is a ptr to + // the default address space. If such pointer is found, recreate the function + // replacing it with a `ptr addrspace(10)`. This function replaces all uses of + // the function return value/argument/declaration with the new version, but + // does not propagate changes further. If the rest of the instruction is not + // cleaned-up, this can produce invalid IR. Returns true if the function has + // been replaced. + bool rewriteFunctionParameters(Module &M, Function *F); + + // Fix all functions in the given module. + // Returns true if any of the module functions have been modified. + // If the module is modified, new global variables could have been added. + // This function only modified global variables referenced by at least one + // function. Returns true if the module was modified. + bool fixFunctions(Module &M); + + // Fix all the globals in the given module, even if not referenced by any + // function. + bool fixGlobals(Module &M); + +public: + static char ID; + + SPIRVFixAddressSpace() : ModulePass(ID) { + initializeSPIRVFixAddressSpacePass(*PassRegistry::getPassRegistry()); + }; + + virtual bool runOnModule(Module &M) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + ModulePass::getAnalysisUsage(AU); + } +}; + +PointerType *SPIRVFixAddressSpace::convertPointerType(PointerType *PT) { + if (PT->getPointerAddressSpace() != 0) + return PT; + + auto It = ConvertedTypes.find(PT); + if (It != ConvertedTypes.end()) + return cast(It->second); + + PointerType *NewType = PointerType::get(PT->getContext(), GlobalAddressSpace); + ConvertedTypes.emplace(PT, NewType); + return NewType; +} + +ArrayType *SPIRVFixAddressSpace::convertArrayType(ArrayType *AT) { + if (!typeRequiresConversion(AT)) + return AT; + + auto It = ConvertedTypes.find(AT); + if (It != ConvertedTypes.end()) + return cast(It->second); + + Type *ElementType = convertType(AT->getElementType()); + ArrayType *NewType = ArrayType::get(ElementType, AT->getNumElements()); + ConvertedTypes.emplace(AT, NewType); + return NewType; +} + +StructType *SPIRVFixAddressSpace::convertStructType(StructType *ST) { + if (!typeRequiresConversion(ST)) + return ST; + std::vector Elements; + Elements.resize(ST->getNumElements()); + for (unsigned I = 0; I < Elements.size(); ++I) + Elements[I] = convertType(ST->getElementType(I)); + + auto It = ConvertedTypes.find(ST); + if (It != ConvertedTypes.end()) + return cast(It->second); + + if (!ST->hasName()) + return StructType::get(ST->getContext(), Elements); + + std::string OldName = ST->getName().str(); + ST->setName(OldName + ".old"); + + StructType *NewType = StructType::create(ST->getContext(), Elements, OldName); + ConvertedTypes.emplace(ST, NewType); + return NewType; +} + +VectorType *SPIRVFixAddressSpace::convertVectorType(VectorType *VT) { + if (!typeRequiresConversion(VT)) + return VT; + + auto It = ConvertedTypes.find(VT); + if (It != ConvertedTypes.end()) + return cast(It->second); + + Type *ElementType = convertType(VT->getElementType()); + VectorType *NewType = VectorType::get(ElementType, VT->getElementCount()); + ConvertedTypes.emplace(VT, NewType); + return NewType; +} + +FunctionType *SPIRVFixAddressSpace::convertFunctionType(FunctionType *FT) { + if (!typeRequiresConversion(FT)) + return FT; + + auto It = ConvertedTypes.find(FT); + if (It != ConvertedTypes.end()) + return cast(It->second); + + Type *ReturnType = FT->getReturnType(); + std::vector Params; + Params.reserve(FT->getNumParams()); + for (Type *P : FT->params()) + Params.push_back(convertType(P)); + + FunctionType *NewType = FunctionType::get(ReturnType, Params, FT->isVarArg()); + ConvertedTypes.emplace(FT, NewType); + return NewType; +} + +// Replace all references of the default address space in `T` with +// the `Private` SPIR-V address space, recreating the type is required. +// Returns the new type if recreated, `T` otherwise. +Type *SPIRVFixAddressSpace::convertType(Type *T) { + if (PointerType *PT = dyn_cast(T)) + return convertPointerType(PT); + + if (ArrayType *AT = dyn_cast(T)) + return convertArrayType(AT); + + if (VectorType *VT = dyn_cast(T)) + return convertVectorType(VT); + + if (StructType *ST = dyn_cast(T)) + return convertStructType(ST); + + if (FunctionType *FT = dyn_cast(T)) + return convertFunctionType(FT); + + if (isa(T)) + return T; + + if (T == Type::getTokenTy(T->getContext())) + return T; + + if (T == Type::getLabelTy(T->getContext())) + return T; + + // TypedPointerType: not implemented on purpose. + + // Make sure pointers & vectors are handled above. + // All other single-value types don't address space conversion. + assert(!T->isPointerTy() && !T->isVectorTy()); + if (T->isSingleValueType()) + return T; + + llvm_unreachable("Unsupported type for address space fixup."); +} + +Constant * +SPIRVFixAddressSpace::convertConstantAggregate(ConstantAggregate *CA) { + Type *NewType = convertType(CA->getType()); + std::vector Elements; + Elements.resize(CA->getNumOperands()); + for (unsigned I = 0; I < CA->getNumOperands(); ++I) + Elements[I] = convertConstant(cast(CA->getOperand(I))); + + if (isa(CA)) + return ConstantArray::get(cast(NewType), Elements); + else if (isa(CA)) + return ConstantStruct::get(cast(NewType), Elements); + return ConstantVector::get(Elements); +} + +ConstantData *SPIRVFixAddressSpace::convertConstantData(ConstantData *CD) { + if (!typeRequiresConversion(CD->getType())) + return CD; + + if (ConstantPointerNull *CPN = dyn_cast(CD)) + return ConstantPointerNull::get(convertPointerType(CPN->getType())); + report_fatal_error("Unsupported ConstantData type."); +} + +// Replace all references of the default address space in `C` with the +// SPIR-V private address space, recreating the constant, and/or modifying the +// type if required. +Constant *SPIRVFixAddressSpace::convertConstant(Constant *C) { + if (!typeRequiresConversion(C->getType())) + return C; + + if (ConstantAggregate *CA = dyn_cast(C)) + return convertConstantAggregate(CA); + if (ConstantData *CD = dyn_cast(C)) + return convertConstantData(CD); + llvm_unreachable("Unsupported constant type."); +} + +bool SPIRVFixAddressSpace::rewriteGlobalVariable(Module &M, + GlobalVariable *GV) { + if (GV->getAddressSpace() != 0) + return false; + + Type *NewType = GV->getValueType(); + if (typeRequiresConversion(GV->getValueType())) + NewType = convertType(NewType); + + std::string OldName = GV->getName().str(); + GV->setName(OldName + ".dead"); + GlobalVariable *NewGV = new GlobalVariable( + M, NewType, + /* isConstant= */ false, GV->getLinkage(), + convertConstant(GV->getInitializer()), OldName, + /* insertBefore= */ GV, GV->getThreadLocalMode(), GlobalAddressSpace); + + std::vector ToFix(GV->user_begin(), GV->user_end()); + for (auto *User : ToFix) { + if (Constant *C = dyn_cast(User)) + C->handleOperandChange(GV, NewGV); + else + User->replaceUsesOfWith(GV, NewGV); + } + M.eraseGlobalVariable(GV); + return true; +} + +bool SPIRVFixAddressSpace::replaceAlloca(Function &F) { + std::unordered_set DeadInstructions; + + for (auto &BB : F) { + for (auto &I : BB) { + AllocaInst *AI = dyn_cast(&I); + if (!AI) + continue; + + // Vulkan doesn't support VLA in local variables. (See + // VUID-StandaloneSpirv-OpTypeRuntimeArray-04680). HLSL doesn't allow VLA, + // meaning we should not encounter this for now, but it another frontend + // is used, we may hit this case. + assert(isa(AI->getArraySize())); + + Type *NewType = convertType(AI->getAllocatedType()); + GlobalVariable *NewGV = new GlobalVariable( + *F.getParent(), NewType, + /* isConstant= */ false, GlobalValue::LinkageTypes::InternalLinkage, + Constant::getNullValue(NewType), F.getName() + ".local", + /* insertBefore= */ nullptr, + GlobalValue::ThreadLocalMode::NotThreadLocal, GlobalAddressSpace); + + std::vector ToFix(AI->user_begin(), AI->user_end()); + for (auto *User : ToFix) + User->replaceUsesOfWith(AI, NewGV); + DeadInstructions.insert(AI); + } + } + + for (auto *I : DeadInstructions) + I->eraseFromParent(); + return DeadInstructions.size() != 0; +} + +bool SPIRVFixAddressSpace::blindlyMutateTypes(Function &F) { + bool Modified = false; + + for (auto &BB : F) { + for (auto &I : BB) { + for (auto &Op : I.operands()) { + if (isa(Op.get()) || isa(Op.get())) + continue; + + Type *NewType = convertType(Op->getType()); + Op->mutateType(NewType); + Modified = true; + } + + if (typeRequiresConversion(I.getType())) { + Type *NewType = convertType(I.getType()); + I.mutateType(NewType); + Modified = true; + } + } + } + + return Modified; +} + +bool SPIRVFixAddressSpace::rewriteGEP(Function &F) { + std::unordered_set DeadInstructions; + + for (auto &BB : F) { + for (auto &I : BB) { + auto *GEP = dyn_cast(&I); + if (!GEP) + continue; + + Type *SourceType = convertType(GEP->getSourceElementType()); + + IRBuilder B(GEP->getParent(), NoFolder()); + B.SetInsertPoint(GEP); + std::vector Indices(GEP->idx_begin(), GEP->idx_end()); + auto *NewInstr = + B.CreateGEP(SourceType, GEP->getPointerOperand(), Indices, + GEP->getName(), GEP->getNoWrapFlags()); + GEP->replaceAllUsesWith(NewInstr); + DeadInstructions.insert(GEP); + } + } + + for (auto *I : DeadInstructions) + I->eraseFromParent(); + return DeadInstructions.size() != 0; +} + +bool SPIRVFixAddressSpace::fixInstructions(Function &F) { + bool Modified = false; + + Modified |= replaceAlloca(F); + Modified |= blindlyMutateTypes(F); + Modified |= rewriteGEP(F); + + return Modified; +} + +bool SPIRVFixAddressSpace::rewriteFunctionParameters(Module &M, Function *F) { + if (F->isDeclaration()) + return false; + + FunctionType *NewType = convertFunctionType(F->getFunctionType()); + if (NewType == F->getFunctionType()) + return false; + + std::string OldName = F->getName().str(); + F->setName(OldName + ".dead"); + Function *NewFunction = Function::Create(NewType, F->getLinkage(), + /* AddressSpace= */ 0, OldName); + NewFunction->copyAttributesFrom(F); + NewFunction->copyMetadata(F, 0); + NewFunction->setIsNewDbgInfoFormat(F->IsNewDbgInfoFormat); + M.getFunctionList().insert(F->getIterator(), NewFunction); + + std::vector ToFix(F->user_begin(), F->user_end()); + for (auto *User : ToFix) { + User->replaceUsesOfWith(F, NewFunction); + CallBase *CB = dyn_cast(User); + if (!CB) + continue; + CB->mutateFunctionType(NewType); + } + + for (size_t I = 0; I < NewFunction->arg_size(); ++I) { + Argument *OldArgument = F->getArg(I); + Argument *NewArgument = NewFunction->getArg(I); + NewArgument->setName(OldArgument->getName()); + std::vector ToFix(OldArgument->user_begin(), + OldArgument->user_end()); + for (auto *User : ToFix) + User->replaceUsesOfWith(OldArgument, NewArgument); + } + + NewFunction->splice(NewFunction->begin(), F); + F->eraseFromParent(); + return true; +} + +bool SPIRVFixAddressSpace::fixFunctions(Module &M) { + bool Modified = false; + std::vector WorkList; + + WorkList.reserve(M.size()); + for (Function &F : M) + WorkList.push_back(&F); + + // If a function has a ptr argument/return value, we must + // rewrite its definition to use ptr addrspace(10) instead. + for (Function *F : WorkList) + Modified |= rewriteFunctionParameters(M, F); + + // Once the function declarations are fixed, we must check each instruction, + // and replace any use of `ptr` with `ptr addrspace(10)`. + for (Function &F : M) + Modified |= fixInstructions(F); + + return Modified; +} + +bool SPIRVFixAddressSpace::fixGlobals(Module &M) { + bool Modified = false; + std::vector WorkList; + + WorkList.reserve(M.global_size()); + for (GlobalVariable &GV : M.globals()) + WorkList.push_back(&GV); + + for (GlobalVariable *GV : WorkList) + Modified |= rewriteGlobalVariable(M, GV); + + return Modified; +} + +} // anonymous namespace + +bool SPIRVFixAddressSpace::runOnModule(Module &M) { + bool Modified = false; + Modified |= fixGlobals(M); + Modified |= fixFunctions(M); + return Modified; +} + +char SPIRVFixAddressSpace::ID = 0; + +INITIALIZE_PASS_BEGIN(SPIRVFixAddressSpace, "spirv-fix-address-space", + "Fixup address space", false, false) + +INITIALIZE_PASS_END(SPIRVFixAddressSpace, "spirv-fix-address-space", + "Fixup address space", false, false) + +ModulePass *llvm::createSPIRVFixAddressSpacePass() { + return new SPIRVFixAddressSpace(); +} diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp index 098c7a6fba50e..4be85903f9e47 100644 --- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp @@ -31,6 +31,7 @@ #include "llvm/Pass.h" #include "llvm/Passes/PassBuilder.h" #include "llvm/Target/TargetOptions.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/Scalar/Reg2Mem.h" #include "llvm/Transforms/Utils.h" #include @@ -177,6 +178,7 @@ TargetPassConfig *SPIRVTargetMachine::createPassConfig(PassManagerBase &PM) { void SPIRVPassConfig::addIRPasses() { TargetPassConfig::addIRPasses(); + // Structurizer is only running if targeting graphical SPIR-V. if (TM.getSubtargetImpl()->isVulkanEnv()) { // 1. Simplify loop for subsequent transformations. After this steps, loops // have the following properties: @@ -202,6 +204,12 @@ void SPIRVPassConfig::addIRPasses() { addPass(createPromoteMemoryToRegisterPass()); } + // Graphical SPIR-V has a specific set of storage classes which requires IR + // fixup. + if (TM.getSubtargetImpl()->isVulkanEnv()) { + addPass(createSPIRVFixAddressSpacePass()); + } + addPass(createSPIRVRegularizerPass()); addPass(createSPIRVPrepareFunctionsPass(TM)); addPass(createSPIRVStripConvergenceIntrinsicsPass()); diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveGetLaneIndex.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveGetLaneIndex.ll index 67e8847a2945f..0b9a91768d1e6 100644 --- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveGetLaneIndex.ll +++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveGetLaneIndex.ll @@ -14,8 +14,9 @@ ; CHECK-DAG: OpDecorate %[[#var:]] BuiltIn SubgroupLocalInvocationId ; CHECK-DAG: %[[#int:]] = OpTypeInt 32 0 ; CHECK-DAG: %[[#ptri:]] = OpTypePointer Input %[[#int]] -; CHECK-DAG: %[[#ptrf:]] = OpTypePointer Function %[[#int]] +; CHECK-DAG: %[[#ptrp:]] = OpTypePointer Private %[[#int]] ; CHECK-DAG: %[[#var]] = OpVariable %[[#ptri]] Input +; CHECK-DAG: %[[#idx:]] = OpVariable %[[#ptrp]] Private ; CHECK-NOT: OpDecorate %[[#var]] LinkageAttributes @@ -30,7 +31,6 @@ define internal spir_func void @main() #0 { entry: %0 = call token @llvm.experimental.convergence.entry() %idx = alloca i32, align 4 -; CHECK: %[[#idx:]] = OpVariable %[[#ptrf]] Function %1 = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %0) ] ; CHECK: %[[#tmp:]] = OpLoad %[[#int]] %[[#var]] diff --git a/llvm/test/CodeGen/SPIRV/logical-access-chain.ll b/llvm/test/CodeGen/SPIRV/logical-access-chain.ll index d56678ecfc2c9..eeac1ab7db29f 100644 --- a/llvm/test/CodeGen/SPIRV/logical-access-chain.ll +++ b/llvm/test/CodeGen/SPIRV/logical-access-chain.ll @@ -3,16 +3,23 @@ ; CHECK-DAG: [[uint:%[0-9]+]] = OpTypeInt 32 0 ; CHECK-DAG: [[uint2:%[0-9]+]] = OpTypeVector [[uint]] 2 ; CHECK-DAG: [[uint_1:%[0-9]+]] = OpConstant [[uint]] 1 -; CHECK-DAG: [[ptr_uint:%[0-9]+]] = OpTypePointer Function [[uint]] -; CHECK-DAG: [[ptr_uint2:%[0-9]+]] = OpTypePointer Function [[uint2]] +; CHECK-DAG: [[uint_2:%[0-9]+]] = OpConstant [[uint]] 2 +; CHECK-DAG: [[ptr_uint:%[0-9]+]] = OpTypePointer Private [[uint]] +; CHECK-DAG: [[ptr_uint2:%[0-9]+]] = OpTypePointer Private [[uint2]] + +; CHECK-DAG: [[var:%[0-9]+]] = OpVariable [[ptr_uint2]] Private define void @main() #1 { entry: %0 = alloca <2 x i32>, align 4 -; CHECK: [[var:%[0-9]+]] = OpVariable [[ptr_uint2]] Function +; CHECK: OpLabel +; CHECK-NOT: OpVariable %1 = getelementptr <2 x i32>, ptr %0, i32 0, i32 1 -; CHECK: {{%[0-9]+}} = OpAccessChain [[ptr_uint]] [[var]] [[uint_1]] +; CHECK: [[tmp:%[0-9]+]] = OpAccessChain [[ptr_uint]] [[var]] [[uint_1]] + + store i32 2, ptr %1 +; CHECK: OpStore [[tmp]] [[uint_2]] Aligned 4 ret void } diff --git a/llvm/test/CodeGen/SPIRV/logical-struct-access.ll b/llvm/test/CodeGen/SPIRV/logical-struct-access.ll index a1ff1e0752e03..7d25e1149edaa 100644 --- a/llvm/test/CodeGen/SPIRV/logical-struct-access.ll +++ b/llvm/test/CodeGen/SPIRV/logical-struct-access.ll @@ -1,3 +1,4 @@ +; RUN: llc -O0 -mtriple=spirv-unknown-vulkan1.3 %s -print-after-all -o - 2>&1 | FileCheck %s ; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s ; CHECK-DAG: [[uint:%[0-9]+]] = OpTypeInt 32 0 @@ -19,14 +20,17 @@ ; CHECK-DAG: [[uint_1:%[0-9]+]] = OpConstant [[uint]] 1 ; CHECK-DAG: [[uint_2:%[0-9]+]] = OpConstant [[uint]] 2 -; CHECK-DAG: [[ptr_uint:%[0-9]+]] = OpTypePointer Function [[uint]] -; CHECK-DAG: [[ptr_A:%[0-9]+]] = OpTypePointer Function [[A]] -; CHECK-DAG: [[ptr_B:%[0-9]+]] = OpTypePointer Function [[B]] +; CHECK-DAG: [[ptr_uint:%[0-9]+]] = OpTypePointer Private [[uint]] +; CHECK-DAG: [[ptr_A:%[0-9]+]] = OpTypePointer Private [[A]] +; CHECK-DAG: [[ptr_B:%[0-9]+]] = OpTypePointer Private [[B]] + +; CHECK-DAG: [[tmp:%[0-9]+]] = OpVariable [[ptr_B]] Private define void @main() #1 { entry: %0 = alloca %B, align 4 -; CHECK: [[tmp:%[0-9]+]] = OpVariable [[ptr_B]] Function +; CHECK: OpLabel +; CHECK-NOT: OpVariable %1 = getelementptr %B, ptr %0, i32 0, i32 0 ; CHECK: {{%[0-9]+}} = OpAccessChain [[ptr_A]] [[tmp]] [[uint_0]] diff --git a/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-call.ll b/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-call.ll new file mode 100644 index 0000000000000..9ad8ac76e8bed --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-call.ll @@ -0,0 +1,31 @@ +; RUN: llc -O0 -mtriple=spirv-unknown-vulkan1.3 %s -print-after-all -o - 2>&1 | FileCheck %s + +; CHECK: *** IR Dump After Fixup address space (spirv-fix-address-space) *** + +@input = internal global i32 0 +; CHECK: @input = internal addrspace(10) global i32 0 + +define spir_func i32 @foo(ptr noundef nonnull align 4 %param) #0 { +; CHECK: define spir_func i32 @foo(ptr addrspace(10) noundef nonnull align 4 %param) #0 { + + %1 = load i32, ptr %param, align 4 +; CHECK: %1 = load i32, ptr addrspace(10) %param, align 4 + ret i32 %1 +} + + +define internal spir_func i32 @main() #1 { +; CHECK: define internal spir_func i32 @main() #1 { +entry: +; CHECK: entry: + %0 = call token @llvm.experimental.convergence.entry() + + %1 = call i32 @foo(ptr @input) +; CHECK: %[[#call:]] = call i32 @foo(ptr addrspace(10) @input) + + ret i32 %1 +; CHECK: ret i32 %[[#call]] +} + +attributes #0 = { alwaysinline } +attributes #1 = { convergent } diff --git a/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-initialization.ll b/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-initialization.ll new file mode 100644 index 0000000000000..c94ff67e743f1 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-initialization.ll @@ -0,0 +1,25 @@ +; RUN: llc -O0 -mtriple=spirv-unknown-vulkan1.3 %s -print-after-all -o - 2>&1 | FileCheck %s + +; CHECK: *** IR Dump After Fixup address space (spirv-fix-address-space) *** + +@scalar = internal global i32 10 +; CHECK: @scalar = internal addrspace(10) global i32 10 + +@pointer = internal global ptr null +; CHECK: @pointer = internal addrspace(10) global ptr addrspace(10) null + +@array = internal global [2 x i32] [i32 1, i32 2] +; CHECK: @array = internal addrspace(10) global [2 x i32] [i32 1, i32 2] + +@aggregate = internal global { i32, i32, ptr } { i32 3, i32 4, ptr null } +; CHECK: @aggregate = internal addrspace(10) global { i32, i32, ptr addrspace(10) } { i32 3, i32 4, ptr addrspace(10) null } + + + +define internal spir_func i32 @main() #0 { +entry: + ret i32 0 +} + +attributes #0 = { convergent } + diff --git a/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-object.ll b/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-object.ll new file mode 100644 index 0000000000000..cd51638aca9a3 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-object.ll @@ -0,0 +1,60 @@ +; RUN: llc -O0 -mtriple=spirv-unknown-vulkan1.3 %s -print-after-all -o - 2>&1 | FileCheck %s + +; CHECK: *** IR Dump After Fixup address space (spirv-fix-address-space) *** + +%struct.S = type { i32 } + +@object = internal global %struct.S zeroinitializer, align 4 +; CHECK: @object = internal addrspace(10) global %struct.S zeroinitializer +@input = internal global i32 0, align 4 +; CHECK: @input = internal addrspace(10) global i32 0 +@output = internal global i32 0, align 4 +; CHECK: @output = internal addrspace(10) global i32 0 + +define linkonce_odr spir_func void @S_set(ptr noundef nonnull align 4 dereferenceable(4) %this, i32 noundef %param) #0 align 2 { +; CHECK: define linkonce_odr spir_func void @S_set(ptr addrspace(10) noundef nonnull align 4 dereferenceable(4) %this, i32 noundef %param) #0 align 2 { +entry: + %0 = call token @llvm.experimental.convergence.entry() + %value = getelementptr inbounds nuw %struct.S, ptr %this, i32 0, i32 0 +; CHECK: %value1 = getelementptr inbounds nuw %struct.S, ptr addrspace(10) %this, i32 0, i32 0 + store i32 %param, ptr %value, align 4 +; CHECK: store i32 %param, ptr addrspace(10) %value1, align 4 + ret void +} + +define linkonce_odr spir_func noundef i32 @S_get(ptr noundef nonnull align 4 dereferenceable(4) %this) #0 align 2 { +; CHECK: define linkonce_odr spir_func noundef i32 @S_get(ptr addrspace(10) noundef nonnull align 4 dereferenceable(4) %this) #0 align 2 { +entry: + %0 = call token @llvm.experimental.convergence.entry() + %value = getelementptr inbounds nuw %struct.S, ptr %this, i32 0, i32 0 +; CHECK: %value1 = getelementptr inbounds nuw %struct.S, ptr addrspace(10) %this, i32 0, i32 0 + %1 = load i32, ptr %value, align 4 +; CHECK: %1 = load i32, ptr addrspace(10) %value1, align 4 + ret i32 %1 +} + +define void @main() #1 { +; CHECK: define void @main() #1 { +entry: +; CHECK: entry: + %0 = call token @llvm.experimental.convergence.entry() + + %1 = load i32, ptr @input, align 4 +; %1 = load i32, ptr addrspace(10) @input, align 4 + call spir_func void @S_set(ptr noundef nonnull align 4 dereferenceable(4) @object, i32 noundef %1) #0 [ "convergencectrl"(token %0) ] +; call spir_func void @S_set(ptr addrspace(10) noundef nonnull align 4 dereferenceable(4) @object, i32 noundef %1) #0 [ "convergencectrl"(token %0) ] + + %call1 = call spir_func noundef i32 @S_get(ptr noundef nonnull align 4 dereferenceable(4) @object) #0 [ "convergencectrl"(token %0) ] +; %call1 = call spir_func noundef i32 @S_get(ptr addrspace(10) noundef nonnull align 4 dereferenceable(4) @object) #0 [ "convergencectrl"(token %0) ] + + store i32 %call1, ptr @output, align 4 +; store i32 %call1, ptr addrspace(10) @output, align 4 + ret void +; ret void +} + +declare token @llvm.experimental.convergence.entry() #2 + +attributes #0 = { convergent alwaysinline } +attributes #1 = { convergent noinline norecurse } +attributes #2 = { convergent nocallback nofree nosync nounwind willreturn memory(none) } diff --git a/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-ptr-ptr.ll b/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-ptr-ptr.ll new file mode 100644 index 0000000000000..676916e542f83 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-ptr-ptr.ll @@ -0,0 +1,31 @@ +; RUN: llc -O0 -mtriple=spirv-unknown-vulkan1.3 %s -print-after-all -o - 2>&1 | FileCheck %s + +; CHECK: *** IR Dump After Fixup address space (spirv-fix-address-space) *** + +%S = type { ptr } + +@input = internal global i32 0 +@struct = internal global %S { ptr @input } +; CHECK: @input = internal addrspace(10) global i32 0 +; CHECK: @struct = internal addrspace(10) global %S { ptr addrspace(10) @input } + + +define internal spir_func i32 @main(i32 %index) #0 { +entry: +; CHECK: define internal spir_func i32 @main(i32 %index) #0 { +; CHECK: entry: + + %0 = getelementptr inbounds %S, ptr @struct, i32 0 +; CHECK: %[[#pptr:]] = getelementptr inbounds %S, ptr addrspace(10) @struct, i32 0 + + %1 = getelementptr ptr, ptr %0, i32 %index +; CHECK: %[[#ptr:]] = getelementptr ptr addrspace(10), ptr addrspace(10) %[[#pptr]], i32 %index + + %2 = load i32, ptr %1 +; CHECK: %[[#load:]] = load i32, ptr addrspace(10) %[[#ptr]] + + ret i32 %2 +; CHECK: ret i32 %[[#load]] +} + +attributes #0 = { convergent } diff --git a/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-simple-local.ll b/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-simple-local.ll new file mode 100644 index 0000000000000..1a8dd3e57b1af --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-simple-local.ll @@ -0,0 +1,42 @@ +; RUN: llc -O0 -mtriple=spirv-unknown-vulkan1.3 %s -print-after-all -o - 2>&1 | FileCheck %s + +; CHECK: *** IR Dump After Fixup address space (spirv-fix-address-space) *** + +@input = internal global i32 0 +; CHECK: @input = internal addrspace(10) global i32 0 +; CHECK: @main.local = internal addrspace(10) global i32 0 + +define spir_func i32 @foo(ptr noundef nonnull align 4 %param) { +; CHECK: define spir_func i32 @foo(ptr addrspace(10) noundef nonnull align 4 %param) { + %1 = load i32, ptr %param, align 4 +; CHECK: %1 = load i32, ptr addrspace(10) %param, align 4 + ret i32 %1 +; CHECK: ret i32 %1 +} + + +define internal spir_func void @main() #0 { +; CHECK: define internal spir_func void @main() #0 { +entry: +; CHECK: entry: + %0 = call token @llvm.experimental.convergence.entry() + + %1 = alloca i32 +; CHECK-NOT: %1 = alloca + + %2 = load i32, ptr @input +; CHECK: %1 = load i32, ptr addrspace(10) @input + store i32 %2, ptr %1 +; CHECK: store i32 %1, ptr addrspace(10) @main.local + + %3 = call i32 @foo(ptr %1) +; CHECK: %2 = call i32 @foo(ptr addrspace(10) @main.local) + %4 = load i32, ptr @input +; CHECK: %3 = load i32, ptr addrspace(10) @input + + ret void +; CHECK: ret void +} + +attributes #0 = { convergent } + diff --git a/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-simple.ll b/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-simple.ll new file mode 100644 index 0000000000000..43d93efae9882 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/passes/SPIRVFixAddressSpace-simple.ll @@ -0,0 +1,30 @@ +; RUN: llc -O0 -mtriple=spirv-unknown-vulkan1.3 %s -print-after-all -o - 2>&1 | FileCheck %s + +; CHECK: *** IR Dump After Fixup address space (spirv-fix-address-space) *** + +@input = internal global i32 0 +; CHECK: @input = internal addrspace(10) global i32 0 + +define spir_func void @foo(ptr noundef nonnull align 4 %param) { +; CHECK: define spir_func void @foo(ptr addrspace(10) noundef nonnull align 4 %param) { + + %1 = load ptr, ptr %param, align 4 +; CHECK: %1 = load ptr addrspace(10), ptr addrspace(10) %param, align 4 + + ret void +} + + +define internal spir_func void @main() #0 { +; CHECK: define internal spir_func void @main() #0 { +entry: +; CHECK: entry: + %0 = call token @llvm.experimental.convergence.entry() + %v = load i32, ptr @input +; CHECK: %v = load i32, ptr addrspace(10) @input + + ret void +; CHECK: ret void +} + +attributes #0 = { convergent } diff --git a/llvm/test/CodeGen/SPIRV/structurizer/basic-phi.ll b/llvm/test/CodeGen/SPIRV/structurizer/basic-phi.ll index 8ad1758a14c84..63488fbb60470 100644 --- a/llvm/test/CodeGen/SPIRV/structurizer/basic-phi.ll +++ b/llvm/test/CodeGen/SPIRV/structurizer/basic-phi.ll @@ -9,9 +9,9 @@ define spir_func noundef i32 @_Z7processv() #0 { ; CHECK-DAG: %[[#int_0:]] = OpConstant %[[#]] 0 ; CHECK-DAG: %[[#int_1:]] = OpConstant %[[#]] 1 +; CHECK-DAG: %[[#var:]] = OpVariable %[[#]] Private ; CHECK: %[[#entry:]] = OpLabel -; CHECK: %[[#var:]] = OpVariable %[[#]] Function ; CHECK: OpSelectionMerge %[[#merge:]] None ; CHECK: OpBranchConditional %[[#]] %[[#left:]] %[[#right:]] entry: diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll index e4cb1ddc594b0..3307ce1d3d017 100644 --- a/llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll +++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.cond-op.ll @@ -9,17 +9,26 @@ target triple = "spirv-unknown-vulkan1.3-compute" ; CHECK-DAG: OpName %[[#fn1:]] "_Z3fn1v" ; CHECK-DAG: OpName %[[#fn2:]] "_Z3fn2v" -; CHECK-DAG: OpName %[[#r2m_a:]] ".reg2mem3" -; CHECK-DAG: OpName %[[#r2m_b:]] ".reg2mem1" -; CHECK-DAG: OpName %[[#r2m_c:]] ".reg2mem" +; CHECK-DAG: OpName %[[#local_0:]] "_Z7processv.local" +; CHECK-DAG: OpName %[[#local_2:]] "_Z7processv.local.2" +; CHECK-DAG: OpName %[[#local_3:]] "_Z7processv.local.3" +; CHECK-DAG: OpName %[[#local_4:]] "_Z7processv.local.4" -; CHECK-DAG: %[[#int_ty:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#int_ty:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#bool_ty:]] = OpTypeBool +; CHECK-DAG: %[[#ptr_bool_ty:]] = OpTypePointer Private %[[#bool_ty]] +; CHECK-DAG: %[[#ptr_int_ty:]] = OpTypePointer Private %[[#int_ty]] ; CHECK-DAG: %[[#int_0:]] = OpConstant %[[#]] 0 ; CHECK-DAG: %[[#int_1:]] = OpConstant %[[#]] 1 -; CHECK-DAG: %[[#true:]] = OpConstantTrue +; CHECK-DAG: %[[#true:]] = OpConstantTrue ; CHECK-DAG: %[[#false:]] = OpConstantFalse +; CHECK-DAG: %[[#local_0]] = OpVariable %[[#ptr_int_ty]] Private +; CHECK-DAG: %[[#local_2]] = OpVariable %[[#ptr_int_ty]] Private +; CHECK-DAG: %[[#local_3]] = OpVariable %[[#ptr_int_ty]] Private +; CHECK-DAG: %[[#local_4]] = OpVariable %[[#ptr_bool_ty]] Private + declare token @llvm.experimental.convergence.entry() #1 ; CHECK: %[[#fn]] = OpFunction %[[#int_ty]] @@ -47,7 +56,6 @@ entry: define spir_func noundef i32 @_Z7processv() #0 { ; CHECK: %[[#entry:]] = OpLabel -; CHECK-DAG: %[[#r2m_a]] = OpVariable %[[#]] Function ; CHECK: OpSelectionMerge %[[#a_merge:]] ; CHECK: OpBranchConditional %[[#]] %[[#a_true:]] %[[#a_false:]] entry: @@ -56,19 +64,19 @@ entry: br i1 true, label %a_true, label %a_false ; CHECK: %[[#a_false]] = OpLabel -; CHECK: OpStore %[[#r2m_a]] %[[#false]] +; CHECK: OpStore %[[#local_4]] %[[#false]] ; CHECK: OpBranch %[[#a_merge]] a_false: br label %a_merge ; CHECK: %[[#a_true]] = OpLabel -; CHECK: OpStore %[[#r2m_a]] %[[#true]] +; CHECK: OpStore %[[#local_4]] %[[#true]] ; CHECK: OpBranch %[[#a_merge]] a_true: br label %a_merge ; CHECK: %[[#a_merge]] = OpLabel -; CHECK: %[[#tmp:]] = OpLoad %[[#]] %[[#r2m_a]] +; CHECK: %[[#tmp:]] = OpLoad %[[#]] %[[#local_4]] ; CHECK: OpSelectionMerge %[[#b_merge:]] ; CHECK: OpBranchConditional %[[#]] %[[#b_true:]] %[[#b_merge]] a_merge: @@ -90,8 +98,8 @@ b_merge: br i1 true, label %c_true, label %c_false ; CHECK: %[[#c_false]] = OpLabel -; CHECK: %[[#]] = OpFunctionCall -; CHECK: OpStore %[[#r2m_b]] %[[#]] +; CHECK: %[[#tmp:]] = OpFunctionCall +; CHECK: OpStore %[[#local_2]] %[[#tmp]] ; CHECK: OpBranch %[[#c_merge]] c_false: %f3 = call spir_func noundef i32 @_Z3fn2v() #4 [ "convergencectrl"(token %0) ] @@ -99,7 +107,7 @@ c_false: ; CHECK: %[[#c_true]] = OpLabel ; CHECK: %[[#]] = OpFunctionCall -; CHECK: OpStore %[[#r2m_b]] %[[#]] +; CHECK: OpStore %[[#local_3]] %[[#]] ; CHECK: OpBranch %[[#c_merge]] c_true: %f2 = call spir_func noundef i32 @_Z3fn1v() #4 [ "convergencectrl"(token %0) ] @@ -107,8 +115,8 @@ c_true: ; CHECK: %[[#c_merge]] = OpLabel -; CHECK: %[[#tmp:]] = OpLoad %[[#]] %[[#r2m_b]] -; CHECK: OpStore %[[#r2m_c]] %[[#tmp:]] +; CHECK: %[[#tmp:]] = OpLoad %[[#]] %[[#local_3]] +; CHECK: OpStore %[[#local_0]] %[[#tmp:]] ; CHECK: OpSelectionMerge %[[#d_merge:]] ; CHECK: OpBranchConditional %[[#]] %[[#d_true:]] %[[#d_merge]] c_merge: @@ -122,7 +130,7 @@ d_true: br label %d_merge ; CHECK: %[[#d_merge]] = OpLabel -; CHECK: %[[#tmp:]] = OpLoad %[[#]] %[[#r2m_c]] +; CHECK: %[[#tmp:]] = OpLoad %[[#]] %[[#local_0]] ; CHECK: OpReturnValue %[[#tmp]] d_merge: ret i32 %5 diff --git a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.plain.ll b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.plain.ll index b7167e01e9224..856f5dad5dacf 100644 --- a/llvm/test/CodeGen/SPIRV/structurizer/cf.for.plain.ll +++ b/llvm/test/CodeGen/SPIRV/structurizer/cf.for.plain.ll @@ -5,12 +5,14 @@ target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256: target triple = "spirv-unknown-vulkan1.3-compute" ; CHECK-DAG: OpName %[[#process:]] "_Z7processv" -; CHECK-DAG: OpName %[[#val:]] "val" -; CHECK-DAG: OpName %[[#i:]] "i" +; CHECK-DAG: OpName %[[#val:]] "_Z7processv.local" +; CHECK-DAG: OpName %[[#i:]] "_Z7processv.local.1" -; CHECK-DAG: %[[#int_ty:]] = OpTypeInt 32 0 -; CHECK-DAG: %[[#int_pfty:]] = OpTypePointer Function %[[#int_ty]] -; CHECK-DAG: %[[#false:]] = OpConstantFalse +; CHECK-DAG: %[[#int_ty:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#int_ppty:]] = OpTypePointer Private %[[#int_ty]] +; CHECK-DAG: %[[#false:]] = OpConstantFalse +; CHECK-DAG: %[[#val]] = OpVariable %[[#int_ppty]] Private +; CHECK-DAG: %[[#i]] = OpVariable %[[#int_ppty]] Private ; CHECK: %[[#process]] = OpFunction %[[#]] DontInline %[[#]] @@ -18,8 +20,6 @@ define spir_func noundef i32 @_Z7processv() #0 { entry: %0 = call token @llvm.experimental.convergence.entry() - ; CHECK-DAG: %[[#val]] = OpVariable %[[#int_pfty]] Function - ; CHECK-DAG: %[[#i]] = OpVariable %[[#int_pfty]] Function %val = alloca i32, align 4 %i = alloca i32, align 4 store i32 0, ptr %val, align 4 diff --git a/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll b/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll index 3b3841142263a..6f3aa0e5a95f6 100644 --- a/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll +++ b/llvm/test/CodeGen/SPIRV/structurizer/condition-linear.ll @@ -26,8 +26,8 @@ entry: } -; CHECK-DAG: OpName %[[#reg_0:]] "cond.reg2mem" -; CHECK-DAG: OpName %[[#reg_1:]] "cond9.reg2mem" +; CHECK-DAG: OpName %[[#reg_0:]] "main.local.5" +; CHECK-DAG: OpName %[[#reg_1:]] "main.local.4" define internal spir_func void @main() #0 { ; CHECK: OpSelectionMerge %[[#cond1_merge:]] None diff --git a/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll b/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll index 732d8161766f2..e2c5de5c438a9 100644 --- a/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll +++ b/llvm/test/CodeGen/SPIRV/structurizer/logical-or.ll @@ -5,15 +5,16 @@ target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256: target triple = "spirv-unknown-vulkan1.3-compute" define internal spir_func void @main() #3 { -; CHECK-DAG: OpName %[[#switch_0:]] "reg1" -; CHECK-DAG: OpName %[[#switch_1:]] "reg" +; CHECK-DAG: OpName %[[#switch_0:]] "main.local" +; CHECK-DAG: OpName %[[#switch_1:]] "main.local.1" ; CHECK-DAG: %[[#int_0:]] = OpConstant %[[#]] 0 ; CHECK-DAG: %[[#int_1:]] = OpConstant %[[#]] 1 +; CHECK-DAG: %[[#switch_0]] = OpVariable %[[#]] Private +; CHECK-DAG: %[[#switch_1]] = OpVariable %[[#]] Private + ; CHECK: %[[#entry:]] = OpLabel -; CHECK-DAG: %[[#switch_0]] = OpVariable %[[#]] Function -; CHECK-DAG: %[[#switch_1]] = OpVariable %[[#]] Function ; CHECK: OpSelectionMerge %[[#merge:]] None ; CHECK: OpBranchConditional %[[#]] %[[#new_header:]] %[[#unreachable:]] diff --git a/llvm/test/CodeGen/SPIRV/structurizer/loop-continue-split.ll b/llvm/test/CodeGen/SPIRV/structurizer/loop-continue-split.ll index 4f8babb7f436f..8dbe186a8d182 100644 --- a/llvm/test/CodeGen/SPIRV/structurizer/loop-continue-split.ll +++ b/llvm/test/CodeGen/SPIRV/structurizer/loop-continue-split.ll @@ -13,8 +13,8 @@ target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1" target triple = "spirv-unknown-vulkan1.3-compute" -; CHECK-DAG: OpName %[[#switch_0:]] "reg1" -; CHECK-DAG: OpName %[[#variable:]] "var" +; CHECK-DAG: OpName %[[#switch_0:]] "main.local" +; CHECK-DAG: OpName %[[#variable:]] "main.local.2" ; CHECK-DAG: %[[#int_0:]] = OpConstant %[[#]] 0 ; CHECK-DAG: %[[#int_1:]] = OpConstant %[[#]] 1 @@ -22,10 +22,11 @@ target triple = "spirv-unknown-vulkan1.3-compute" ; CHECK-DAG: %[[#int_3:]] = OpConstant %[[#]] 3 ; CHECK-DAG: %[[#int_4:]] = OpConstant %[[#]] 4 +; CHECK-DAG: %[[#switch_0]] = OpVariable %[[#]] Private +; CHECK-DAG: %[[#variable]] = OpVariable %[[#]] Private + define internal spir_func void @main() #1 { ; CHECK: %[[#entry:]] = OpLabel -; CHECK: %[[#switch_0]] = OpVariable %[[#]] Function -; CHECK: %[[#variable]] = OpVariable %[[#]] Function ; CHECK: OpBranch %[[#header:]] entry: %0 = call token @llvm.experimental.convergence.entry() diff --git a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll index b421ae7990c67..d547965d7fdbe 100644 --- a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll +++ b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-break.ll @@ -6,17 +6,18 @@ target triple = "spirv-unknown-vulkan-compute" define internal spir_func void @main() #0 { -; CHECK-DAG: OpName %[[#idx:]] "idx" +; CHECK-DAG: OpName %[[#idx:]] "main.local.1" ; CHECK-DAG: OpDecorate %[[#builtin:]] BuiltIn SubgroupLocalInvocationId ; CHECK-DAG: %[[#int_ty:]] = OpTypeInt 32 0 ; CHECK-DAG: %[[#int_ipty:]] = OpTypePointer Input %[[#int_ty]] +; CHECK-DAG: %[[#int_ppty:]] = OpTypePointer Private %[[#int_ty]] ; CHECK-DAG: %[[#bool_ty:]] = OpTypeBool ; CHECK-DAG: %[[#int_0:]] = OpConstant %[[#int_ty]] 0 ; CHECK-DAG: %[[#int_10:]] = OpConstant %[[#int_ty]] 10 ; CHECK-DAG: %[[#builtin]] = OpVariable %[[#int_ipty]] Input +; CHECK-DAG: %[[#idx]] = OpVariable %[[#int_ppty]] Private %[[#int_0]] ; CHECK: %[[#entry:]] = OpLabel -; CHECK: %[[#idx]] = OpVariable %[[#]] Function ; ACHECK: OpStore %[[#idx]] %[[#int_0]] Aligned 4 ; CHECK: OpBranch %[[#while_cond:]] entry: diff --git a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll index ac330a96444b8..cd6936d483511 100644 --- a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll +++ b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-convergence-in-break.ll @@ -6,15 +6,15 @@ target triple = "spirv-unknown-vulkan-compute" define internal spir_func void @main() #0 { -; CHECK-DAG: OpName %[[#idx:]] "idx" +; CHECK-DAG: OpName %[[#idx:]] "main.local.1" ; CHECK-DAG: OpDecorate %[[#builtin:]] BuiltIn SubgroupLocalInvocationId ; CHECK-DAG: %[[#int_ty:]] = OpTypeInt 32 0 ; CHECK-DAG: %[[#bool_ty:]] = OpTypeBool ; CHECK-DAG: %[[#int_0:]] = OpConstant %[[#int_ty]] 0 ; CHECK-DAG: %[[#int_10:]] = OpConstant %[[#int_ty]] 10 +; CHECK-DAG: %[[#idx]] = OpVariable %[[#]] Private %[[#int_0]] ; CHECK: %[[#entry:]] = OpLabel -; CHECK: %[[#idx]] = OpVariable %[[#]] Function ; CHECK: OpStore %[[#idx]] %[[#int_0]] Aligned 4 ; CHECK: OpBranch %[[#while_cond:]] entry: diff --git a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll index 784bd38a6fbae..20f8a8a7a7107 100644 --- a/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll +++ b/llvm/test/CodeGen/SPIRV/structurizer/merge-exit-multiple-break.ll @@ -6,8 +6,8 @@ target triple = "spirv-unknown-vulkan-compute" define internal spir_func void @main() #0 { -; CHECK-DAG: OpName %[[#idx:]] "idx" -; CHECK-DAG: OpName %[[#reg_0:]] "reg" +; CHECK-DAG: OpName %[[#idx:]] "main.local.1" +; CHECK-DAG: OpName %[[#reg_0:]] "main.local" ; CHECK-DAG: OpDecorate %[[#builtin:]] BuiltIn SubgroupLocalInvocationId ; CHECK-DAG: %[[#int_ty:]] = OpTypeInt 32 0 ; CHECK-DAG: %[[#bool_ty:]] = OpTypeBool @@ -15,9 +15,9 @@ define internal spir_func void @main() #0 { ; CHECK-DAG: %[[#int_1:]] = OpConstant %[[#int_ty]] 1 ; CHECK-DAG: %[[#int_2:]] = OpConstant %[[#int_ty]] 2 ; CHECK-DAG: %[[#int_10:]] = OpConstant %[[#int_ty]] 10 +; CHECK-DAG: %[[#idx]] = OpVariable %[[#]] Private %[[#int_0]] ; CHECK: %[[#entry:]] = OpLabel -; CHECK: %[[#idx]] = OpVariable %[[#]] Function ; CHECK: OpStore %[[#idx]] %[[#int_0]] Aligned 4 ; CHECK: OpBranch %[[#while_cond:]] entry: diff --git a/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll b/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll index 5e64c24b78784..e365907dbe0a5 100644 --- a/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll +++ b/llvm/test/CodeGen/SPIRV/structurizer/return-early.ll @@ -1,8 +1,8 @@ ; RUN: llc -mtriple=spirv-unknown-vulkan-compute -O0 %s -o - | FileCheck %s ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %} -; CHECK-DAG: OpName %[[#reg_0:]] "reg2" -; CHECK-DAG: OpName %[[#reg_1:]] "reg1" +; CHECK-DAG: OpName %[[#reg_0:]] "main.local" +; CHECK-DAG: OpName %[[#reg_1:]] "main.local.1" target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1" target triple = "spirv-unknown-vulkan1.3-compute" diff --git a/llvm/test/CodeGen/SPIRV/transcoding/GlobalFunAnnotate.ll b/llvm/test/CodeGen/SPIRV/transcoding/GlobalFunAnnotate.ll index d17e228c2ef88..43d3a1d438ee3 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/GlobalFunAnnotate.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/GlobalFunAnnotate.ll @@ -5,7 +5,13 @@ @.str = private unnamed_addr constant [23 x i8] c"annotation_on_function\00", section "llvm.metadata" @.str.1 = private unnamed_addr constant [6 x i8] c"an.cl\00", section "llvm.metadata" -@llvm.global.annotations = appending global [1 x { i8*, i8*, i8*, i32, i8* }] [{ i8*, i8*, i8*, i32, i8* } { i8* bitcast (void ()* @foo to i8*), i8* getelementptr inbounds ([23 x i8], [23 x i8]* @.str, i32 0, i32 0), i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str.1, i32 0, i32 0), i32 2, i8* null }], section "llvm.metadata" +@llvm.global.annotations = appending global [1 x { i8*, i8*, i8*, i32, i8* }] + [ { i8*, i8*, i8*, i32, i8* } + { i8* bitcast (void ()* @foo to i8*), + i8* getelementptr inbounds ([23 x i8], [23 x i8]* @.str, i32 0, i32 0), + i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str.1, i32 0, i32 0), + i32 2, i8* null } + ], section "llvm.metadata" define dso_local spir_func void @foo() { entry: