diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp index ee887447972bf..39e8009547a13 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp @@ -14,6 +14,8 @@ #include "GCNSubtarget.h" #include "Utils/AMDGPUBaseInfo.h" #include "llvm/Analysis/CycleAnalysis.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Analysis/UniformityAnalysis.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/IntrinsicsAMDGPU.h" #include "llvm/IR/IntrinsicsR600.h" @@ -1299,6 +1301,111 @@ struct AAAMDGPUNoAGPR const char AAAMDGPUNoAGPR::ID = 0; +struct AAAMDGPUUniform : public StateWrapper { + using Base = StateWrapper; + AAAMDGPUUniform(const IRPosition &IRP, Attributor &A) : Base(IRP) {} + + /// Create an abstract attribute view for the position \p IRP. + static AAAMDGPUUniform &createForPosition(const IRPosition &IRP, + Attributor &A); + + /// See AbstractAttribute::getName() + const std::string getName() const override { return "AAAMDGPUUniform"; } + + const std::string getAsStr(Attributor *A) const override { + return getAssumed() ? "inreg" : "non-inreg"; + } + + void trackStatistics() const override {} + + /// See AbstractAttribute::getIdAddr() + const char *getIdAddr() const override { return &ID; } + + /// This function should return true if the type of the \p AA is + /// AAAMDGPUUniform + static bool classof(const AbstractAttribute *AA) { + return (AA->getIdAddr() == &ID); + } + + /// Unique ID (due to the unique address) + static const char ID; +}; + +const char AAAMDGPUUniform::ID = 0; + +struct AAAMDGPUUniformArgument : public AAAMDGPUUniform { + AAAMDGPUUniformArgument(const IRPosition &IRP, Attributor &A) + : AAAMDGPUUniform(IRP, A) {} + + void initialize(Attributor &A) override { + Argument *Arg = getAssociatedArgument(); + if (Arg->hasAttribute(Attribute::InReg) || + AMDGPU::isEntryFunctionCC(Arg->getParent()->getCallingConv())) + indicateOptimisticFixpoint(); + } + + ChangeStatus updateImpl(Attributor &A) override { + unsigned ArgNo = getAssociatedArgument()->getArgNo(); + + auto isUniform = [&](AbstractCallSite ACS) -> bool { + CallBase *CB = ACS.getInstruction(); + Value *V = CB->getArgOperandUse(ArgNo); + if (isa(V)) + return true; + Function *F = nullptr; + if (auto *Arg = dyn_cast(V)) { + auto *AA = + A.getOrCreateAAFor(IRPosition::argument(*Arg)); + if (AA) + return AA->isValidState(); + F = Arg->getParent(); + } else if (auto *I = dyn_cast(V)) { + F = I->getFunction(); + } + + if (F) { + auto *UA = + A.getInfoCache() + .getAnalysisResultForFunction(*F); + return UA && UA->isUniform(V); + } + + // What else can it be here? + return false; + }; + + bool UsedAssumedInformation = true; + if (!A.checkForAllCallSites(isUniform, *this, /*RequireAllCallSites=*/true, + UsedAssumedInformation)) + return indicatePessimisticFixpoint(); + + if (!UsedAssumedInformation) + return indicateOptimisticFixpoint(); + + return ChangeStatus::UNCHANGED; + } + + ChangeStatus manifest(Attributor &A) override { + Argument *Arg = getAssociatedArgument(); + if (AMDGPU::isEntryFunctionCC(Arg->getParent()->getCallingConv())) + return ChangeStatus::UNCHANGED; + return A.manifestAttrs( + getIRPosition(), {Attribute::get(Arg->getContext(), Attribute::InReg)}); + } +}; + +AAAMDGPUUniform &AAAMDGPUUniform::createForPosition(const IRPosition &IRP, + Attributor &A) { + switch (IRP.getPositionKind()) { + case IRPosition::IRP_ARGUMENT: + return *new (A.Allocator) AAAMDGPUUniformArgument(IRP, A); + // TODO: Since inreg is also allowed for return value, maybe we need to add + // AAAMDGPUUniformCallSiteReturned? + default: + llvm_unreachable("not a valid position for AAAMDGPUUniform"); + } +} + /// Performs the final check and updates the 'amdgpu-waves-per-eu' attribute /// based on the finalized 'amdgpu-flat-work-group-size' attribute. /// Both attributes start with narrow ranges that expand during iteration. @@ -1385,7 +1492,7 @@ static bool runImpl(Module &M, AnalysisGetter &AG, TargetMachine &TM, &AAAMDMaxNumWorkgroups::ID, &AAAMDWavesPerEU::ID, &AAAMDGPUNoAGPR::ID, &AACallEdges::ID, &AAPointerInfo::ID, &AAPotentialConstantValues::ID, &AAUnderlyingObjects::ID, &AAAddressSpace::ID, &AAIndirectCallInfo::ID, - &AAInstanceInfo::ID}); + &AAInstanceInfo::ID, &AAAMDGPUUniform::ID}); AttributorConfig AC(CGUpdater); AC.IsClosedWorldModule = Options.IsClosedWorld; @@ -1438,6 +1545,11 @@ static bool runImpl(Module &M, AnalysisGetter &AG, TargetMachine &TM, IRPosition::value(*CmpX->getPointerOperand())); } } + + if (!AMDGPU::isEntryFunctionCC(F->getCallingConv())) { + for (auto &Arg : F->args()) + A.getOrCreateAAFor(IRPosition::argument(Arg)); + } } bool Changed = A.run() == ChangeStatus::CHANGED; @@ -1470,6 +1582,7 @@ class AMDGPUAttributorLegacy : public ModulePass { void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired(); + AU.addRequired(); } StringRef getPassName() const override { return "AMDGPU Attributor"; } diff --git a/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll b/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll index d1a6414fe49ae..f77639f21342e 100644 --- a/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll +++ b/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll @@ -90,7 +90,7 @@ define void @call_volatile_load_store_as_4(ptr addrspace(4) %p1, ptr addrspace(4 define internal void @can_infer_cmpxchg(ptr %word) { ; CHECK-LABEL: define internal void @can_infer_cmpxchg( -; CHECK-SAME: ptr [[WORD:%.*]]) #[[ATTR0]] { +; CHECK-SAME: ptr inreg [[WORD:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1) ; CHECK-NEXT: [[CMPXCHG_0:%.*]] = cmpxchg ptr addrspace(1) [[TMP1]], i32 0, i32 4 monotonic monotonic, align 4 ; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1) @@ -144,7 +144,7 @@ define internal void @can_not_infer_cmpxchg(ptr %word) { define internal void @can_infer_atomicrmw(ptr %word) { ; CHECK-LABEL: define internal void @can_infer_atomicrmw( -; CHECK-SAME: ptr [[WORD:%.*]]) #[[ATTR0]] { +; CHECK-SAME: ptr inreg [[WORD:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1) ; CHECK-NEXT: [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr addrspace(1) [[TMP1]], i32 12 monotonic, align 4 ; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1) diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-attributor-accesslist-offsetbins-out-of-sync.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-attributor-accesslist-offsetbins-out-of-sync.ll index d58a62408427d..4f46e08921a49 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgpu-attributor-accesslist-offsetbins-out-of-sync.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-attributor-accesslist-offsetbins-out-of-sync.ll @@ -8,7 +8,7 @@ define internal fastcc void @foo(ptr %kg) { ; CHECK-LABEL: define internal fastcc void @foo( -; CHECK-SAME: ptr [[KG:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-SAME: ptr inreg [[KG:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: [[CLOSURE_I25_I:%.*]] = getelementptr i8, ptr [[KG]], i64 336 ; CHECK-NEXT: [[NUM_CLOSURE_I26_I:%.*]] = getelementptr i8, ptr [[KG]], i64 276 diff --git a/llvm/test/CodeGen/AMDGPU/inreg-inference.ll b/llvm/test/CodeGen/AMDGPU/inreg-inference.ll new file mode 100644 index 0000000000000..a3939ad62acd9 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/inreg-inference.ll @@ -0,0 +1,246 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals +; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-attributor %s -o - | FileCheck %s + +@g1 = protected addrspace(1) externally_initialized global i32 0, align 4 +@g2 = protected addrspace(1) externally_initialized global i32 0, align 4 +@g3 = protected addrspace(1) externally_initialized global i32 0, align 4 +@g4 = protected addrspace(1) externally_initialized global i32 0, align 4 + +;. +; CHECK: @g1 = protected addrspace(1) externally_initialized global i32 0, align 4 +; CHECK: @g2 = protected addrspace(1) externally_initialized global i32 0, align 4 +; CHECK: @g3 = protected addrspace(1) externally_initialized global i32 0, align 4 +; CHECK: @g4 = protected addrspace(1) externally_initialized global i32 0, align 4 +;. +define internal fastcc void @callee_infer(ptr addrspace(1) %x, i32 %y) { +; CHECK-LABEL: define {{[^@]+}}@callee_infer +; CHECK-SAME: (ptr addrspace(1) inreg [[X:%.*]], i32 inreg [[Y:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[X_VAL:%.*]] = load i32, ptr addrspace(1) [[X]], align 4 +; CHECK-NEXT: store i32 [[X_VAL]], ptr addrspace(1) @g3, align 4 +; CHECK-NEXT: store i32 [[Y]], ptr addrspace(1) @g4, align 4 +; CHECK-NEXT: ret void +; +entry: + %x.val = load i32, ptr addrspace(1) %x, align 4 + store i32 %x.val, ptr addrspace(1) @g3, align 4 + store i32 %y, ptr addrspace(1) @g4, align 4 + ret void +} + +define amdgpu_kernel void @kernel_infer(ptr addrspace(1) %p1, ptr addrspace(1) %p2, i32 %x) { +; CHECK-LABEL: define {{[^@]+}}@kernel_infer +; CHECK-SAME: (ptr addrspace(1) [[P1:%.*]], ptr addrspace(1) [[P2:%.*]], i32 [[X:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[X]], 0 +; CHECK-NEXT: [[P:%.*]] = select i1 [[CMP]], ptr addrspace(1) [[P1]], ptr addrspace(1) [[P2]] +; CHECK-NEXT: tail call fastcc void @callee_infer(ptr addrspace(1) @g1, i32 [[X]]) +; CHECK-NEXT: tail call fastcc void @callee_infer(ptr addrspace(1) @g2, i32 [[X]]) +; CHECK-NEXT: tail call fastcc void @callee_infer(ptr addrspace(1) @g1, i32 1) +; CHECK-NEXT: tail call fastcc void @callee_infer(ptr addrspace(1) @g2, i32 2) +; CHECK-NEXT: tail call fastcc void @callee_infer(ptr addrspace(1) [[P]], i32 [[X]]) +; CHECK-NEXT: ret void +; +entry: + %cmp = icmp sgt i32 %x, 0 + %p = select i1 %cmp, ptr addrspace(1) %p1, ptr addrspace(1) %p2 + tail call fastcc void @callee_infer(ptr addrspace(1) @g1, i32 %x) + tail call fastcc void @callee_infer(ptr addrspace(1) @g2, i32 %x) + tail call fastcc void @callee_infer(ptr addrspace(1) @g1, i32 1) + tail call fastcc void @callee_infer(ptr addrspace(1) @g2, i32 2) + tail call fastcc void @callee_infer(ptr addrspace(1) %p, i32 %x) + ret void +} + +define amdgpu_kernel void @kernel_infer_indirect(ptr addrspace(1) %p1, ptr addrspace(1) %p2, i32 %x) { +; CHECK-LABEL: define {{[^@]+}}@kernel_infer_indirect +; CHECK-SAME: (ptr addrspace(1) [[P1:%.*]], ptr addrspace(1) [[P2:%.*]], i32 [[X:%.*]]) #[[ATTR1:[0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[FN:%.*]] = alloca ptr, align 8, addrspace(5) +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[X]], 0 +; CHECK-NEXT: [[P:%.*]] = select i1 [[CMP]], ptr addrspace(1) [[P1]], ptr addrspace(1) [[P2]] +; CHECK-NEXT: store ptr @kernel_infer, ptr addrspace(5) [[FN]], align 8 +; CHECK-NEXT: [[FN_CAST:%.*]] = addrspacecast ptr addrspace(5) [[FN]] to ptr +; CHECK-NEXT: tail call fastcc void [[FN_CAST]](ptr addrspace(1) @g1, i32 [[X]]) +; CHECK-NEXT: tail call fastcc void [[FN_CAST]](ptr addrspace(1) @g2, i32 [[X]]) +; CHECK-NEXT: tail call fastcc void [[FN_CAST]](ptr addrspace(1) @g1, i32 1) +; CHECK-NEXT: tail call fastcc void [[FN_CAST]](ptr addrspace(1) @g2, i32 2) +; CHECK-NEXT: tail call fastcc void [[FN_CAST]](ptr addrspace(1) [[P]], i32 [[X]]) +; CHECK-NEXT: ret void +; +entry: + %fn = alloca ptr, addrspace(5) + %cmp = icmp sgt i32 %x, 0 + %p = select i1 %cmp, ptr addrspace(1) %p1, ptr addrspace(1) %p2 + store ptr @kernel_infer, ptr addrspace(5) %fn + %fn.cast = addrspacecast ptr addrspace(5) %fn to ptr + tail call fastcc void %fn.cast(ptr addrspace(1) @g1, i32 %x) + tail call fastcc void %fn.cast(ptr addrspace(1) @g2, i32 %x) + tail call fastcc void %fn.cast(ptr addrspace(1) @g1, i32 1) + tail call fastcc void %fn.cast(ptr addrspace(1) @g2, i32 2) + tail call fastcc void %fn.cast(ptr addrspace(1) %p, i32 %x) + ret void +} + +define internal fastcc void @callee_not_infer(ptr addrspace(1) %x, i32 %y) { +; CHECK-LABEL: define {{[^@]+}}@callee_not_infer +; CHECK-SAME: (ptr addrspace(1) [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[X_VAL:%.*]] = load i32, ptr addrspace(1) [[X]], align 4 +; CHECK-NEXT: store i32 [[X_VAL]], ptr addrspace(1) @g3, align 4 +; CHECK-NEXT: store i32 [[Y]], ptr addrspace(1) @g4, align 4 +; CHECK-NEXT: ret void +; +entry: + %x.val = load i32, ptr addrspace(1) %x, align 4 + store i32 %x.val, ptr addrspace(1) @g3, align 4 + store i32 %y, ptr addrspace(1) @g4, align 4 + ret void +} + +define amdgpu_kernel void @kernel_not_infer(ptr addrspace(1) %q, ptr addrspace(1) %p1, ptr addrspace(1) %p2) { +; CHECK-LABEL: define {{[^@]+}}@kernel_not_infer +; CHECK-SAME: (ptr addrspace(1) [[Q:%.*]], ptr addrspace(1) [[P1:%.*]], ptr addrspace(1) [[P2:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ID_X:%.*]] = call i32 @llvm.amdgcn.workitem.id.x() +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[Q]], i32 [[ID_X]] +; CHECK-NEXT: [[D:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4 +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[D]], 0 +; CHECK-NEXT: [[P:%.*]] = select i1 [[CMP]], ptr addrspace(1) [[P1]], ptr addrspace(1) [[P2]] +; CHECK-NEXT: tail call fastcc void @callee_not_infer(ptr addrspace(1) [[Q]], i32 [[ID_X]]) +; CHECK-NEXT: tail call fastcc void @callee_not_infer(ptr addrspace(1) [[P]], i32 [[ID_X]]) +; CHECK-NEXT: ret void +; +entry: + %id.x = call i32 @llvm.amdgcn.workitem.id.x() + %gep = getelementptr i32, ptr addrspace(1) %q, i32 %id.x + %d = load i32, ptr addrspace(1) %gep + %cmp = icmp sgt i32 %d, 0 + %p = select i1 %cmp, ptr addrspace(1) %p1, ptr addrspace(1) %p2 + tail call fastcc void @callee_not_infer(ptr addrspace(1) %q, i32 %id.x) + tail call fastcc void @callee_not_infer(ptr addrspace(1) %p, i32 %id.x) + ret void +} + +define amdgpu_kernel void @kernel_not_infer_indirect(ptr addrspace(1) %q, ptr addrspace(1) %p1, ptr addrspace(1) %p2) { +; CHECK-LABEL: define {{[^@]+}}@kernel_not_infer_indirect +; CHECK-SAME: (ptr addrspace(1) [[Q:%.*]], ptr addrspace(1) [[P1:%.*]], ptr addrspace(1) [[P2:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[FN:%.*]] = alloca ptr, align 8, addrspace(5) +; CHECK-NEXT: [[ID_X:%.*]] = call i32 @llvm.amdgcn.workitem.id.x() +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[Q]], i32 [[ID_X]] +; CHECK-NEXT: [[D:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4 +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[D]], 0 +; CHECK-NEXT: [[P:%.*]] = select i1 [[CMP]], ptr addrspace(1) [[P1]], ptr addrspace(1) [[P2]] +; CHECK-NEXT: store ptr @kernel_not_infer, ptr addrspace(5) [[FN]], align 8 +; CHECK-NEXT: [[FN_CAST:%.*]] = addrspacecast ptr addrspace(5) [[FN]] to ptr +; CHECK-NEXT: tail call fastcc void [[FN_CAST]](ptr addrspace(1) [[Q]], i32 [[ID_X]]) +; CHECK-NEXT: tail call fastcc void [[FN_CAST]](ptr addrspace(1) [[P]], i32 [[ID_X]]) +; CHECK-NEXT: ret void +; +entry: + %fn = alloca ptr, addrspace(5) + %id.x = call i32 @llvm.amdgcn.workitem.id.x() + %gep = getelementptr i32, ptr addrspace(1) %q, i32 %id.x + %d = load i32, ptr addrspace(1) %gep + %cmp = icmp sgt i32 %d, 0 + %p = select i1 %cmp, ptr addrspace(1) %p1, ptr addrspace(1) %p2 + store ptr @kernel_not_infer, ptr addrspace(5) %fn + %fn.cast = addrspacecast ptr addrspace(5) %fn to ptr + tail call fastcc void %fn.cast(ptr addrspace(1) %q, i32 %id.x) + tail call fastcc void %fn.cast(ptr addrspace(1) %p, i32 %id.x) + ret void +} + +define internal fastcc void @cs_callee_not_infer(ptr addrspace(1) %x, i32 %y) { +; CHECK-LABEL: define {{[^@]+}}@cs_callee_not_infer +; CHECK-SAME: (ptr addrspace(1) [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[X_VAL:%.*]] = load i32, ptr addrspace(1) [[X]], align 4 +; CHECK-NEXT: store i32 [[X_VAL]], ptr addrspace(1) @g3, align 4 +; CHECK-NEXT: store i32 [[Y]], ptr addrspace(1) @g4, align 4 +; CHECK-NEXT: ret void +; +entry: + %x.val = load i32, ptr addrspace(1) %x, align 4 + store i32 %x.val, ptr addrspace(1) @g3, align 4 + store i32 %y, ptr addrspace(1) @g4, align 4 + ret void +} + +define amdgpu_cs void @cs_kernel_not_infer(ptr addrspace(1) %q, ptr addrspace(1) %p1, ptr addrspace(1) %p2) { +; CHECK-LABEL: define {{[^@]+}}@cs_kernel_not_infer +; CHECK-SAME: (ptr addrspace(1) [[Q:%.*]], ptr addrspace(1) [[P1:%.*]], ptr addrspace(1) [[P2:%.*]]) #[[ATTR2:[0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ID_X:%.*]] = call i32 @llvm.amdgcn.workitem.id.x() +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[Q]], i32 [[ID_X]] +; CHECK-NEXT: [[D:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4 +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[D]], 0 +; CHECK-NEXT: [[P:%.*]] = select i1 [[CMP]], ptr addrspace(1) [[P1]], ptr addrspace(1) [[P2]] +; CHECK-NEXT: tail call fastcc void @cs_callee_not_infer(ptr addrspace(1) [[Q]], i32 [[ID_X]]) +; CHECK-NEXT: tail call fastcc void @cs_callee_not_infer(ptr addrspace(1) [[P]], i32 [[ID_X]]) +; CHECK-NEXT: ret void +; +entry: + %id.x = call i32 @llvm.amdgcn.workitem.id.x() + %gep = getelementptr i32, ptr addrspace(1) %q, i32 %id.x + %d = load i32, ptr addrspace(1) %gep + %cmp = icmp sgt i32 %d, 0 + %p = select i1 %cmp, ptr addrspace(1) %p1, ptr addrspace(1) %p2 + tail call fastcc void @cs_callee_not_infer(ptr addrspace(1) %q, i32 %id.x) + tail call fastcc void @cs_callee_not_infer(ptr addrspace(1) %p, i32 %id.x) + ret void +} + +define internal fastcc void @cs_callee_not_infer_indirect(ptr addrspace(1) %x, i32 %y) { +; CHECK-LABEL: define {{[^@]+}}@cs_callee_not_infer_indirect +; CHECK-SAME: (ptr addrspace(1) [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[X_VAL:%.*]] = load i32, ptr addrspace(1) [[X]], align 4 +; CHECK-NEXT: store i32 [[X_VAL]], ptr addrspace(1) @g3, align 4 +; CHECK-NEXT: store i32 [[Y]], ptr addrspace(1) @g4, align 4 +; CHECK-NEXT: ret void +; +entry: + %x.val = load i32, ptr addrspace(1) %x, align 4 + store i32 %x.val, ptr addrspace(1) @g3, align 4 + store i32 %y, ptr addrspace(1) @g4, align 4 + ret void +} + + +define amdgpu_cs void @cs_kernel_not_infer_indirect(ptr addrspace(1) %q, ptr addrspace(1) %p1, ptr addrspace(1) %p2) { +; CHECK-LABEL: define {{[^@]+}}@cs_kernel_not_infer_indirect +; CHECK-SAME: (ptr addrspace(1) [[Q:%.*]], ptr addrspace(1) [[P1:%.*]], ptr addrspace(1) [[P2:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[FN:%.*]] = alloca ptr, align 8, addrspace(5) +; CHECK-NEXT: [[ID_X:%.*]] = call i32 @llvm.amdgcn.workitem.id.x() +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[Q]], i32 [[ID_X]] +; CHECK-NEXT: [[D:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4 +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[D]], 0 +; CHECK-NEXT: [[P:%.*]] = select i1 [[CMP]], ptr addrspace(1) [[P1]], ptr addrspace(1) [[P2]] +; CHECK-NEXT: store ptr @cs_callee_not_infer_indirect, ptr addrspace(5) [[FN]], align 8 +; CHECK-NEXT: [[FN_CAST:%.*]] = addrspacecast ptr addrspace(5) [[FN]] to ptr +; CHECK-NEXT: tail call fastcc void [[FN_CAST]](ptr addrspace(1) [[Q]], i32 [[ID_X]]) +; CHECK-NEXT: tail call fastcc void [[FN_CAST]](ptr addrspace(1) [[P]], i32 [[ID_X]]) +; CHECK-NEXT: ret void +; +entry: + %fn = alloca ptr, addrspace(5) + %id.x = call i32 @llvm.amdgcn.workitem.id.x() + %gep = getelementptr i32, ptr addrspace(1) %q, i32 %id.x + %d = load i32, ptr addrspace(1) %gep + %cmp = icmp sgt i32 %d, 0 + %p = select i1 %cmp, ptr addrspace(1) %p1, ptr addrspace(1) %p2 + store ptr @cs_callee_not_infer_indirect, ptr addrspace(5) %fn + %fn.cast = addrspacecast ptr addrspace(5) %fn to ptr + tail call fastcc void %fn.cast(ptr addrspace(1) %q, i32 %id.x) + tail call fastcc void %fn.cast(ptr addrspace(1) %p, i32 %id.x) + ret void +} +;. +; CHECK: attributes #[[ATTR0]] = { "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR2]] = { "amdgpu-agpr-alloc"="0" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR3:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +;. diff --git a/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll b/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll index ed4e691fbf154..dddaadc471b35 100644 --- a/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll +++ b/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll @@ -148,7 +148,7 @@ define amdgpu_kernel void @kernel_lds() { define internal i16 @mutual_recursion_0(i16 %arg) { ; CHECK-LABEL: define internal i16 @mutual_recursion_0( -; CHECK-SAME: i16 [[ARG:%.*]]) #[[ATTR0]] { +; CHECK-SAME: i16 inreg [[ARG:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.lds.kernel.id() ; CHECK-NEXT: [[RECURSIVE_KERNEL_LDS:%.*]] = getelementptr inbounds [3 x [2 x i32]], ptr addrspace(4) @llvm.amdgcn.lds.offset.table, i32 0, i32 [[TMP1]], i32 1 ; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(4) [[RECURSIVE_KERNEL_LDS]], align 4 @@ -168,7 +168,7 @@ define internal i16 @mutual_recursion_0(i16 %arg) { define internal void @mutual_recursion_1(i16 %arg) { ; CHECK-LABEL: define internal void @mutual_recursion_1( -; CHECK-SAME: i16 [[ARG:%.*]]) #[[ATTR0]] { +; CHECK-SAME: i16 inreg [[ARG:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: call void @mutual_recursion_0(i16 [[ARG]]) ; CHECK-NEXT: ret void ;