From 65a5f9d13d2d60c8b9d4b0486b2aa25a19666e38 Mon Sep 17 00:00:00 2001 From: Dominik Adamski Date: Mon, 11 Sep 2023 05:31:37 -0400 Subject: [PATCH] [OpenMPIRBuilder]Add support to omp target parallel Added support for LLVM IR code generation which is used for handling omp target parallel code. The call for __kmpc_parallel_51 is generated and the parallel region is outlined to separate function. The proper setup of kmpc_target_init mode is not included in the commit. It is assumed that the SPMD mode for target init is properly set by other codegen functions. --- llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 311 +++++++++++++----- llvm/lib/Transforms/IPO/OpenMPOpt.cpp | 1 + .../Frontend/OpenMPIRBuilderTest.cpp | 137 +++++++- 3 files changed, 361 insertions(+), 88 deletions(-) diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index 46db9c209a6a4..6b7ab48a93130 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -651,6 +651,13 @@ void OpenMPIRBuilder::finalize(Function *Fn) { Function *OuterFn = OI.getFunction(); CodeExtractorAnalysisCache CEAC(*OuterFn); + // If we generate code for the target device, we need to allocate + // struct for aggregate params in the device default alloca address space. + // OpenMP runtime requires that the params of the extracted functions are + // passed as zero address space pointers. This flag ensures that + // CodeExtractor generates correct code for extracted functions + // which are used by OpenMP runtime. + bool ArgsInZeroAddressSpace = Config.isTargetDevice(); CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr, /* AggregateArgs */ true, /* BlockFrequencyInfo */ nullptr, @@ -659,7 +666,7 @@ void OpenMPIRBuilder::finalize(Function *Fn) { /* AllowVarArgs */ true, /* AllowAlloca */ true, /* AllocaBlock*/ OI.OuterAllocaBB, - /* Suffix */ ".omp_par"); + /* Suffix */ ".omp_par", ArgsInZeroAddressSpace); LLVM_DEBUG(dbgs() << "Before outlining: " << *OuterFn << "\n"); LLVM_DEBUG(dbgs() << "Entry " << OI.EntryBB->getName() @@ -1101,6 +1108,182 @@ void OpenMPIRBuilder::emitCancelationCheckImpl(Value *CancelFlag, Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin()); } +// Callback used to create OpenMP runtime calls to support +// omp parallel clause for the device. +// We need to use this callback to replace call to the OutlinedFn in OuterFn +// by the call to the OpenMP DeviceRTL runtime function (kmpc_parallel_51) +static void targetParallelCallback( + OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn, Function *OuterFn, + BasicBlock *OuterAllocaBB, Value *Ident, Value *IfCondition, + Value *NumThreads, Instruction *PrivTID, AllocaInst *PrivTIDAddr, + Value *ThreadID, const SmallVector &ToBeDeleted) { + // Add some known attributes. + IRBuilder<> &Builder = OMPIRBuilder->Builder; + OutlinedFn.addParamAttr(0, Attribute::NoAlias); + OutlinedFn.addParamAttr(1, Attribute::NoAlias); + OutlinedFn.addParamAttr(0, Attribute::NoUndef); + OutlinedFn.addParamAttr(1, Attribute::NoUndef); + OutlinedFn.addFnAttr(Attribute::NoUnwind); + + assert(OutlinedFn.arg_size() >= 2 && + "Expected at least tid and bounded tid as arguments"); + unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2; + + CallInst *CI = cast(OutlinedFn.user_back()); + assert(CI && "Expected call instruction to outlined function"); + CI->getParent()->setName("omp_parallel"); + + Builder.SetInsertPoint(CI); + Type *PtrTy = OMPIRBuilder->VoidPtr; + Value *NullPtrValue = Constant::getNullValue(PtrTy); + + // Add alloca for kernel args + OpenMPIRBuilder ::InsertPointTy CurrentIP = Builder.saveIP(); + Builder.SetInsertPoint(OuterAllocaBB, OuterAllocaBB->getFirstInsertionPt()); + AllocaInst *ArgsAlloca = + Builder.CreateAlloca(ArrayType::get(PtrTy, NumCapturedVars)); + Value *Args = ArgsAlloca; + // Add address space cast if array for storing arguments is not allocated + // in address space 0 + if (ArgsAlloca->getAddressSpace()) + Args = Builder.CreatePointerCast(ArgsAlloca, PtrTy); + Builder.restoreIP(CurrentIP); + + // Store captured vars which are used by kmpc_parallel_51 + for (unsigned Idx = 0; Idx < NumCapturedVars; Idx++) { + Value *V = *(CI->arg_begin() + 2 + Idx); + Value *StoreAddress = Builder.CreateConstInBoundsGEP2_64( + ArrayType::get(PtrTy, NumCapturedVars), Args, 0, Idx); + Builder.CreateStore(V, StoreAddress); + } + + Value *Cond = + IfCondition ? Builder.CreateSExtOrTrunc(IfCondition, OMPIRBuilder->Int32) + : Builder.getInt32(1); + + // Build kmpc_parallel_51 call + Value *Parallel51CallArgs[] = { + /* identifier*/ Ident, + /* global thread num*/ ThreadID, + /* if expression */ Cond, + /* number of threads */ NumThreads ? NumThreads : Builder.getInt32(-1), + /* Proc bind */ Builder.getInt32(-1), + /* outlined function */ + Builder.CreateBitCast(&OutlinedFn, OMPIRBuilder->ParallelTaskPtr), + /* wrapper function */ NullPtrValue, + /* arguments of the outlined funciton*/ Args, + /* number of arguments */ Builder.getInt64(NumCapturedVars)}; + + FunctionCallee RTLFn = + OMPIRBuilder->getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_parallel_51); + + Builder.CreateCall(RTLFn, Parallel51CallArgs); + + LLVM_DEBUG(dbgs() << "With kmpc_parallel_51 placed: " + << *Builder.GetInsertBlock()->getParent() << "\n"); + + // Initialize the local TID stack location with the argument value. + Builder.SetInsertPoint(PrivTID); + Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin(); + Builder.CreateStore(Builder.CreateLoad(OMPIRBuilder->Int32, OutlinedAI), + PrivTIDAddr); + + // Remove redundant call to the outlined function. + CI->eraseFromParent(); + + for (Instruction *I : ToBeDeleted) { + I->eraseFromParent(); + } +} + +// Callback used to create OpenMP runtime calls to support +// omp parallel clause for the host. +// We need to use this callback to replace call to the OutlinedFn in OuterFn +// by the call to the OpenMP host runtime function ( __kmpc_fork_call[_if]) +static void +hostParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn, + Function *OuterFn, Value *Ident, Value *IfCondition, + Instruction *PrivTID, AllocaInst *PrivTIDAddr, + const SmallVector &ToBeDeleted) { + IRBuilder<> &Builder = OMPIRBuilder->Builder; + FunctionCallee RTLFn; + if (IfCondition) { + RTLFn = + OMPIRBuilder->getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call_if); + } else { + RTLFn = + OMPIRBuilder->getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call); + } + if (auto *F = dyn_cast(RTLFn.getCallee())) { + if (!F->hasMetadata(LLVMContext::MD_callback)) { + LLVMContext &Ctx = F->getContext(); + MDBuilder MDB(Ctx); + // Annotate the callback behavior of the __kmpc_fork_call: + // - The callback callee is argument number 2 (microtask). + // - The first two arguments of the callback callee are unknown (-1). + // - All variadic arguments to the __kmpc_fork_call are passed to the + // callback callee. + F->addMetadata(LLVMContext::MD_callback, + *MDNode::get(Ctx, {MDB.createCallbackEncoding( + 2, {-1, -1}, + /* VarArgsArePassed */ true)})); + } + } + // Add some known attributes. + OutlinedFn.addParamAttr(0, Attribute::NoAlias); + OutlinedFn.addParamAttr(1, Attribute::NoAlias); + OutlinedFn.addFnAttr(Attribute::NoUnwind); + + assert(OutlinedFn.arg_size() >= 2 && + "Expected at least tid and bounded tid as arguments"); + unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2; + + CallInst *CI = cast(OutlinedFn.user_back()); + CI->getParent()->setName("omp_parallel"); + Builder.SetInsertPoint(CI); + + // Build call __kmpc_fork_call[_if](Ident, n, microtask, var1, .., varn); + Value *ForkCallArgs[] = { + Ident, Builder.getInt32(NumCapturedVars), + Builder.CreateBitCast(&OutlinedFn, OMPIRBuilder->ParallelTaskPtr)}; + + SmallVector RealArgs; + RealArgs.append(std::begin(ForkCallArgs), std::end(ForkCallArgs)); + if (IfCondition) { + Value *Cond = Builder.CreateSExtOrTrunc(IfCondition, OMPIRBuilder->Int32); + RealArgs.push_back(Cond); + } + RealArgs.append(CI->arg_begin() + /* tid & bound tid */ 2, CI->arg_end()); + + // __kmpc_fork_call_if always expects a void ptr as the last argument + // If there are no arguments, pass a null pointer. + auto PtrTy = OMPIRBuilder->VoidPtr; + if (IfCondition && NumCapturedVars == 0) { + Value *NullPtrValue = Constant::getNullValue(PtrTy); + RealArgs.push_back(NullPtrValue); + } + if (IfCondition && RealArgs.back()->getType() != PtrTy) + RealArgs.back() = Builder.CreateBitCast(RealArgs.back(), PtrTy); + + Builder.CreateCall(RTLFn, RealArgs); + + LLVM_DEBUG(dbgs() << "With fork_call placed: " + << *Builder.GetInsertBlock()->getParent() << "\n"); + + // Initialize the local TID stack location with the argument value. + Builder.SetInsertPoint(PrivTID); + Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin(); + Builder.CreateStore(Builder.CreateLoad(OMPIRBuilder->Int32, OutlinedAI), + PrivTIDAddr); + + // Remove redundant call to the outlined function. + CI->eraseFromParent(); + + for (Instruction *I : ToBeDeleted) { + I->eraseFromParent(); + } +} + IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel( const LocationDescription &Loc, InsertPointTy OuterAllocaIP, BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB, @@ -1115,6 +1298,12 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel( Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize); Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize); Value *ThreadID = getOrCreateThreadID(Ident); + // If we generate code for the target device, we need to allocate + // struct for aggregate params in the device default alloca address space. + // OpenMP runtime requires that the params of the extracted functions are + // passed as zero address space pointers. This flag ensures that extracted + // function arguments are declared in zero address space + bool ArgsInZeroAddressSpace = Config.isTargetDevice(); if (NumThreads) { // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads) @@ -1148,13 +1337,28 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel( // Change the location to the outer alloca insertion point to create and // initialize the allocas we pass into the parallel region. Builder.restoreIP(OuterAllocaIP); - AllocaInst *TIDAddr = Builder.CreateAlloca(Int32, nullptr, "tid.addr"); - AllocaInst *ZeroAddr = Builder.CreateAlloca(Int32, nullptr, "zero.addr"); + AllocaInst *TIDAddrAlloca = Builder.CreateAlloca(Int32, nullptr, "tid.addr"); + AllocaInst *ZeroAddrAlloca = + Builder.CreateAlloca(Int32, nullptr, "zero.addr"); + Instruction *TIDAddr = TIDAddrAlloca; + Instruction *ZeroAddr = ZeroAddrAlloca; + if (ArgsInZeroAddressSpace && M.getDataLayout().getAllocaAddrSpace() != 0) { + // Add additional casts to enforce pointers in zero address space + TIDAddr = new AddrSpaceCastInst( + TIDAddrAlloca, PointerType ::get(M.getContext(), 0), "tid.addr.ascast"); + TIDAddr->insertAfter(TIDAddrAlloca); + ToBeDeleted.push_back(TIDAddr); + ZeroAddr = new AddrSpaceCastInst(ZeroAddrAlloca, + PointerType ::get(M.getContext(), 0), + "zero.addr.ascast"); + ZeroAddr->insertAfter(ZeroAddrAlloca); + ToBeDeleted.push_back(ZeroAddr); + } // We only need TIDAddr and ZeroAddr for modeling purposes to get the // associated arguments in the outlined function, so we delete them later. - ToBeDeleted.push_back(TIDAddr); - ToBeDeleted.push_back(ZeroAddr); + ToBeDeleted.push_back(TIDAddrAlloca); + ToBeDeleted.push_back(ZeroAddrAlloca); // Create an artificial insertion point that will also ensure the blocks we // are about to split are not degenerated. @@ -1222,87 +1426,24 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel( BodyGenCB(InnerAllocaIP, CodeGenIP); LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n"); - FunctionCallee RTLFn; - if (IfCondition) - RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call_if); - else - RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call); - - if (auto *F = dyn_cast(RTLFn.getCallee())) { - if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) { - llvm::LLVMContext &Ctx = F->getContext(); - MDBuilder MDB(Ctx); - // Annotate the callback behavior of the __kmpc_fork_call: - // - The callback callee is argument number 2 (microtask). - // - The first two arguments of the callback callee are unknown (-1). - // - All variadic arguments to the __kmpc_fork_call are passed to the - // callback callee. - F->addMetadata( - llvm::LLVMContext::MD_callback, - *llvm::MDNode::get( - Ctx, {MDB.createCallbackEncoding(2, {-1, -1}, - /* VarArgsArePassed */ true)})); - } - } OutlineInfo OI; - OI.PostOutlineCB = [=](Function &OutlinedFn) { - // Add some known attributes. - OutlinedFn.addParamAttr(0, Attribute::NoAlias); - OutlinedFn.addParamAttr(1, Attribute::NoAlias); - OutlinedFn.addFnAttr(Attribute::NoUnwind); - OutlinedFn.addFnAttr(Attribute::NoRecurse); - - assert(OutlinedFn.arg_size() >= 2 && - "Expected at least tid and bounded tid as arguments"); - unsigned NumCapturedVars = - OutlinedFn.arg_size() - /* tid & bounded tid */ 2; - - CallInst *CI = cast(OutlinedFn.user_back()); - CI->getParent()->setName("omp_parallel"); - Builder.SetInsertPoint(CI); - - // Build call __kmpc_fork_call[_if](Ident, n, microtask, var1, .., varn); - Value *ForkCallArgs[] = { - Ident, Builder.getInt32(NumCapturedVars), - Builder.CreateBitCast(&OutlinedFn, ParallelTaskPtr)}; - - SmallVector RealArgs; - RealArgs.append(std::begin(ForkCallArgs), std::end(ForkCallArgs)); - if (IfCondition) { - Value *Cond = Builder.CreateSExtOrTrunc(IfCondition, - Type::getInt32Ty(M.getContext())); - RealArgs.push_back(Cond); - } - RealArgs.append(CI->arg_begin() + /* tid & bound tid */ 2, CI->arg_end()); - - // __kmpc_fork_call_if always expects a void ptr as the last argument - // If there are no arguments, pass a null pointer. - auto PtrTy = Type::getInt8PtrTy(M.getContext()); - if (IfCondition && NumCapturedVars == 0) { - llvm::Value *Void = ConstantPointerNull::get(PtrTy); - RealArgs.push_back(Void); - } - if (IfCondition && RealArgs.back()->getType() != PtrTy) - RealArgs.back() = Builder.CreateBitCast(RealArgs.back(), PtrTy); - - Builder.CreateCall(RTLFn, RealArgs); - - LLVM_DEBUG(dbgs() << "With fork_call placed: " - << *Builder.GetInsertBlock()->getParent() << "\n"); - - InsertPointTy ExitIP(PRegExitBB, PRegExitBB->end()); - - // Initialize the local TID stack location with the argument value. - Builder.SetInsertPoint(PrivTID); - Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin(); - Builder.CreateStore(Builder.CreateLoad(Int32, OutlinedAI), PrivTIDAddr); - - CI->eraseFromParent(); - - for (Instruction *I : ToBeDeleted) - I->eraseFromParent(); - }; + if (Config.isTargetDevice()) { + // Generate OpenMP target specific runtime call + OI.PostOutlineCB = [=, ToBeDeletedVec = + std::move(ToBeDeleted)](Function &OutlinedFn) { + targetParallelCallback(this, OutlinedFn, OuterFn, OuterAllocaBlock, Ident, + IfCondition, NumThreads, PrivTID, PrivTIDAddr, + ThreadID, ToBeDeletedVec); + }; + } else { + // Generate OpenMP host runtime call + OI.PostOutlineCB = [=, ToBeDeletedVec = + std::move(ToBeDeleted)](Function &OutlinedFn) { + hostParallelCallback(this, OutlinedFn, OuterFn, Ident, IfCondition, + PrivTID, PrivTIDAddr, ToBeDeletedVec); + }; + } // Adjust the finalization stack, verify the adjustment, and call the // finalize function a last time to finalize values between the pre-fini @@ -1342,7 +1483,7 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel( /* AllowVarArgs */ true, /* AllowAlloca */ true, /* AllocationBlock */ OuterAllocaBlock, - /* Suffix */ ".omp_par"); + /* Suffix */ ".omp_par", ArgsInZeroAddressSpace); // Find inputs to, outputs from the code region. BasicBlock *CommonExit = nullptr; diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp index fe250047759cd..3ef9755a5dc2c 100644 --- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp +++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp @@ -286,6 +286,7 @@ struct OMPInformationCache : public InformationCache { : InformationCache(M, AG, Allocator, CGSCC), OMPBuilder(M), OpenMPPostLink(OpenMPPostLink) { + OMPBuilder.Config.IsTargetDevice = isOpenMPDevice(OMPBuilder.M); OMPBuilder.initialize(); initializeRuntimeFunctions(M); initializeInternalControlVars(); diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp index b9e9655b89108..58aea370e2719 100644 --- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp +++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp @@ -591,9 +591,122 @@ TEST_F(OpenMPIRBuilderTest, DbgLoc) { EXPECT_EQ(SrcSrc->getAsCString(), ";/src/test.dbg;foo;3;7;;"); } +TEST_F(OpenMPIRBuilderTest, ParallelSimpleGPU) { + using InsertPointTy = OpenMPIRBuilder::InsertPointTy; + std::string oldDLStr = M->getDataLayoutStr(); + M->setDataLayout( + "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:" + "256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:" + "256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"); + OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = true; + OMPBuilder.initialize(); + F->setName("func"); + IRBuilder<> Builder(BB); + BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F); + Builder.CreateBr(EnterBB); + Builder.SetInsertPoint(EnterBB); + OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); + + AllocaInst *PrivAI = nullptr; + + unsigned NumBodiesGenerated = 0; + unsigned NumPrivatizedVars = 0; + unsigned NumFinalizationPoints = 0; + + auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) { + ++NumBodiesGenerated; + + Builder.restoreIP(AllocaIP); + PrivAI = Builder.CreateAlloca(F->arg_begin()->getType()); + Builder.CreateStore(F->arg_begin(), PrivAI); + + Builder.restoreIP(CodeGenIP); + Value *PrivLoad = + Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use"); + Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad); + Instruction *ThenTerm, *ElseTerm; + SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(), + &ThenTerm, &ElseTerm); + }; + + auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, + Value &Orig, Value &Inner, + Value *&ReplacementValue) -> InsertPointTy { + ++NumPrivatizedVars; + + if (!isa(Orig)) { + EXPECT_EQ(&Orig, F->arg_begin()); + ReplacementValue = &Inner; + return CodeGenIP; + } + + // Since the original value is an allocation, it has a pointer type and + // therefore no additional wrapping should happen. + EXPECT_EQ(&Orig, &Inner); + + // Trivial copy (=firstprivate). + Builder.restoreIP(AllocaIP); + Type *VTy = ReplacementValue->getType(); + Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload"); + ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy"); + Builder.restoreIP(CodeGenIP); + Builder.CreateStore(V, ReplacementValue); + return CodeGenIP; + }; + + auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; }; + + IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), + F->getEntryBlock().getFirstInsertionPt()); + IRBuilder<>::InsertPoint AfterIP = + OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, + nullptr, nullptr, OMP_PROC_BIND_default, false); + + EXPECT_EQ(NumBodiesGenerated, 1U); + EXPECT_EQ(NumPrivatizedVars, 1U); + EXPECT_EQ(NumFinalizationPoints, 1U); + + Builder.restoreIP(AfterIP); + Builder.CreateRetVoid(); + + OMPBuilder.finalize(); + Function *OutlinedFn = PrivAI->getFunction(); + EXPECT_FALSE(verifyModule(*M, &errs())); + EXPECT_NE(OutlinedFn, F); + EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoUnwind)); + EXPECT_TRUE(OutlinedFn->hasParamAttribute(0, Attribute::NoAlias)); + EXPECT_TRUE(OutlinedFn->hasParamAttribute(1, Attribute::NoAlias)); + + EXPECT_TRUE(OutlinedFn->hasInternalLinkage()); + EXPECT_EQ(OutlinedFn->arg_size(), 3U); + // Make sure that arguments are pointers in 0 address address space + EXPECT_EQ(OutlinedFn->getArg(0)->getType(), + PointerType::get(M->getContext(), 0)); + EXPECT_EQ(OutlinedFn->getArg(1)->getType(), + PointerType::get(M->getContext(), 0)); + EXPECT_EQ(OutlinedFn->getArg(2)->getType(), + PointerType::get(M->getContext(), 0)); + EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent()); + EXPECT_EQ(OutlinedFn->getNumUses(), 1U); + User *Usr = OutlinedFn->user_back(); + ASSERT_TRUE(isa(Usr)); + CallInst *Parallel51CI = dyn_cast(Usr); + ASSERT_NE(Parallel51CI, nullptr); + + EXPECT_EQ(Parallel51CI->getCalledFunction()->getName(), "__kmpc_parallel_51"); + EXPECT_EQ(Parallel51CI->arg_size(), 9U); + EXPECT_EQ(Parallel51CI->getArgOperand(5), OutlinedFn); + EXPECT_TRUE( + isa(Parallel51CI->getArgOperand(0)->stripPointerCasts())); + EXPECT_EQ(Parallel51CI, Usr); + M->setDataLayout(oldDLStr); +} + TEST_F(OpenMPIRBuilderTest, ParallelSimple) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); @@ -671,7 +784,6 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimple) { EXPECT_NE(F, OutlinedFn); EXPECT_FALSE(verifyModule(*M, &errs())); EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoUnwind)); - EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoRecurse)); EXPECT_TRUE(OutlinedFn->hasParamAttribute(0, Attribute::NoAlias)); EXPECT_TRUE(OutlinedFn->hasParamAttribute(1, Attribute::NoAlias)); @@ -699,6 +811,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimple) { TEST_F(OpenMPIRBuilderTest, ParallelNested) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); @@ -768,7 +881,6 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested) { continue; EXPECT_FALSE(verifyModule(*M, &errs())); EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind)); - EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse)); EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias)); EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias)); @@ -793,6 +905,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested) { TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); @@ -872,7 +985,6 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) { continue; EXPECT_FALSE(verifyModule(*M, &errs())); EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind)); - EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse)); EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias)); EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias)); @@ -902,6 +1014,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) { TEST_F(OpenMPIRBuilderTest, ParallelIfCond) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); @@ -1006,6 +1119,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelIfCond) { TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); @@ -1119,6 +1233,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) { TEST_F(OpenMPIRBuilderTest, ParallelForwardAsPointers) { OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); @@ -4004,6 +4119,7 @@ TEST_F(OpenMPIRBuilderTest, OMPAtomicCompareCapture) { TEST_F(OpenMPIRBuilderTest, CreateTeams) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); @@ -4079,6 +4195,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTeams) { TEST_F(OpenMPIRBuilderTest, CreateTeamsWithThreadLimit) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> &Builder = OMPBuilder.Builder; @@ -4129,6 +4246,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithThreadLimit) { TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsUpper) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> &Builder = OMPBuilder.Builder; @@ -4180,6 +4298,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsUpper) { TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsBoth) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> &Builder = OMPBuilder.Builder; @@ -4234,6 +4353,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsBoth) { TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsAndThreadLimit) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> &Builder = OMPBuilder.Builder; @@ -4293,6 +4413,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsAndThreadLimit) { TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfCondition) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> &Builder = OMPBuilder.Builder; @@ -4351,6 +4472,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfCondition) { TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfConditionAndNumTeams) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> &Builder = OMPBuilder.Builder; @@ -4548,6 +4670,7 @@ xorAtomicReduction(OpenMPIRBuilder::InsertPointTy IP, Type *Ty, Value *LHS, TEST_F(OpenMPIRBuilderTest, CreateReductions) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); @@ -4780,6 +4903,7 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) { TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); @@ -5797,6 +5921,7 @@ TEST_F(OpenMPIRBuilderTest, TargetRegionDevice) { TEST_F(OpenMPIRBuilderTest, CreateTask) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); @@ -5925,6 +6050,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTask) { TEST_F(OpenMPIRBuilderTest, CreateTaskNoArgs) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); @@ -5955,6 +6081,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskNoArgs) { TEST_F(OpenMPIRBuilderTest, CreateTaskUntied) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); @@ -5984,6 +6111,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskUntied) { TEST_F(OpenMPIRBuilderTest, CreateTaskDepend) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); @@ -6057,6 +6185,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskDepend) { TEST_F(OpenMPIRBuilderTest, CreateTaskFinal) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); @@ -6110,6 +6239,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskFinal) { TEST_F(OpenMPIRBuilderTest, CreateTaskIfCondition) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB); @@ -6270,6 +6400,7 @@ TEST_F(OpenMPIRBuilderTest, CreateTaskgroup) { TEST_F(OpenMPIRBuilderTest, CreateTaskgroupWithTasks) { using InsertPointTy = OpenMPIRBuilder::InsertPointTy; OpenMPIRBuilder OMPBuilder(*M); + OMPBuilder.Config.IsTargetDevice = false; OMPBuilder.initialize(); F->setName("func"); IRBuilder<> Builder(BB);