diff --git a/clang/include/clang/AST/ExprCXX.h b/clang/include/clang/AST/ExprCXX.h index a0e467b35778c..6003b866c9f56 100644 --- a/clang/include/clang/AST/ExprCXX.h +++ b/clang/include/clang/AST/ExprCXX.h @@ -5038,6 +5038,9 @@ class CoroutineSuspendExpr : public Expr { OpaqueValueExpr *OpaqueValue = nullptr; public: + // These types correspond to the three C++ 'await_suspend' return variants + enum class SuspendReturnType { SuspendVoid, SuspendBool, SuspendHandle }; + CoroutineSuspendExpr(StmtClass SC, SourceLocation KeywordLoc, Expr *Operand, Expr *Common, Expr *Ready, Expr *Suspend, Expr *Resume, OpaqueValueExpr *OpaqueValue) @@ -5097,6 +5100,24 @@ class CoroutineSuspendExpr : public Expr { return static_cast(SubExprs[SubExpr::Operand]); } + SuspendReturnType getSuspendReturnType() const { + auto *SuspendExpr = getSuspendExpr(); + assert(SuspendExpr); + + auto SuspendType = SuspendExpr->getType(); + + if (SuspendType->isVoidType()) + return SuspendReturnType::SuspendVoid; + if (SuspendType->isBooleanType()) + return SuspendReturnType::SuspendBool; + + // Void pointer is the type of handle.address(), which is returned + // from the await suspend wrapper so that the temporary coroutine handle + // value won't go to the frame by mistake + assert(SuspendType->isVoidPointerType()); + return SuspendReturnType::SuspendHandle; + } + SourceLocation getKeywordLoc() const { return KeywordLoc; } SourceLocation getBeginLoc() const LLVM_READONLY { return KeywordLoc; } diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp index 888d30bfb3e1d..b7142ec08af98 100644 --- a/clang/lib/CodeGen/CGCoroutine.cpp +++ b/clang/lib/CodeGen/CGCoroutine.cpp @@ -141,7 +141,7 @@ static bool FunctionCanThrow(const FunctionDecl *D) { Proto->canThrow() != CT_Cannot; } -static bool ResumeStmtCanThrow(const Stmt *S) { +static bool StmtCanThrow(const Stmt *S) { if (const auto *CE = dyn_cast(S)) { const auto *Callee = CE->getDirectCallee(); if (!Callee) @@ -167,7 +167,7 @@ static bool ResumeStmtCanThrow(const Stmt *S) { } for (const auto *child : S->children()) - if (ResumeStmtCanThrow(child)) + if (StmtCanThrow(child)) return true; return false; @@ -178,18 +178,31 @@ static bool ResumeStmtCanThrow(const Stmt *S) { // auto && x = CommonExpr(); // if (!x.await_ready()) { // llvm_coro_save(); -// x.await_suspend(...); (*) -// llvm_coro_suspend(); (**) +// llvm_coro_await_suspend(&x, frame, wrapper) (*) (**) +// llvm_coro_suspend(); (***) // } // x.await_resume(); // // where the result of the entire expression is the result of x.await_resume() // -// (*) If x.await_suspend return type is bool, it allows to veto a suspend: +// (*) llvm_coro_await_suspend_{void, bool, handle} is lowered to +// wrapper(&x, frame) when it's certain not to interfere with +// coroutine transform. await_suspend expression is +// asynchronous to the coroutine body and not all analyses +// and transformations can handle it correctly at the moment. +// +// Wrapper function encapsulates x.await_suspend(...) call and looks like: +// +// auto __await_suspend_wrapper(auto& awaiter, void* frame) { +// std::coroutine_handle<> handle(frame); +// return awaiter.await_suspend(handle); +// } +// +// (**) If x.await_suspend return type is bool, it allows to veto a suspend: // if (x.await_suspend(...)) // llvm_coro_suspend(); // -// (**) llvm_coro_suspend() encodes three possible continuations as +// (***) llvm_coro_suspend() encodes three possible continuations as // a switch instruction: // // %where-to = call i8 @llvm.coro.suspend(...) @@ -212,9 +225,10 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co bool ignoreResult, bool forLValue) { auto *E = S.getCommonExpr(); - auto Binder = + auto CommonBinder = CodeGenFunction::OpaqueValueMappingData::bind(CGF, S.getOpaqueValue(), E); - auto UnbindOnExit = llvm::make_scope_exit([&] { Binder.unbind(CGF); }); + auto UnbindCommonOnExit = + llvm::make_scope_exit([&] { CommonBinder.unbind(CGF); }); auto Prefix = buildSuspendPrefixStr(Coro, Kind); BasicBlock *ReadyBlock = CGF.createBasicBlock(Prefix + Twine(".ready")); @@ -232,16 +246,73 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co auto *NullPtr = llvm::ConstantPointerNull::get(CGF.CGM.Int8PtrTy); auto *SaveCall = Builder.CreateCall(CoroSave, {NullPtr}); + auto SuspendWrapper = CodeGenFunction(CGF.CGM).generateAwaitSuspendWrapper( + CGF.CurFn->getName(), Prefix, S); + CGF.CurCoro.InSuspendBlock = true; - auto *SuspendRet = CGF.EmitScalarExpr(S.getSuspendExpr()); + + assert(CGF.CurCoro.Data && CGF.CurCoro.Data->CoroBegin && + "expected to be called in coroutine context"); + + SmallVector SuspendIntrinsicCallArgs; + SuspendIntrinsicCallArgs.push_back( + CGF.getOrCreateOpaqueLValueMapping(S.getOpaqueValue()).getPointer(CGF)); + + SuspendIntrinsicCallArgs.push_back(CGF.CurCoro.Data->CoroBegin); + SuspendIntrinsicCallArgs.push_back(SuspendWrapper); + + const auto SuspendReturnType = S.getSuspendReturnType(); + llvm::Intrinsic::ID AwaitSuspendIID; + + switch (SuspendReturnType) { + case CoroutineSuspendExpr::SuspendReturnType::SuspendVoid: + AwaitSuspendIID = llvm::Intrinsic::coro_await_suspend_void; + break; + case CoroutineSuspendExpr::SuspendReturnType::SuspendBool: + AwaitSuspendIID = llvm::Intrinsic::coro_await_suspend_bool; + break; + case CoroutineSuspendExpr::SuspendReturnType::SuspendHandle: + AwaitSuspendIID = llvm::Intrinsic::coro_await_suspend_handle; + break; + } + + llvm::Function *AwaitSuspendIntrinsic = CGF.CGM.getIntrinsic(AwaitSuspendIID); + + const auto AwaitSuspendCanThrow = StmtCanThrow(S.getSuspendExpr()); + + llvm::CallBase *SuspendRet = nullptr; + // FIXME: add call attributes? + if (AwaitSuspendCanThrow) + SuspendRet = + CGF.EmitCallOrInvoke(AwaitSuspendIntrinsic, SuspendIntrinsicCallArgs); + else + SuspendRet = CGF.EmitNounwindRuntimeCall(AwaitSuspendIntrinsic, + SuspendIntrinsicCallArgs); + + assert(SuspendRet); CGF.CurCoro.InSuspendBlock = false; - if (SuspendRet != nullptr && SuspendRet->getType()->isIntegerTy(1)) { + switch (SuspendReturnType) { + case CoroutineSuspendExpr::SuspendReturnType::SuspendVoid: + assert(SuspendRet->getType()->isVoidTy()); + break; + case CoroutineSuspendExpr::SuspendReturnType::SuspendBool: { + assert(SuspendRet->getType()->isIntegerTy()); + // Veto suspension if requested by bool returning await_suspend. BasicBlock *RealSuspendBlock = CGF.createBasicBlock(Prefix + Twine(".suspend.bool")); CGF.Builder.CreateCondBr(SuspendRet, RealSuspendBlock, ReadyBlock); CGF.EmitBlock(RealSuspendBlock); + break; + } + case CoroutineSuspendExpr::SuspendReturnType::SuspendHandle: { + assert(SuspendRet->getType()->isPointerTy()); + + auto ResumeIntrinsic = CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_resume); + Builder.CreateCall(ResumeIntrinsic, SuspendRet); + break; + } } // Emit the suspend point. @@ -267,7 +338,7 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co // is marked as 'noexcept', we avoid generating this additional IR. CXXTryStmt *TryStmt = nullptr; if (Coro.ExceptionHandler && Kind == AwaitKind::Init && - ResumeStmtCanThrow(S.getResumeExpr())) { + StmtCanThrow(S.getResumeExpr())) { Coro.ResumeEHVar = CGF.CreateTempAlloca(Builder.getInt1Ty(), Prefix + Twine("resume.eh")); Builder.CreateFlagStore(true, Coro.ResumeEHVar); @@ -338,6 +409,69 @@ static QualType getCoroutineSuspendExprReturnType(const ASTContext &Ctx, } #endif +llvm::Function * +CodeGenFunction::generateAwaitSuspendWrapper(Twine const &CoroName, + Twine const &SuspendPointName, + CoroutineSuspendExpr const &S) { + std::string FuncName = "__await_suspend_wrapper_"; + FuncName += CoroName.str(); + FuncName += '_'; + FuncName += SuspendPointName.str(); + + ASTContext &C = getContext(); + + FunctionArgList args; + + ImplicitParamDecl AwaiterDecl(C, C.VoidPtrTy, ImplicitParamKind::Other); + ImplicitParamDecl FrameDecl(C, C.VoidPtrTy, ImplicitParamKind::Other); + QualType ReturnTy = S.getSuspendExpr()->getType(); + + args.push_back(&AwaiterDecl); + args.push_back(&FrameDecl); + + const CGFunctionInfo &FI = + CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, args); + + llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI); + + llvm::Function *Fn = llvm::Function::Create( + LTy, llvm::GlobalValue::PrivateLinkage, FuncName, &CGM.getModule()); + + Fn->addParamAttr(0, llvm::Attribute::AttrKind::NonNull); + Fn->addParamAttr(0, llvm::Attribute::AttrKind::NoUndef); + + Fn->addParamAttr(1, llvm::Attribute::AttrKind::NoUndef); + + Fn->setMustProgress(); + Fn->addFnAttr(llvm::Attribute::AttrKind::AlwaysInline); + + StartFunction(GlobalDecl(), ReturnTy, Fn, FI, args); + + // FIXME: add TBAA metadata to the loads + llvm::Value *AwaiterPtr = Builder.CreateLoad(GetAddrOfLocalVar(&AwaiterDecl)); + auto AwaiterLValue = + MakeNaturalAlignAddrLValue(AwaiterPtr, AwaiterDecl.getType()); + + CurAwaitSuspendWrapper.FramePtr = + Builder.CreateLoad(GetAddrOfLocalVar(&FrameDecl)); + + auto AwaiterBinder = CodeGenFunction::OpaqueValueMappingData::bind( + *this, S.getOpaqueValue(), AwaiterLValue); + + auto *SuspendRet = EmitScalarExpr(S.getSuspendExpr()); + + auto UnbindCommonOnExit = + llvm::make_scope_exit([&] { AwaiterBinder.unbind(*this); }); + if (SuspendRet != nullptr) { + Fn->addRetAttr(llvm::Attribute::AttrKind::NoUndef); + Builder.CreateStore(SuspendRet, ReturnValue); + } + + CurAwaitSuspendWrapper.FramePtr = nullptr; + FinishFunction(); + return Fn; +} + LValue CodeGenFunction::EmitCoawaitLValue(const CoawaitExpr *E) { assert(getCoroutineSuspendExprReturnType(getContext(), E)->isReferenceType() && @@ -834,6 +968,11 @@ RValue CodeGenFunction::EmitCoroutineIntrinsic(const CallExpr *E, if (CurCoro.Data && CurCoro.Data->CoroBegin) { return RValue::get(CurCoro.Data->CoroBegin); } + + if (CurAwaitSuspendWrapper.FramePtr) { + return RValue::get(CurAwaitSuspendWrapper.FramePtr); + } + CGM.Error(E->getBeginLoc(), "this builtin expect that __builtin_coro_begin " "has been used earlier in this function"); auto *NullPtr = llvm::ConstantPointerNull::get(Builder.getPtrTy()); diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index 06327a1847177..6c825a302913d 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -352,6 +352,25 @@ class CodeGenFunction : public CodeGenTypeCache { return isCoroutine() && CurCoro.InSuspendBlock; } + // Holds FramePtr for await_suspend wrapper generation, + // so that __builtin_coro_frame call can be lowered + // directly to value of its second argument + struct AwaitSuspendWrapperInfo { + llvm::Value *FramePtr = nullptr; + }; + AwaitSuspendWrapperInfo CurAwaitSuspendWrapper; + + // Generates wrapper function for `llvm.coro.await.suspend.*` intrinisics. + // It encapsulates SuspendExpr in a function, to separate it's body + // from the main coroutine to avoid miscompilations. Intrinisic + // is lowered to this function call in CoroSplit pass + // Function signature is: + // __await_suspend_wrapper_(ptr %awaiter, ptr %hdl) + // where type is one of (void, i1, ptr) + llvm::Function *generateAwaitSuspendWrapper(Twine const &CoroName, + Twine const &SuspendPointName, + CoroutineSuspendExpr const &S); + /// CurGD - The GlobalDecl for the current function being compiled. GlobalDecl CurGD; diff --git a/clang/lib/Sema/SemaCoroutine.cpp b/clang/lib/Sema/SemaCoroutine.cpp index 301a5ff72a3b2..5206fc7621c7c 100644 --- a/clang/lib/Sema/SemaCoroutine.cpp +++ b/clang/lib/Sema/SemaCoroutine.cpp @@ -348,99 +348,15 @@ static Expr *maybeTailCall(Sema &S, QualType RetType, Expr *E, Expr *JustAddress = AddressExpr.get(); - // FIXME: Without optimizations, the temporary result from `await_suspend()` - // may be put on the coroutine frame since the coroutine frame constructor - // will think the temporary variable will escape from the - // `coroutine_handle<>::address()` call. This is problematic since the - // coroutine should be considered to be suspended after it enters - // `await_suspend` so it shouldn't access/update the coroutine frame after - // that. - // - // See https://github.com/llvm/llvm-project/issues/65054 for the report. - // - // The long term solution may wrap the whole logic about `await-suspend` - // into a standalone function. This is similar to the proposed solution - // in tryMarkAwaitSuspendNoInline. See the comments there for details. - // - // The short term solution here is to mark `coroutine_handle<>::address()` - // function as always-inline so that the coroutine frame constructor won't - // think the temporary result is escaped incorrectly. - if (auto *FD = cast(JustAddress)->getDirectCallee()) - if (!FD->hasAttr() && !FD->hasAttr()) - FD->addAttr(AlwaysInlineAttr::CreateImplicit(S.getASTContext(), - FD->getLocation())); - // Check that the type of AddressExpr is void* if (!JustAddress->getType().getTypePtr()->isVoidPointerType()) S.Diag(cast(JustAddress)->getCalleeDecl()->getLocation(), diag::warn_coroutine_handle_address_invalid_return_type) << JustAddress->getType(); - // Clean up temporary objects so that they don't live across suspension points - // unnecessarily. We choose to clean up before the call to - // __builtin_coro_resume so that the cleanup code are not inserted in-between - // the resume call and return instruction, which would interfere with the - // musttail call contract. - JustAddress = S.MaybeCreateExprWithCleanups(JustAddress); - return S.BuildBuiltinCallExpr(Loc, Builtin::BI__builtin_coro_resume, - JustAddress); -} - -/// The await_suspend call performed by co_await is essentially asynchronous -/// to the execution of the coroutine. Inlining it normally into an unsplit -/// coroutine can cause miscompilation because the coroutine CFG misrepresents -/// the true control flow of the program: things that happen in the -/// await_suspend are not guaranteed to happen prior to the resumption of the -/// coroutine, and things that happen after the resumption of the coroutine -/// (including its exit and the potential deallocation of the coroutine frame) -/// are not guaranteed to happen only after the end of await_suspend. -/// -/// See https://github.com/llvm/llvm-project/issues/56301 and -/// https://reviews.llvm.org/D157070 for the example and the full discussion. -/// -/// The short-term solution to this problem is to mark the call as uninlinable. -/// But we don't want to do this if the call is known to be trivial, which is -/// very common. -/// -/// The long-term solution may introduce patterns like: -/// -/// call @llvm.coro.await_suspend(ptr %awaiter, ptr %handle, -/// ptr @awaitSuspendFn) -/// -/// Then it is much easier to perform the safety analysis in the middle end. -/// If it is safe to inline the call to awaitSuspend, we can replace it in the -/// CoroEarly pass. Otherwise we could replace it in the CoroSplit pass. -static void tryMarkAwaitSuspendNoInline(Sema &S, OpaqueValueExpr *Awaiter, - CallExpr *AwaitSuspend) { - // The method here to extract the awaiter decl is not precise. - // This is intentional. Since it is hard to perform the analysis in the - // frontend due to the complexity of C++'s type systems. - // And we prefer to perform such analysis in the middle end since it is - // easier to implement and more powerful. - CXXRecordDecl *AwaiterDecl = - Awaiter->getType().getNonReferenceType()->getAsCXXRecordDecl(); - - if (AwaiterDecl && AwaiterDecl->field_empty()) - return; - - FunctionDecl *FD = AwaitSuspend->getDirectCallee(); - - assert(FD); - - // If the `await_suspend()` function is marked as `always_inline` explicitly, - // we should give the user the right to control the codegen. - if (FD->hasAttr() || FD->hasAttr()) - return; - - // This is problematic if the user calls the await_suspend standalone. But on - // the on hand, it is not incorrect semantically since inlining is not part - // of the standard. On the other hand, it is relatively rare to call - // the await_suspend function standalone. - // - // And given we've already had the long-term plan, the current workaround - // looks relatively tolerant. - FD->addAttr( - NoInlineAttr::CreateImplicit(S.getASTContext(), FD->getLocation())); + // Clean up temporary objects, because the resulting expression + // will become the body of await_suspend wrapper. + return S.MaybeCreateExprWithCleanups(JustAddress); } /// Build calls to await_ready, await_suspend, and await_resume for a co_await @@ -514,10 +430,6 @@ static ReadySuspendResumeResult buildCoawaitCalls(Sema &S, VarDecl *CoroPromise, // type Z. QualType RetType = AwaitSuspend->getCallReturnType(S.Context); - // We need to mark await_suspend as noinline temporarily. See the comment - // of tryMarkAwaitSuspendNoInline for details. - tryMarkAwaitSuspendNoInline(S, Operand, AwaitSuspend); - // Support for coroutine_handle returning await_suspend. if (Expr *TailCallSuspend = maybeTailCall(S, RetType, AwaitSuspend, Loc)) diff --git a/clang/test/AST/coroutine-locals-cleanup.cpp b/clang/test/AST/coroutine-locals-cleanup.cpp index ce106b8e230a1..6264df01fa2ac 100644 --- a/clang/test/AST/coroutine-locals-cleanup.cpp +++ b/clang/test/AST/coroutine-locals-cleanup.cpp @@ -90,10 +90,7 @@ Task bar() { // CHECK: ExprWithCleanups {{.*}} 'bool' // CHECK-NEXT: CXXMemberCallExpr {{.*}} 'bool' // CHECK-NEXT: MemberExpr {{.*}} .await_ready -// CHECK: CallExpr {{.*}} 'void' -// CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(void *)' -// CHECK-NEXT: DeclRefExpr {{.*}} '__builtin_coro_resume' 'void (void *)' -// CHECK-NEXT: ExprWithCleanups {{.*}} 'void *' +// CHECK: ExprWithCleanups {{.*}} 'void *' // CHECK: CaseStmt // CHECK: ExprWithCleanups {{.*}} 'void' @@ -103,7 +100,4 @@ Task bar() { // CHECK: ExprWithCleanups {{.*}} 'bool' // CHECK-NEXT: CXXMemberCallExpr {{.*}} 'bool' // CHECK-NEXT: MemberExpr {{.*}} .await_ready -// CHECK: CallExpr {{.*}} 'void' -// CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(void *)' -// CHECK-NEXT: DeclRefExpr {{.*}} '__builtin_coro_resume' 'void (void *)' -// CHECK-NEXT: ExprWithCleanups {{.*}} 'void *' +// CHECK: ExprWithCleanups {{.*}} 'void *' diff --git a/clang/test/CodeGenCoroutines/coro-always-inline.cpp b/clang/test/CodeGenCoroutines/coro-always-inline.cpp index 6e13a62fbd986..d4f67a73f5172 100644 --- a/clang/test/CodeGenCoroutines/coro-always-inline.cpp +++ b/clang/test/CodeGenCoroutines/coro-always-inline.cpp @@ -34,7 +34,7 @@ struct coroutine_traits { // CHECK-LABEL: @_Z3foov // CHECK-LABEL: entry: // CHECK: %ref.tmp.reload.addr = getelementptr -// CHECK: %ref.tmp4.reload.addr = getelementptr +// CHECK: %ref.tmp3.reload.addr = getelementptr void foo() { co_return; } // Check that bar is not inlined even it's marked as always_inline. diff --git a/clang/test/CodeGenCoroutines/coro-await.cpp b/clang/test/CodeGenCoroutines/coro-await.cpp index dc5a765ccb83d..75851d8805bb6 100644 --- a/clang/test/CodeGenCoroutines/coro-await.cpp +++ b/clang/test/CodeGenCoroutines/coro-await.cpp @@ -71,16 +71,13 @@ extern "C" void f0() { // CHECK: [[SUSPEND_BB]]: // CHECK: %[[SUSPEND_ID:.+]] = call token @llvm.coro.save( // --------------------------- - // Build the coroutine handle and pass it to await_suspend + // Call coro.await.suspend // --------------------------- - // CHECK: call ptr @_ZNSt16coroutine_handleINSt16coroutine_traitsIJvEE12promise_typeEE12from_addressEPv(ptr %[[FRAME]]) - // ... many lines of code to coerce coroutine_handle into an ptr scalar - // CHECK: %[[CH:.+]] = load ptr, ptr %{{.+}} - // CHECK: call void @_ZN14suspend_always13await_suspendESt16coroutine_handleIvE(ptr {{[^,]*}} %[[AWAITABLE]], ptr %[[CH]]) + // CHECK-NEXT: call void @llvm.coro.await.suspend.void(ptr %[[AWAITABLE]], ptr %[[FRAME]], ptr @__await_suspend_wrapper_f0_await) // ------------------------- // Generate a suspend point: // ------------------------- - // CHECK: %[[OUTCOME:.+]] = call i8 @llvm.coro.suspend(token %[[SUSPEND_ID]], i1 false) + // CHECK-NEXT: %[[OUTCOME:.+]] = call i8 @llvm.coro.suspend(token %[[SUSPEND_ID]], i1 false) // CHECK: switch i8 %[[OUTCOME]], label %[[RET_BB:.+]] [ // CHECK: i8 0, label %[[READY_BB]] // CHECK: i8 1, label %[[CLEANUP_BB:.+]] @@ -101,6 +98,17 @@ extern "C" void f0() { // CHECK-NEXT: call zeroext i1 @_ZN10final_susp11await_readyEv(ptr // CHECK: %[[FINALSP_ID:.+]] = call token @llvm.coro.save( // CHECK: call i8 @llvm.coro.suspend(token %[[FINALSP_ID]], i1 true) + + // Await suspend wrapper + // CHECK: define{{.*}} @__await_suspend_wrapper_f0_await(ptr {{[^,]*}} %[[AWAITABLE_ARG:.+]], ptr {{[^,]*}} %[[FRAME_ARG:.+]]) + // CHECK: store ptr %[[AWAITABLE_ARG]], ptr %[[AWAITABLE_TMP:.+]], + // CHECK: store ptr %[[FRAME_ARG]], ptr %[[FRAME_TMP:.+]], + // CHECK: %[[AWAITABLE:.+]] = load ptr, ptr %[[AWAITABLE_TMP]] + // CHECK: %[[FRAME:.+]] = load ptr, ptr %[[FRAME_TMP]] + // CHECK: call ptr @_ZNSt16coroutine_handleINSt16coroutine_traitsIJvEE12promise_typeEE12from_addressEPv(ptr %[[FRAME]]) + // ... many lines of code to coerce coroutine_handle into an ptr scalar + // CHECK: %[[CH:.+]] = load ptr, ptr %{{.+}} + // CHECK: call void @_ZN14suspend_always13await_suspendESt16coroutine_handleIvE(ptr {{[^,]*}} %[[AWAITABLE]], ptr %[[CH]]) } struct suspend_maybe { @@ -131,7 +139,7 @@ extern "C" void f1(int) { // See if we need to suspend: // -------------------------- - // CHECK: %[[READY:.+]] = call zeroext i1 @_ZN13suspend_maybe11await_readyEv(ptr {{[^,]*}} %[[AWAITABLE]]) + // CHECK: %[[READY:.+]] = call zeroext i1 @_ZN13suspend_maybe11await_readyEv(ptr {{[^,]*}} %[[AWAITABLE:.+]]) // CHECK: br i1 %[[READY]], label %[[READY_BB:.+]], label %[[SUSPEND_BB:.+]] // If we are suspending: @@ -139,12 +147,9 @@ extern "C" void f1(int) { // CHECK: [[SUSPEND_BB]]: // CHECK: %[[SUSPEND_ID:.+]] = call token @llvm.coro.save( // --------------------------- - // Build the coroutine handle and pass it to await_suspend + // Call coro.await.suspend // --------------------------- - // CHECK: call ptr @_ZNSt16coroutine_handleINSt16coroutine_traitsIJviEE12promise_typeEE12from_addressEPv(ptr %[[FRAME]]) - // ... many lines of code to coerce coroutine_handle into an ptr scalar - // CHECK: %[[CH:.+]] = load ptr, ptr %{{.+}} - // CHECK: %[[YES:.+]] = call zeroext i1 @_ZN13suspend_maybe13await_suspendESt16coroutine_handleIvE(ptr {{[^,]*}} %[[AWAITABLE]], ptr %[[CH]]) + // CHECK-NEXT: %[[YES:.+]] = call i1 @llvm.coro.await.suspend.bool(ptr %[[AWAITABLE]], ptr %[[FRAME]], ptr @__await_suspend_wrapper_f1_yield) // ------------------------------------------- // See if await_suspend decided not to suspend // ------------------------------------------- @@ -155,6 +160,18 @@ extern "C" void f1(int) { // CHECK: [[READY_BB]]: // CHECK: call void @_ZN13suspend_maybe12await_resumeEv(ptr {{[^,]*}} %[[AWAITABLE]]) + + // Await suspend wrapper + // CHECK: define {{.*}} i1 @__await_suspend_wrapper_f1_yield(ptr {{[^,]*}} %[[AWAITABLE_ARG:.+]], ptr {{[^,]*}} %[[FRAME_ARG:.+]]) + // CHECK: store ptr %[[AWAITABLE_ARG]], ptr %[[AWAITABLE_TMP:.+]], + // CHECK: store ptr %[[FRAME_ARG]], ptr %[[FRAME_TMP:.+]], + // CHECK: %[[AWAITABLE:.+]] = load ptr, ptr %[[AWAITABLE_TMP]] + // CHECK: %[[FRAME:.+]] = load ptr, ptr %[[FRAME_TMP]] + // CHECK: call ptr @_ZNSt16coroutine_handleINSt16coroutine_traitsIJviEE12promise_typeEE12from_addressEPv(ptr %[[FRAME]]) + // ... many lines of code to coerce coroutine_handle into an ptr scalar + // CHECK: %[[CH:.+]] = load ptr, ptr %{{.+}} + // CHECK: %[[YES:.+]] = call zeroext i1 @_ZN13suspend_maybe13await_suspendESt16coroutine_handleIvE(ptr {{[^,]*}} %[[AWAITABLE]], ptr %[[CH]]) + // CHECK-NEXT: ret i1 %[[YES]] } struct ComplexAwaiter { @@ -340,11 +357,39 @@ struct TailCallAwait { // CHECK-LABEL: @TestTailcall( extern "C" void TestTailcall() { + // CHECK: %[[PROMISE:.+]] = alloca %"struct.std::coroutine_traits::promise_type" + // CHECK: %[[FRAME:.+]] = call ptr @llvm.coro.begin( co_await TailCallAwait{}; + // CHECK: %[[READY:.+]] = call zeroext i1 @_ZN13TailCallAwait11await_readyEv(ptr {{[^,]*}} %[[AWAITABLE:.+]]) + // CHECK: br i1 %[[READY]], label %[[READY_BB:.+]], label %[[SUSPEND_BB:.+]] - // CHECK: %[[RESULT:.+]] = call ptr @_ZN13TailCallAwait13await_suspendESt16coroutine_handleIvE(ptr - // CHECK: %[[COERCE:.+]] = getelementptr inbounds %"struct.std::coroutine_handle", ptr %[[TMP:.+]], i32 0, i32 0 - // CHECK: store ptr %[[RESULT]], ptr %[[COERCE]] - // CHECK: %[[ADDR:.+]] = call ptr @_ZNSt16coroutine_handleIvE7addressEv(ptr {{[^,]*}} %[[TMP]]) - // CHECK: call void @llvm.coro.resume(ptr %[[ADDR]]) + // If we are suspending: + // --------------------- + // CHECK: [[SUSPEND_BB]]: + // CHECK: %[[SUSPEND_ID:.+]] = call token @llvm.coro.save( + // --------------------------- + // Call coro.await.suspend + // --------------------------- + // CHECK-NEXT: %[[RESUMED:.+]] = call ptr @llvm.coro.await.suspend.handle(ptr %[[AWAITABLE]], ptr %[[FRAME]], ptr @__await_suspend_wrapper_TestTailcall_await) + // CHECK-NEXT: call void @llvm.coro.resume(ptr %[[RESUMED]]) + // CHECK-NEXT: %[[OUTCOME:.+]] = call i8 @llvm.coro.suspend(token %[[SUSPEND_ID]], i1 false) + // CHECK-NEXT: switch i8 %[[OUTCOME]], label %[[RET_BB:.+]] [ + // CHECK-NEXT: i8 0, label %[[READY_BB]] + // CHECK-NEXT: i8 1, label %{{.+}} + // CHECK-NEXT: ] + + // Await suspend wrapper + // CHECK: define {{.*}} ptr @__await_suspend_wrapper_TestTailcall_await(ptr {{[^,]*}} %[[AWAITABLE_ARG:.+]], ptr {{[^,]*}} %[[FRAME_ARG:.+]]) + // CHECK: store ptr %[[AWAITABLE_ARG]], ptr %[[AWAITABLE_TMP:.+]], + // CHECK: store ptr %[[FRAME_ARG]], ptr %[[FRAME_TMP:.+]], + // CHECK: %[[AWAITABLE:.+]] = load ptr, ptr %[[AWAITABLE_TMP]] + // CHECK: %[[FRAME:.+]] = load ptr, ptr %[[FRAME_TMP]] + // CHECK: call ptr @_ZNSt16coroutine_handleINSt16coroutine_traitsIJvEE12promise_typeEE12from_addressEPv(ptr %[[FRAME]]) + // ... many lines of code to coerce coroutine_handle into an ptr scalar + // CHECK: %[[CH:.+]] = load ptr, ptr %{{.+}} + // CHECK-NEXT: %[[RESULT:.+]] = call ptr @_ZN13TailCallAwait13await_suspendESt16coroutine_handleIvE(ptr {{[^,]*}} %[[AWAITABLE]], ptr %[[CH]]) + // CHECK-NEXT: %[[COERCE:.+]] = getelementptr inbounds %"struct.std::coroutine_handle", ptr %[[TMP:.+]], i32 0, i32 0 + // CHECK-NEXT: store ptr %[[RESULT]], ptr %[[COERCE]] + // CHECK-NEXT: %[[ADDR:.+]] = call ptr @_ZNSt16coroutine_handleIvE7addressEv(ptr {{[^,]*}} %[[TMP]]) + // CHECK-NEXT: ret ptr %[[ADDR]] } diff --git a/clang/test/CodeGenCoroutines/coro-awaiter-noinline-suspend.cpp b/clang/test/CodeGenCoroutines/coro-awaiter-noinline-suspend.cpp deleted file mode 100644 index f95286faf46ec..0000000000000 --- a/clang/test/CodeGenCoroutines/coro-awaiter-noinline-suspend.cpp +++ /dev/null @@ -1,168 +0,0 @@ -// Tests that we can mark await-suspend as noinline correctly. -// -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -emit-llvm -o - %s \ -// RUN: -O1 -disable-llvm-passes | FileCheck %s - -#include "Inputs/coroutine.h" - -struct Task { - struct promise_type { - struct FinalAwaiter { - bool await_ready() const noexcept { return false; } - template - std::coroutine_handle<> await_suspend(std::coroutine_handle h) noexcept { - return h.promise().continuation; - } - void await_resume() noexcept {} - }; - - Task get_return_object() noexcept { - return std::coroutine_handle::from_promise(*this); - } - - std::suspend_always initial_suspend() noexcept { return {}; } - FinalAwaiter final_suspend() noexcept { return {}; } - void unhandled_exception() noexcept {} - void return_void() noexcept {} - - std::coroutine_handle<> continuation; - }; - - Task(std::coroutine_handle handle); - ~Task(); - -private: - std::coroutine_handle handle; -}; - -struct StatefulAwaiter { - int value; - bool await_ready() const noexcept { return false; } - template - void await_suspend(std::coroutine_handle h) noexcept {} - void await_resume() noexcept {} -}; - -typedef std::suspend_always NoStateAwaiter; -using AnotherStatefulAwaiter = StatefulAwaiter; - -template -struct TemplatedAwaiter { - T value; - bool await_ready() const noexcept { return false; } - template - void await_suspend(std::coroutine_handle h) noexcept {} - void await_resume() noexcept {} -}; - - -class Awaitable {}; -StatefulAwaiter operator co_await(Awaitable) { - return StatefulAwaiter{}; -} - -StatefulAwaiter GlobalAwaiter; -class Awaitable2 {}; -StatefulAwaiter& operator co_await(Awaitable2) { - return GlobalAwaiter; -} - -struct AlwaysInlineStatefulAwaiter { - void* value; - bool await_ready() const noexcept { return false; } - - template - __attribute__((always_inline)) - void await_suspend(std::coroutine_handle h) noexcept {} - - void await_resume() noexcept {} -}; - -Task testing() { - co_await std::suspend_always{}; - co_await StatefulAwaiter{}; - co_await AnotherStatefulAwaiter{}; - - // Test lvalue case. - StatefulAwaiter awaiter; - co_await awaiter; - - // The explicit call to await_suspend is not considered suspended. - awaiter.await_suspend(std::coroutine_handle::from_address(nullptr)); - - co_await TemplatedAwaiter{}; - TemplatedAwaiter TemplatedAwaiterInstace; - co_await TemplatedAwaiterInstace; - - co_await Awaitable{}; - co_await Awaitable2{}; - - co_await AlwaysInlineStatefulAwaiter{}; -} - -struct AwaitTransformTask { - struct promise_type { - struct FinalAwaiter { - bool await_ready() const noexcept { return false; } - template - std::coroutine_handle<> await_suspend(std::coroutine_handle h) noexcept { - return h.promise().continuation; - } - void await_resume() noexcept {} - }; - - AwaitTransformTask get_return_object() noexcept { - return std::coroutine_handle::from_promise(*this); - } - - std::suspend_always initial_suspend() noexcept { return {}; } - FinalAwaiter final_suspend() noexcept { return {}; } - void unhandled_exception() noexcept {} - void return_void() noexcept {} - - template - auto await_transform(Awaitable &&awaitable) { - return awaitable; - } - - std::coroutine_handle<> continuation; - }; - - AwaitTransformTask(std::coroutine_handle handle); - ~AwaitTransformTask(); - -private: - std::coroutine_handle handle; -}; - -struct awaitableWithGetAwaiter { - bool await_ready() const noexcept { return false; } - template - void await_suspend(std::coroutine_handle h) noexcept {} - void await_resume() noexcept {} -}; - -AwaitTransformTask testingWithAwaitTransform() { - co_await awaitableWithGetAwaiter{}; -} - -// CHECK: define{{.*}}@_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE{{.*}}#[[NORMAL_ATTR:[0-9]+]] - -// CHECK: define{{.*}}@_ZN15StatefulAwaiter13await_suspendIN4Task12promise_typeEEEvSt16coroutine_handleIT_E{{.*}}#[[NOINLINE_ATTR:[0-9]+]] - -// CHECK: define{{.*}}@_ZN15StatefulAwaiter13await_suspendIvEEvSt16coroutine_handleIT_E{{.*}}#[[NORMAL_ATTR]] - -// CHECK: define{{.*}}@_ZN16TemplatedAwaiterIiE13await_suspendIN4Task12promise_typeEEEvSt16coroutine_handleIT_E{{.*}}#[[NOINLINE_ATTR]] - -// CHECK: define{{.*}}@_ZN27AlwaysInlineStatefulAwaiter13await_suspendIN4Task12promise_typeEEEvSt16coroutine_handleIT_E{{.*}}#[[ALWAYS_INLINE_ATTR:[0-9]+]] - -// CHECK: define{{.*}}@_ZN4Task12promise_type12FinalAwaiter13await_suspendIS0_EESt16coroutine_handleIvES3_IT_E{{.*}}#[[NORMAL_ATTR]] - -// CHECK: define{{.*}}@_ZN23awaitableWithGetAwaiter13await_suspendIN18AwaitTransformTask12promise_typeEEEvSt16coroutine_handleIT_E{{.*}}#[[NORMAL_ATTR]] - -// CHECK: define{{.*}}@_ZN18AwaitTransformTask12promise_type12FinalAwaiter13await_suspendIS0_EESt16coroutine_handleIvES3_IT_E{{.*}}#[[NORMAL_ATTR]] - -// CHECK-NOT: attributes #[[NORMAL_ATTR]] = noinline -// CHECK: attributes #[[NOINLINE_ATTR]] = {{.*}}noinline -// CHECK-NOT: attributes #[[ALWAYS_INLINE_ATTR]] = {{.*}}noinline -// CHECK: attributes #[[ALWAYS_INLINE_ATTR]] = {{.*}}alwaysinline diff --git a/clang/test/CodeGenCoroutines/coro-dwarf.cpp b/clang/test/CodeGenCoroutines/coro-dwarf.cpp index 7914babe5483a..2c9c827e6753d 100644 --- a/clang/test/CodeGenCoroutines/coro-dwarf.cpp +++ b/clang/test/CodeGenCoroutines/coro-dwarf.cpp @@ -70,3 +70,15 @@ void f_coro(int val, MoveOnly moParam, MoveAndCopy mcParam) { // CHECK: !{{[0-9]+}} = !DILocalVariable(name: "moParam", arg: 2, scope: ![[SP]], file: !{{[0-9]+}}, line: {{[0-9]+}}, type: !{{[0-9]+}}) // CHECK: !{{[0-9]+}} = !DILocalVariable(name: "mcParam", arg: 3, scope: ![[SP]], file: !{{[0-9]+}}, line: {{[0-9]+}}, type: !{{[0-9]+}}) // CHECK: !{{[0-9]+}} = !DILocalVariable(name: "__promise", + +// CHECK: !{{[0-9]+}} = distinct !DISubprogram(linkageName: "__await_suspend_wrapper__Z6f_coroi8MoveOnly11MoveAndCopy_init" +// CHECK-NEXT: !{{[0-9]+}} = !DIFile +// CHECK-NEXT: !{{[0-9]+}} = !DISubroutineType +// CHECK-NEXT: !{{[0-9]+}} = !DILocalVariable(arg: 1, +// CHECK-NEXT: !{{[0-9]+}} = !DILocation +// CHECK-NEXT: !{{[0-9]+}} = !DILocalVariable(arg: 2, + +// CHECK: !{{[0-9]+}} = distinct !DISubprogram(linkageName: "__await_suspend_wrapper__Z6f_coroi8MoveOnly11MoveAndCopy_final" +// CHECK-NEXT: !{{[0-9]+}} = !DILocalVariable(arg: 1, +// CHECK-NEXT: !{{[0-9]+}} = !DILocation +// CHECK-NEXT: !{{[0-9]+}} = !DILocalVariable(arg: 2, diff --git a/clang/test/CodeGenCoroutines/coro-function-try-block.cpp b/clang/test/CodeGenCoroutines/coro-function-try-block.cpp index f609eb55b8e77..b7a796cc241af 100644 --- a/clang/test/CodeGenCoroutines/coro-function-try-block.cpp +++ b/clang/test/CodeGenCoroutines/coro-function-try-block.cpp @@ -19,5 +19,5 @@ task f() try { } // CHECK-LABEL: define{{.*}} void @_Z1fv( -// CHECK: call void @_ZNSt13suspend_never13await_suspendESt16coroutine_handleIvE( +// CHECK: call void @llvm.coro.await.suspend.void( // CHECK: call void @_ZN4task12promise_type11return_voidEv( diff --git a/clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp b/clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp index c0b9e9ee2c558..da30e12c63cff 100644 --- a/clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp +++ b/clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp @@ -50,10 +50,5 @@ detached_task foo() { // check that the lifetime of the coroutine handle used to obtain the address is contained within single basic block, and hence does not live across suspension points. // CHECK-LABEL: final.suspend: // CHECK: %{{.+}} = call token @llvm.coro.save(ptr null) -// CHECK: call void @llvm.lifetime.start.p0(i64 8, ptr %[[HDL:.+]]) -// CHECK: %[[CALL:.+]] = call ptr @_ZN13detached_task12promise_type13final_awaiter13await_suspendESt16coroutine_handleIS0_E( -// CHECK: %[[HDL_CAST2:.+]] = getelementptr inbounds %"struct.std::coroutine_handle.0", ptr %[[HDL]], i32 0, i32 0 -// CHECK: store ptr %[[CALL]], ptr %[[HDL_CAST2]], align 8 -// CHECK: %[[HDL_TRANSFER:.+]] = call noundef ptr @_ZNKSt16coroutine_handleIvE7addressEv(ptr noundef {{.*}}%[[HDL]]) -// CHECK: call void @llvm.lifetime.end.p0(i64 8, ptr %[[HDL]]) +// CHECK: %[[HDL_TRANSFER:.+]] = call ptr @llvm.coro.await.suspend.handle // CHECK: call void @llvm.coro.resume(ptr %[[HDL_TRANSFER]]) diff --git a/clang/test/CodeGenCoroutines/coro-symmetric-transfer-02.cpp b/clang/test/CodeGenCoroutines/coro-symmetric-transfer-02.cpp index 890d55e41de95..ca6cf74115a3b 100644 --- a/clang/test/CodeGenCoroutines/coro-symmetric-transfer-02.cpp +++ b/clang/test/CodeGenCoroutines/coro-symmetric-transfer-02.cpp @@ -89,10 +89,8 @@ Task bar() { // CHECK: br i1 %{{.+}}, label %[[CASE1_AWAIT_READY:.+]], label %[[CASE1_AWAIT_SUSPEND:.+]] // CHECK: [[CASE1_AWAIT_SUSPEND]]: // CHECK-NEXT: %{{.+}} = call token @llvm.coro.save(ptr null) -// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr %[[TMP1:.+]]) - -// CHECK: call void @llvm.lifetime.end.p0(i64 8, ptr %[[TMP1]]) -// CHECK-NEXT: call void @llvm.coro.resume +// CHECK-NEXT: %[[HANDLE1_PTR:.+]] = call ptr @llvm.coro.await.suspend.handle +// CHECK-NEXT: call void @llvm.coro.resume(ptr %[[HANDLE1_PTR]]) // CHECK-NEXT: %{{.+}} = call i8 @llvm.coro.suspend // CHECK-NEXT: switch i8 %{{.+}}, label %coro.ret [ // CHECK-NEXT: i8 0, label %[[CASE1_AWAIT_READY]] @@ -106,10 +104,8 @@ Task bar() { // CHECK: br i1 %{{.+}}, label %[[CASE2_AWAIT_READY:.+]], label %[[CASE2_AWAIT_SUSPEND:.+]] // CHECK: [[CASE2_AWAIT_SUSPEND]]: // CHECK-NEXT: %{{.+}} = call token @llvm.coro.save(ptr null) -// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr %[[TMP2:.+]]) - -// CHECK: call void @llvm.lifetime.end.p0(i64 8, ptr %[[TMP2]]) -// CHECK-NEXT: call void @llvm.coro.resume +// CHECK-NEXT: %[[HANDLE2_PTR:.+]] = call ptr @llvm.coro.await.suspend.handle +// CHECK-NEXT: call void @llvm.coro.resume(ptr %[[HANDLE2_PTR]]) // CHECK-NEXT: %{{.+}} = call i8 @llvm.coro.suspend // CHECK-NEXT: switch i8 %{{.+}}, label %coro.ret [ // CHECK-NEXT: i8 0, label %[[CASE2_AWAIT_READY]] diff --git a/clang/test/CodeGenCoroutines/pr56329.cpp b/clang/test/CodeGenCoroutines/pr56329.cpp index 31d4849af4e71..ad8b1b990179c 100644 --- a/clang/test/CodeGenCoroutines/pr56329.cpp +++ b/clang/test/CodeGenCoroutines/pr56329.cpp @@ -115,5 +115,9 @@ Task Outer() { // CHECK-NOT: _exit // CHECK: musttail call // CHECK: musttail call +// CHECK: musttail call // CHECK-NEXT: ret void +// CHECK-EMPTY: +// CHECK-NEXT: unreachable: +// CHECK-NEXT: unreachable // CHECK-NEXT: } diff --git a/clang/test/CodeGenCoroutines/pr59181.cpp b/clang/test/CodeGenCoroutines/pr59181.cpp index 80f4634db2521..21e784e0031de 100644 --- a/clang/test/CodeGenCoroutines/pr59181.cpp +++ b/clang/test/CodeGenCoroutines/pr59181.cpp @@ -52,9 +52,8 @@ void foo() { // CHECK-NEXT: load i8 // CHECK-NEXT: trunc // CHECK-NEXT: store i1 false -// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[REF:%ref.tmp[0-9]+]]) // CHECK: await.suspend:{{.*}} -// CHECK-NOT: call void @llvm.lifetime.start.p0(i64 8, ptr [[REF]]) -// CHECK: call void @_ZZN4Task12promise_type15await_transformES_EN10Suspension13await_suspendESt16coroutine_handleIvE -// CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[REF]]) +// CHECK-NOT: call void @llvm.lifetime +// CHECK: call void @llvm.coro.await.suspend.void( +// CHECK-NEXT: %{{[0-9]+}} = call i8 @llvm.coro.suspend( diff --git a/clang/test/CodeGenCoroutines/pr65054.cpp b/clang/test/CodeGenCoroutines/pr65054.cpp index 834b71050f59f..7af9c04fca180 100644 --- a/clang/test/CodeGenCoroutines/pr65054.cpp +++ b/clang/test/CodeGenCoroutines/pr65054.cpp @@ -1,7 +1,3 @@ -// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++20 \ -// RUN: -O0 -disable-llvm-passes -emit-llvm %s -o - \ -// RUN: | FileCheck %s --check-prefix=FRONTEND - // The output of O0 is highly redundant and hard to test. Also it is not good // limit the output of O0. So we test the optimized output from O0. The idea // is the optimizations shouldn't change the semantics of the program. @@ -51,10 +47,7 @@ MyTask FooBar() { } } -// FRONTEND: define{{.*}}@_ZNKSt16coroutine_handleIvE7addressEv{{.*}}#[[address_attr:[0-9]+]] -// FRONTEND: attributes #[[address_attr]] = {{.*}}alwaysinline - // CHECK-O0: define{{.*}}@_Z6FooBarv.resume -// CHECK-O0: call{{.*}}@_ZN7Awaiter13await_suspendESt16coroutine_handleIvE +// CHECK-O0: call{{.*}}@__await_suspend_wrapper__Z6FooBarv_await( // CHECK-O0-NOT: store // CHECK-O0: ret void diff --git a/llvm/docs/Coroutines.rst b/llvm/docs/Coroutines.rst index d6219d264e4a9..83369d93c309a 100644 --- a/llvm/docs/Coroutines.rst +++ b/llvm/docs/Coroutines.rst @@ -1744,6 +1744,266 @@ a call to ``llvm.coro.suspend.retcon`` after resuming abnormally. In a yield-once coroutine, it is undefined behavior if the coroutine executes a call to ``llvm.coro.suspend.retcon`` after resuming in any way. +.. _coro.await.suspend.void: + +'llvm.coro.await.suspend.void' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + declare void @llvm.coro.await.suspend.void( + ptr , + ptr , + ptr ) + +Overview: +""""""""" + +The '``llvm.coro.await.suspend.void``' intrinsic encapsulates C++ +`await-suspend` block until it can't interfere with coroutine transform. + +The `await_suspend` block of `co_await` is essentially asynchronous +to the execution of the coroutine. Inlining it normally into an unsplit +coroutine can cause miscompilation because the coroutine CFG misrepresents +the true control flow of the program: things that happen in the +await_suspend are not guaranteed to happen prior to the resumption of the +coroutine, and things that happen after the resumption of the coroutine +(including its exit and the potential deallocation of the coroutine frame) +are not guaranteed to happen only after the end of `await_suspend`. + +This version of intrinsic corresponds to +'``void awaiter.await_suspend(...)``' variant. + +Arguments: +"""""""""" + +The first argument is a pointer to `awaiter` object. + +The second argument is a pointer to the current coroutine's frame. + +The third argument is a pointer to the wrapper function encapsulating +`await-suspend` logic. Its signature must be + +.. code-block:: llvm + + declare void @await_suspend_function(ptr %awaiter, ptr %hdl) + +Semantics: +"""""""""" + +The intrinsic must be used between corresponding `coro.save`_ and +`coro.suspend`_ calls. It is lowered to a direct +`await_suspend_function` call during `CoroSplit`_ pass. + +Example: +"""""""" + +.. code-block:: llvm + + ; before lowering + await.suspend: + %save = call token @llvm.coro.save(ptr %hdl) + call void @llvm.coro.await.suspend.void( + ptr %awaiter, + ptr %hdl, + ptr @await_suspend_function) + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + ... + + ; after lowering + await.suspend: + %save = call token @llvm.coro.save(ptr %hdl) + ; the call to await_suspend_function can be inlined + call void @await_suspend_function( + ptr %awaiter, + ptr %hdl) + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + ... + + ; wrapper function example + define void @await_suspend_function(ptr %awaiter, ptr %hdl) + entry: + %hdl.arg = ... ; construct std::coroutine_handle from %hdl + call void @"Awaiter::await_suspend"(ptr %awaiter, ptr %hdl.arg) + ret void + +.. _coro.await.suspend.bool: + +'llvm.coro.await.suspend.bool' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + declare i1 @llvm.coro.await.suspend.bool( + ptr , + ptr , + ptr ) + +Overview: +""""""""" + +The '``llvm.coro.await.suspend.bool``' intrinsic encapsulates C++ +`await-suspend` block until it can't interfere with coroutine transform. + +The `await_suspend` block of `co_await` is essentially asynchronous +to the execution of the coroutine. Inlining it normally into an unsplit +coroutine can cause miscompilation because the coroutine CFG misrepresents +the true control flow of the program: things that happen in the +await_suspend are not guaranteed to happen prior to the resumption of the +coroutine, and things that happen after the resumption of the coroutine +(including its exit and the potential deallocation of the coroutine frame) +are not guaranteed to happen only after the end of `await_suspend`. + +This version of intrinsic corresponds to +'``bool awaiter.await_suspend(...)``' variant. + +Arguments: +"""""""""" + +The first argument is a pointer to `awaiter` object. + +The second argument is a pointer to the current coroutine's frame. + +The third argument is a pointer to the wrapper function encapsulating +`await-suspend` logic. Its signature must be + +.. code-block:: llvm + + declare i1 @await_suspend_function(ptr %awaiter, ptr %hdl) + +Semantics: +"""""""""" + +The intrinsic must be used between corresponding `coro.save`_ and +`coro.suspend`_ calls. It is lowered to a direct +`await_suspend_function` call during `CoroSplit`_ pass. + +If `await_suspend_function` call returns `true`, the current coroutine is +immediately resumed. + +Example: +"""""""" + +.. code-block:: llvm + + ; before lowering + await.suspend: + %save = call token @llvm.coro.save(ptr %hdl) + %resume = call i1 @llvm.coro.await.suspend.bool( + ptr %awaiter, + ptr %hdl, + ptr @await_suspend_function) + br i1 %resume, %await.suspend.bool, %await.ready + await.suspend.bool: + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + ... + await.ready: + call void @"Awaiter::await_resume"(ptr %awaiter) + ... + + ; after lowering + await.suspend: + %save = call token @llvm.coro.save(ptr %hdl) + ; the call to await_suspend_function can inlined + %resume = call i1 @await_suspend_function( + ptr %awaiter, + ptr %hdl) + br i1 %resume, %await.suspend.bool, %await.ready + ... + + ; wrapper function example + define i1 @await_suspend_function(ptr %awaiter, ptr %hdl) + entry: + %hdl.arg = ... ; construct std::coroutine_handle from %hdl + %resume = call i1 @"Awaiter::await_suspend"(ptr %awaiter, ptr %hdl.arg) + ret i1 %resume + +.. _coro.await.suspend.handle: + +'llvm.coro.await.suspend.handle' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + declare ptr @llvm.coro.await.suspend.handle( + ptr , + ptr , + ptr ) + +Overview: +""""""""" + +The '``llvm.coro.await.suspend.handle``' intrinsic encapsulates C++ +`await-suspend` block until it can't interfere with coroutine transform. + +The `await_suspend` block of `co_await` is essentially asynchronous +to the execution of the coroutine. Inlining it normally into an unsplit +coroutine can cause miscompilation because the coroutine CFG misrepresents +the true control flow of the program: things that happen in the +await_suspend are not guaranteed to happen prior to the resumption of the +coroutine, and things that happen after the resumption of the coroutine +(including its exit and the potential deallocation of the coroutine frame) +are not guaranteed to happen only after the end of `await_suspend`. + +This version of intrinsic corresponds to +'``std::corouine_handle<> awaiter.await_suspend(...)``' variant. + +Arguments: +"""""""""" + +The first argument is a pointer to `awaiter` object. + +The second argument is a pointer to the current coroutine's frame. + +The third argument is a pointer to the wrapper function encapsulating +`await-suspend` logic. Its signature must be + +.. code-block:: llvm + + declare ptr @await_suspend_function(ptr %awaiter, ptr %hdl) + +Semantics: +"""""""""" + +The intrinsic must be used between corresponding `coro.save`_ and +`coro.suspend`_ calls. It is lowered to a direct +`await_suspend_function` call during `CoroSplit`_ pass. + +`await_suspend_function` must return a pointer to a valid +coroutine frame, which is immediately resumed + +Example: +"""""""" + +.. code-block:: llvm + + ; before lowering + await.suspend: + %save = call token @llvm.coro.save(ptr %hdl) + %next = call ptr @llvm.coro.await.suspend.handle( + ptr %awaiter, + ptr %hdl, + ptr @await_suspend_function) + call void @llvm.coro.resume(%next) + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + ... + + ; after lowering + await.suspend: + %save = call token @llvm.coro.save(ptr %hdl) + ; the call to await_suspend_function can be inlined + %next = call ptr @await_suspend_function( + ptr %awaiter, + ptr %hdl) + call void @llvm.coro.resume(%next) + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + ... + + ; wrapper function example + define ptr @await_suspend_function(ptr %awaiter, ptr %hdl) + entry: + %hdl.arg = ... ; construct std::coroutine_handle from %hdl + %hdl.raw = call ptr @"Awaiter::await_suspend"(ptr %awaiter, ptr %hdl.arg) + %hdl.result = ... ; get address of returned coroutine handle + ret ptr %hdl.result + Coroutine Transformation Passes =============================== CoroEarly @@ -1758,7 +2018,9 @@ and `coro.promise`_ intrinsics. CoroSplit --------- The pass CoroSplit builds coroutine frame and outlines resume and destroy parts -into separate functions. +into separate functions. This pass also lowers `coro.await.suspend.void`_, +`coro.await.suspend.bool`_ and `coro.await.suspend.handle`_ intrinsics. + CoroElide --------- diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index c2c0f74c315ba..144298fd7c016 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1692,6 +1692,18 @@ def int_coro_promise : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i32_ty, llvm_i1_ty], [IntrNoMem, NoCapture>]>; +def int_coro_await_suspend_void : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty], + [Throws]>; + +def int_coro_await_suspend_bool : Intrinsic<[llvm_i1_ty], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty], + [Throws]>; + +def int_coro_await_suspend_handle : Intrinsic<[llvm_ptr_ty], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty], + [Throws]>; + // Coroutine Lowering Intrinsics. Used internally by coroutine passes. def int_coro_subfn_addr : DefaultAttrsIntrinsic< diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 3cf5e81efb3b1..ce090c3b8a744 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -4999,6 +4999,9 @@ void Verifier::visitInstruction(Instruction &I) { F->getIntrinsicID() == Intrinsic::seh_scope_end || F->getIntrinsicID() == Intrinsic::coro_resume || F->getIntrinsicID() == Intrinsic::coro_destroy || + F->getIntrinsicID() == Intrinsic::coro_await_suspend_void || + F->getIntrinsicID() == Intrinsic::coro_await_suspend_bool || + F->getIntrinsicID() == Intrinsic::coro_await_suspend_handle || F->getIntrinsicID() == Intrinsic::experimental_patchpoint_void || F->getIntrinsicID() == Intrinsic::experimental_patchpoint_i64 || diff --git a/llvm/lib/Transforms/Coroutines/CoroInstr.h b/llvm/lib/Transforms/Coroutines/CoroInstr.h index f01aa58eb8999..79e745bb162cd 100644 --- a/llvm/lib/Transforms/Coroutines/CoroInstr.h +++ b/llvm/lib/Transforms/Coroutines/CoroInstr.h @@ -78,6 +78,39 @@ class LLVM_LIBRARY_VISIBILITY CoroAllocInst : public IntrinsicInst { } }; +/// This represents the llvm.coro.await.suspend instruction. +// FIXME: add callback metadata +// FIXME: make a proper IntrinisicInst. Currently this is not possible, +// because llvm.coro.await.suspend can be invoked. +class LLVM_LIBRARY_VISIBILITY CoroAwaitSuspendInst : public CallBase { + enum { AwaiterArg, FrameArg, WrapperArg }; + +public: + Value *getAwaiter() const { return getArgOperand(AwaiterArg); } + + Value *getFrame() const { return getArgOperand(FrameArg); } + + Function *getWrapperFunction() const { + return cast(getArgOperand(WrapperArg)); + } + + // Methods to support type inquiry through isa, cast, and dyn_cast: + static bool classof(const CallBase *CB) { + if (const Function *CF = CB->getCalledFunction()) { + auto IID = CF->getIntrinsicID(); + return IID == Intrinsic::coro_await_suspend_void || + IID == Intrinsic::coro_await_suspend_bool || + IID == Intrinsic::coro_await_suspend_handle; + } + + return false; + } + + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } +}; + /// This represents a common base class for llvm.coro.id instructions. class LLVM_LIBRARY_VISIBILITY AnyCoroIdInst : public IntrinsicInst { public: diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h index 388cf8d2aee71..09d1430b4c559 100644 --- a/llvm/lib/Transforms/Coroutines/CoroInternal.h +++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h @@ -84,6 +84,7 @@ struct LLVM_LIBRARY_VISIBILITY Shape { SmallVector CoroAligns; SmallVector CoroSuspends; SmallVector SwiftErrorOps; + SmallVector CoroAwaitSuspends; // Field indexes for special fields in the switch lowering. struct SwitchFieldIndex { diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp index 99675aa495f53..58b95e43b8994 100644 --- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp @@ -167,6 +167,55 @@ class CoroCloner { } // end anonymous namespace +// FIXME: +// Lower the intrinisc in CoroEarly phase if coroutine frame doesn't escape +// and it is known that other transformations, for example, sanitizers +// won't lead to incorrect code. +static void lowerAwaitSuspend(IRBuilder<> &Builder, CoroAwaitSuspendInst *CB) { + auto Wrapper = CB->getWrapperFunction(); + auto Awaiter = CB->getAwaiter(); + auto FramePtr = CB->getFrame(); + + Builder.SetInsertPoint(CB); + + CallBase *NewCall = nullptr; + // await_suspend has only 2 parameters, awaiter and handle. + // Copy parameter attributes from the intrinsic call, but remove the last, + // because the last parameter now becomes the function that is being called. + AttributeList NewAttributes = + CB->getAttributes().removeParamAttributes(CB->getContext(), 2); + + if (auto Invoke = dyn_cast(CB)) { + auto WrapperInvoke = + Builder.CreateInvoke(Wrapper, Invoke->getNormalDest(), + Invoke->getUnwindDest(), {Awaiter, FramePtr}); + + WrapperInvoke->setCallingConv(Invoke->getCallingConv()); + std::copy(Invoke->bundle_op_info_begin(), Invoke->bundle_op_info_end(), + WrapperInvoke->bundle_op_info_begin()); + WrapperInvoke->setAttributes(NewAttributes); + WrapperInvoke->setDebugLoc(Invoke->getDebugLoc()); + NewCall = WrapperInvoke; + } else if (auto Call = dyn_cast(CB)) { + auto WrapperCall = Builder.CreateCall(Wrapper, {Awaiter, FramePtr}); + + WrapperCall->setAttributes(NewAttributes); + WrapperCall->setDebugLoc(Call->getDebugLoc()); + NewCall = WrapperCall; + } else { + llvm_unreachable("Unexpected coro_await_suspend invocation method"); + } + + CB->replaceAllUsesWith(NewCall); + CB->eraseFromParent(); +} + +static void lowerAwaitSuspends(Function &F, coro::Shape &Shape) { + IRBuilder<> Builder(F.getContext()); + for (auto *AWS : Shape.CoroAwaitSuspends) + lowerAwaitSuspend(Builder, AWS); +} + static void maybeFreeRetconStorage(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr, CallGraph *CG) { @@ -2025,6 +2074,8 @@ splitCoroutine(Function &F, SmallVectorImpl &Clones, if (!Shape.CoroBegin) return Shape; + lowerAwaitSuspends(F, Shape); + simplifySuspendPoints(Shape); buildCoroutineFrame(F, Shape, TTI, MaterializableCallback); replaceFrameSizeAndAlignment(Shape); diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp index 7bd151ed4dc1e..a1c78d6a44ef4 100644 --- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp +++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp @@ -67,6 +67,9 @@ static const char *const CoroIntrinsics[] = { "llvm.coro.async.resume", "llvm.coro.async.size.replace", "llvm.coro.async.store_resume", + "llvm.coro.await.suspend.bool", + "llvm.coro.await.suspend.handle", + "llvm.coro.await.suspend.void", "llvm.coro.begin", "llvm.coro.destroy", "llvm.coro.done", @@ -174,7 +177,11 @@ void coro::Shape::buildFrom(Function &F) { SmallVector UnusedCoroSaves; for (Instruction &I : instructions(F)) { - if (auto II = dyn_cast(&I)) { + // FIXME: coro_await_suspend_* are not proper `IntrinisicInst`s + // because they might be invoked + if (auto AWS = dyn_cast(&I)) { + CoroAwaitSuspends.push_back(AWS); + } else if (auto II = dyn_cast(&I)) { switch (II->getIntrinsicID()) { default: continue; diff --git a/llvm/test/Transforms/Coroutines/coro-await-suspend-lower-invoke.ll b/llvm/test/Transforms/Coroutines/coro-await-suspend-lower-invoke.ll new file mode 100644 index 0000000000000..fbc4a2c006f84 --- /dev/null +++ b/llvm/test/Transforms/Coroutines/coro-await-suspend-lower-invoke.ll @@ -0,0 +1,123 @@ +; Tests that invoke @llvm.coro.await.suspend lowers to invoke @helper +; RUN: opt < %s -passes='module(coro-early),cgscc(coro-split),simplifycfg' -S | FileCheck %s + +%Awaiter = type {} + +; CHECK: define {{[^@]*}} @f.resume(ptr {{[^%]*}} %[[HDL:.+]]) +; CHECK: %[[AWAITER:.+]] = getelementptr inbounds %f.Frame, ptr %[[HDL]], i32 0, i32 0 +define void @f() presplitcoroutine personality i32 0 { +entry: + %awaiter = alloca %Awaiter + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %size = call i32 @llvm.coro.size.i32() + %alloc = call ptr @malloc(i32 %size) + %hdl = call ptr @llvm.coro.begin(token %id, ptr %alloc) + ; Initial suspend so that all 3 await_suspend invocations are inside f.resume + %suspend.init = call i8 @llvm.coro.suspend(token none, i1 false) + switch i8 %suspend.init, label %ret [ + i8 0, label %step + i8 1, label %cleanup + ] + +; CHECK: invoke void @await_suspend_wrapper_void(ptr %[[AWAITER]], ptr %[[HDL]]) +; CHECK-NEXT: to label %[[STEP_CONT:[^ ]+]] unwind label %[[PAD:[^ ]+]] +step: + %save = call token @llvm.coro.save(ptr null) + invoke void @llvm.coro.await.suspend.void(ptr %awaiter, ptr %hdl, ptr @await_suspend_wrapper_void) + to label %step.continue unwind label %pad + +; CHECK [[STEP_CONT]]: +step.continue: + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + switch i8 %suspend, label %ret [ + i8 0, label %step1 + i8 1, label %cleanup + ] + +; CHECK: %[[RESUME:.+]] = invoke i1 @await_suspend_wrapper_bool(ptr %[[AWAITER]], ptr %[[HDL]]) +; CHECK-NEXT: to label %[[STEP1_CONT:[^ ]+]] unwind label %[[PAD]] +step1: + %save1 = call token @llvm.coro.save(ptr null) + %resume.bool = invoke i1 @llvm.coro.await.suspend.bool(ptr %awaiter, ptr %hdl, ptr @await_suspend_wrapper_bool) + to label %step1.continue unwind label %pad + +; CHECK: [[STEP1_CONT]]: +; CHECK-NEXT: br i1 %[[RESUME]], label %{{[^,]+}}, label %[[STEP2:.+]] +step1.continue: + br i1 %resume.bool, label %suspend.cond, label %step2 + +suspend.cond: + %suspend1 = call i8 @llvm.coro.suspend(token %save1, i1 false) + switch i8 %suspend1, label %ret [ + i8 0, label %step2 + i8 1, label %cleanup + ] + +; CHECK: [[STEP2]]: +; CHECK: %[[NEXT_HDL:.+]] = invoke ptr @await_suspend_wrapper_handle(ptr %[[AWAITER]], ptr %[[HDL]]) +; CHECK-NEXT: to label %[[STEP2_CONT:[^ ]+]] unwind label %[[PAD]] +step2: + %save2 = call token @llvm.coro.save(ptr null) + %resume.handle = invoke ptr @llvm.coro.await.suspend.handle(ptr %awaiter, ptr %hdl, ptr @await_suspend_wrapper_handle) + to label %step2.continue unwind label %pad + +; CHECK: [[STEP2_CONT]]: +; CHECK-NEXT: %[[NEXT_RESUME:.+]] = call ptr @llvm.coro.subfn.addr(ptr %[[NEXT_HDL]], i8 0) +; CHECK-NEXT: musttail call {{.*}} void %[[NEXT_RESUME]](ptr %[[NEXT_HDL]]) +step2.continue: + call void @llvm.coro.resume(ptr %resume.handle) + %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false) + switch i8 %suspend2, label %ret [ + i8 0, label %step3 + i8 1, label %cleanup + ] + +step3: + br label %cleanup + +pad: + %lp = landingpad { ptr, i32 } + catch ptr null + %exn = extractvalue { ptr, i32 } %lp, 0 + call ptr @__cxa_begin_catch(ptr %exn) + call void @__cxa_end_catch() + br label %cleanup + +cleanup: + %mem = call ptr @llvm.coro.free(token %id, ptr %hdl) + call void @free(ptr %mem) + br label %ret + +ret: + call i1 @llvm.coro.end(ptr %hdl, i1 0, token none) + ret void +} + +; check that we were haven't accidentally went out of @f.resume body +; CHECK-LABEL: @f.destroy( +; CHECK-LABEL: @f.cleanup( + +declare void @await_suspend_wrapper_void(ptr, ptr) +declare i1 @await_suspend_wrapper_bool(ptr, ptr) +declare ptr @await_suspend_wrapper_handle(ptr, ptr) + +declare ptr @llvm.coro.free(token, ptr) +declare i32 @llvm.coro.size.i32() +declare i8 @llvm.coro.suspend(token, i1) +declare void @llvm.coro.resume(ptr) +declare void @llvm.coro.destroy(ptr) + +declare token @llvm.coro.id(i32, ptr, ptr, ptr) +declare i1 @llvm.coro.alloc(token) +declare ptr @llvm.coro.begin(token, ptr) +declare void @llvm.coro.await.suspend.void(ptr, ptr, ptr) +declare i1 @llvm.coro.await.suspend.bool(ptr, ptr, ptr) +declare ptr @llvm.coro.await.suspend.handle(ptr, ptr, ptr) +declare i1 @llvm.coro.end(ptr, i1, token) + +declare ptr @__cxa_begin_catch(ptr) +declare void @use_val(i32) +declare void @__cxa_end_catch() + +declare noalias ptr @malloc(i32) +declare void @free(ptr) diff --git a/llvm/test/Transforms/Coroutines/coro-await-suspend-lower.ll b/llvm/test/Transforms/Coroutines/coro-await-suspend-lower.ll new file mode 100644 index 0000000000000..0f574c4acc26e --- /dev/null +++ b/llvm/test/Transforms/Coroutines/coro-await-suspend-lower.ll @@ -0,0 +1,96 @@ +; Tests lowerings of different versions of coro.await.suspend +; RUN: opt < %s -passes='module(coro-early),cgscc(coro-split),simplifycfg' -S | FileCheck %s + +%Awaiter = type {} + +; CHECK: define {{[^@]*}} @f.resume(ptr {{[^%]*}} %[[HDL:.+]]) +; CHECK: %[[AWAITER:.+]] = getelementptr inbounds %f.Frame, ptr %[[HDL]], i32 0, i32 0 +define void @f() presplitcoroutine { +entry: + %awaiter = alloca %Awaiter + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %size = call i32 @llvm.coro.size.i32() + %alloc = call ptr @malloc(i32 %size) + %hdl = call ptr @llvm.coro.begin(token %id, ptr %alloc) + %suspend.init = call i8 @llvm.coro.suspend(token none, i1 false) + switch i8 %suspend.init, label %ret [ + i8 0, label %step + i8 1, label %cleanup + ] + +; CHECK: call void @await_suspend_wrapper_void(ptr %[[AWAITER]], ptr %[[HDL]]) +; CHECK-NEXT: br label %{{.*}} +step: + %save = call token @llvm.coro.save(ptr null) + call void @llvm.coro.await.suspend.void(ptr %awaiter, ptr %hdl, ptr @await_suspend_wrapper_void) + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + switch i8 %suspend, label %ret [ + i8 0, label %step1 + i8 1, label %cleanup + ] + +; CHECK: %[[RESUME:.+]] = call i1 @await_suspend_wrapper_bool(ptr %[[AWAITER]], ptr %[[HDL]]) +; CHECK-NEXT: br i1 %[[RESUME]], label %{{[^,]+}}, label %[[STEP2:.+]] +step1: + %save1 = call token @llvm.coro.save(ptr null) + %resume.bool = call i1 @llvm.coro.await.suspend.bool(ptr %awaiter, ptr %hdl, ptr @await_suspend_wrapper_bool) + br i1 %resume.bool, label %suspend.cond, label %step2 + +suspend.cond: + %suspend1 = call i8 @llvm.coro.suspend(token %save1, i1 false) + switch i8 %suspend1, label %ret [ + i8 0, label %step2 + i8 1, label %cleanup + ] + +; CHECK: [[STEP2]]: +; CHECK: %[[NEXT_HDL:.+]] = call ptr @await_suspend_wrapper_handle(ptr %[[AWAITER]], ptr %[[HDL]]) +; CHECK-NEXT: %[[CONT:.+]] = call ptr @llvm.coro.subfn.addr(ptr %[[NEXT_HDL]], i8 0) +; CHECK-NEXT: musttail call {{.*}} void %[[CONT]](ptr %[[NEXT_HDL]]) +step2: + %save2 = call token @llvm.coro.save(ptr null) + %resume.handle = call ptr @llvm.coro.await.suspend.handle(ptr %awaiter, ptr %hdl, ptr @await_suspend_wrapper_handle) + call void @llvm.coro.resume(ptr %resume.handle) + %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false) + switch i8 %suspend2, label %ret [ + i8 0, label %step3 + i8 1, label %cleanup + ] + +step3: + br label %cleanup + +cleanup: + %mem = call ptr @llvm.coro.free(token %id, ptr %hdl) + call void @free(ptr %mem) + br label %ret + +ret: + call i1 @llvm.coro.end(ptr %hdl, i1 0, token none) + ret void +} + +; check that we were haven't accidentally went out of @f.resume body +; CHECK-LABEL: @f.destroy( +; CHECK-LABEL: @f.cleanup( + +declare void @await_suspend_wrapper_void(ptr, ptr) +declare i1 @await_suspend_wrapper_bool(ptr, ptr) +declare ptr @await_suspend_wrapper_handle(ptr, ptr) + +declare ptr @llvm.coro.free(token, ptr) +declare i32 @llvm.coro.size.i32() +declare i8 @llvm.coro.suspend(token, i1) +declare void @llvm.coro.resume(ptr) +declare void @llvm.coro.destroy(ptr) + +declare token @llvm.coro.id(i32, ptr, ptr, ptr) +declare i1 @llvm.coro.alloc(token) +declare ptr @llvm.coro.begin(token, ptr) +declare void @llvm.coro.await.suspend.void(ptr, ptr, ptr) +declare i1 @llvm.coro.await.suspend.bool(ptr, ptr, ptr) +declare ptr @llvm.coro.await.suspend.handle(ptr, ptr, ptr) +declare i1 @llvm.coro.end(ptr, i1, token) + +declare noalias ptr @malloc(i32) +declare void @free(ptr)