@@ -212,9 +212,10 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co
212
212
bool ignoreResult, bool forLValue) {
213
213
auto *E = S.getCommonExpr ();
214
214
215
- auto Binder =
215
+ auto CommonBinder =
216
216
CodeGenFunction::OpaqueValueMappingData::bind (CGF, S.getOpaqueValue (), E);
217
- auto UnbindOnExit = llvm::make_scope_exit ([&] { Binder.unbind (CGF); });
217
+ auto UnbindCommonOnExit =
218
+ llvm::make_scope_exit ([&] { CommonBinder.unbind (CGF); });
218
219
219
220
auto Prefix = buildSuspendPrefixStr (Coro, Kind);
220
221
BasicBlock *ReadyBlock = CGF.createBasicBlock (Prefix + Twine (" .ready" ));
@@ -232,16 +233,57 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co
232
233
auto *NullPtr = llvm::ConstantPointerNull::get (CGF.CGM .Int8PtrTy );
233
234
auto *SaveCall = Builder.CreateCall (CoroSave, {NullPtr});
234
235
235
- CGF.CurCoro .InSuspendBlock = true ;
236
- auto *SuspendRet = CGF.EmitScalarExpr (S.getSuspendExpr ());
237
- CGF.CurCoro .InSuspendBlock = false ;
236
+ auto SuspendHelper = CodeGenFunction (CGF.CGM ).generateAwaitSuspendHelper (
237
+ CGF.CurFn ->getName (), Prefix, S);
238
238
239
- if (SuspendRet != nullptr && SuspendRet->getType ()->isIntegerTy (1 )) {
240
- // Veto suspension if requested by bool returning await_suspend.
241
- BasicBlock *RealSuspendBlock =
242
- CGF.createBasicBlock (Prefix + Twine (" .suspend.bool" ));
243
- CGF.Builder .CreateCondBr (SuspendRet, RealSuspendBlock, ReadyBlock);
244
- CGF.EmitBlock (RealSuspendBlock);
239
+ llvm::CallBase *SuspendRet = nullptr ;
240
+
241
+ {
242
+ CGF.CurCoro .InSuspendBlock = true ;
243
+
244
+ auto FramePtrBinder = CodeGenFunction::OpaqueValueMappingData::bind (
245
+ CGF, S.getOpaqueFramePtr (), S.getOpaqueFramePtr ()->getSourceExpr ());
246
+ auto UnbindFramePtrOnExit =
247
+ llvm::make_scope_exit ([&] { FramePtrBinder.unbind (CGF); });
248
+
249
+ SmallVector<llvm::Value *, 3 > SuspendHelperCallArgs;
250
+ SuspendHelperCallArgs.push_back (
251
+ CGF.getOrCreateOpaqueLValueMapping (S.getOpaqueValue ()).getPointer (CGF));
252
+ SuspendHelperCallArgs.push_back (
253
+ CGF.getOrCreateOpaqueRValueMapping (S.getOpaqueFramePtr ())
254
+ .getScalarVal ());
255
+ SuspendHelperCallArgs.push_back (SuspendHelper);
256
+
257
+ auto IID = llvm::Intrinsic::coro_await_suspend;
258
+ if (S.getSuspendExpr ()->getType ()->isBooleanType ())
259
+ IID = llvm::Intrinsic::coro_await_suspend_bool;
260
+ else if (S.getSuspendExpr ()->getType ()->isVoidPointerType ())
261
+ IID = llvm::Intrinsic::coro_await_suspend_handle;
262
+
263
+ llvm::Function *AwaitSuspendIntrinsic = CGF.CGM .getIntrinsic (IID);
264
+ // FIXME: add call attributes?
265
+ if (S.isSuspendNoThrow ()) {
266
+ SuspendRet = CGF.EmitNounwindRuntimeCall (AwaitSuspendIntrinsic,
267
+ SuspendHelperCallArgs);
268
+ } else {
269
+ SuspendRet =
270
+ CGF.EmitCallOrInvoke (AwaitSuspendIntrinsic, SuspendHelperCallArgs);
271
+ }
272
+
273
+ CGF.CurCoro .InSuspendBlock = false ;
274
+ }
275
+
276
+ if (SuspendRet != nullptr ) {
277
+ if (SuspendRet->getType ()->isIntegerTy (1 )) {
278
+ // Veto suspension if requested by bool returning await_suspend.
279
+ BasicBlock *RealSuspendBlock =
280
+ CGF.createBasicBlock (Prefix + Twine (" .suspend.bool" ));
281
+ CGF.Builder .CreateCondBr (SuspendRet, RealSuspendBlock, ReadyBlock);
282
+ CGF.EmitBlock (RealSuspendBlock);
283
+ } else if (SuspendRet->getType ()->isPointerTy ()) {
284
+ auto ResumeIntrinsic = CGF.CGM .getIntrinsic (llvm::Intrinsic::coro_resume);
285
+ Builder.CreateCall (ResumeIntrinsic, SuspendRet);
286
+ }
245
287
}
246
288
247
289
// Emit the suspend point.
@@ -338,6 +380,85 @@ static QualType getCoroutineSuspendExprReturnType(const ASTContext &Ctx,
338
380
}
339
381
#endif
340
382
383
+ llvm::Function *
384
+ CodeGenFunction::generateAwaitSuspendHelper (Twine const &CoroName,
385
+ Twine const &SuspendPointName,
386
+ CoroutineSuspendExpr const &S) {
387
+ std::string FuncName = " __await_suspend_helper_" ;
388
+ FuncName += CoroName.str ();
389
+ FuncName += SuspendPointName.str ();
390
+
391
+ ASTContext &C = getContext ();
392
+
393
+ FunctionArgList args;
394
+
395
+ ImplicitParamDecl AwaiterDecl (C, C.VoidPtrTy , ImplicitParamKind::Other);
396
+ ImplicitParamDecl FrameDecl (C, C.VoidPtrTy , ImplicitParamKind::Other);
397
+ QualType ReturnTy = S.getSuspendExpr ()->getType ();
398
+
399
+ if (ReturnTy->isBooleanType ()) {
400
+ ReturnTy = C.BoolTy ;
401
+ } else if (ReturnTy->isVoidPointerType ()) {
402
+ ReturnTy = C.VoidPtrTy ;
403
+ } else {
404
+ ReturnTy = C.VoidTy ;
405
+ }
406
+
407
+ args.push_back (&AwaiterDecl);
408
+ args.push_back (&FrameDecl);
409
+
410
+ const CGFunctionInfo &FI =
411
+ CGM.getTypes ().arrangeBuiltinFunctionDeclaration (ReturnTy, args);
412
+
413
+ llvm::FunctionType *LTy = CGM.getTypes ().GetFunctionType (FI);
414
+
415
+ llvm::Function *Fn = llvm::Function::Create (
416
+ LTy, llvm::GlobalValue::PrivateLinkage, FuncName, &CGM.getModule ());
417
+
418
+ Fn->addParamAttr (0 , llvm::Attribute::AttrKind::NonNull);
419
+ Fn->addParamAttr (0 , llvm::Attribute::AttrKind::NoUndef);
420
+
421
+ Fn->addParamAttr (1 , llvm::Attribute::AttrKind::NoUndef);
422
+
423
+ Fn->setMustProgress ();
424
+ Fn->addFnAttr (llvm::Attribute::AttrKind::AlwaysInline);
425
+
426
+ if (S.isSuspendNoThrow ()) {
427
+ Fn->addFnAttr (llvm::Attribute::AttrKind::NoUnwind);
428
+ }
429
+
430
+ StartFunction (GlobalDecl (), ReturnTy, Fn, FI, args);
431
+
432
+ llvm::Value *AwaiterPtr = Builder.CreateLoad (GetAddrOfLocalVar (&AwaiterDecl));
433
+ auto AwaiterLValue =
434
+ MakeNaturalAlignAddrLValue (AwaiterPtr, AwaiterDecl.getType ());
435
+
436
+ // FIXME: mark as aliasing with awaiter?
437
+ // FIXME: TBAA?
438
+ // FIXME: emit in a better way (maybe egenerate AST in SemaCoroutine)?
439
+ auto FramePtrRValue =
440
+ RValue::get (Builder.CreateLoad (GetAddrOfLocalVar (&FrameDecl)));
441
+
442
+ auto AwaiterBinder = CodeGenFunction::OpaqueValueMappingData::bind (
443
+ *this , S.getOpaqueValue (), AwaiterLValue);
444
+ auto FramePtrBinder = CodeGenFunction::OpaqueValueMappingData::bind (
445
+ *this , S.getOpaqueFramePtr (), FramePtrRValue);
446
+
447
+ auto *SuspendRet = EmitScalarExpr (S.getSuspendExpr ());
448
+
449
+ auto UnbindCommonOnExit =
450
+ llvm::make_scope_exit ([&] { AwaiterBinder.unbind (*this ); });
451
+ auto UnbindFramePtrOnExit =
452
+ llvm::make_scope_exit ([&] { FramePtrBinder.unbind (*this ); });
453
+ if (SuspendRet != nullptr ) {
454
+ Fn->addRetAttr (llvm::Attribute::AttrKind::NoUndef);
455
+ Builder.CreateStore (SuspendRet, ReturnValue);
456
+ }
457
+
458
+ FinishFunction ();
459
+ return Fn;
460
+ }
461
+
341
462
LValue
342
463
CodeGenFunction::EmitCoawaitLValue (const CoawaitExpr *E) {
343
464
assert (getCoroutineSuspendExprReturnType (getContext (), E)->isReferenceType () &&
0 commit comments