Skip to content

Commit 858e207

Browse files
committed
Implement llvm.coro.await.suspend intrinsic
1 parent 53fea6f commit 858e207

20 files changed

+590
-336
lines changed

clang/include/clang/AST/ExprCXX.h

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5035,15 +5035,19 @@ class CoroutineSuspendExpr : public Expr {
50355035
enum SubExpr { Operand, Common, Ready, Suspend, Resume, Count };
50365036

50375037
Stmt *SubExprs[SubExpr::Count];
5038+
bool IsSuspendNoThrow = false;
50385039
OpaqueValueExpr *OpaqueValue = nullptr;
5040+
OpaqueValueExpr *OpaqueFramePtr = nullptr;
50395041

50405042
public:
50415043
CoroutineSuspendExpr(StmtClass SC, SourceLocation KeywordLoc, Expr *Operand,
50425044
Expr *Common, Expr *Ready, Expr *Suspend, Expr *Resume,
5043-
OpaqueValueExpr *OpaqueValue)
5045+
bool IsSuspendNoThrow, OpaqueValueExpr *OpaqueValue,
5046+
OpaqueValueExpr *OpaqueFramePtr)
50445047
: Expr(SC, Resume->getType(), Resume->getValueKind(),
50455048
Resume->getObjectKind()),
5046-
KeywordLoc(KeywordLoc), OpaqueValue(OpaqueValue) {
5049+
KeywordLoc(KeywordLoc), IsSuspendNoThrow(IsSuspendNoThrow),
5050+
OpaqueValue(OpaqueValue), OpaqueFramePtr(OpaqueFramePtr) {
50475051
SubExprs[SubExpr::Operand] = Operand;
50485052
SubExprs[SubExpr::Common] = Common;
50495053
SubExprs[SubExpr::Ready] = Ready;
@@ -5080,6 +5084,9 @@ class CoroutineSuspendExpr : public Expr {
50805084
/// getOpaqueValue - Return the opaque value placeholder.
50815085
OpaqueValueExpr *getOpaqueValue() const { return OpaqueValue; }
50825086

5087+
/// getOpaqueFramePtr - Return coroutine frame pointer placeholder.
5088+
OpaqueValueExpr *getOpaqueFramePtr() const { return OpaqueFramePtr; }
5089+
50835090
Expr *getReadyExpr() const {
50845091
return static_cast<Expr*>(SubExprs[SubExpr::Ready]);
50855092
}
@@ -5097,6 +5104,8 @@ class CoroutineSuspendExpr : public Expr {
50975104
return static_cast<Expr *>(SubExprs[SubExpr::Operand]);
50985105
}
50995106

5107+
bool isSuspendNoThrow() const { return IsSuspendNoThrow; }
5108+
51005109
SourceLocation getKeywordLoc() const { return KeywordLoc; }
51015110

51025111
SourceLocation getBeginLoc() const LLVM_READONLY { return KeywordLoc; }
@@ -5125,10 +5134,12 @@ class CoawaitExpr : public CoroutineSuspendExpr {
51255134

51265135
public:
51275136
CoawaitExpr(SourceLocation CoawaitLoc, Expr *Operand, Expr *Common,
5128-
Expr *Ready, Expr *Suspend, Expr *Resume,
5129-
OpaqueValueExpr *OpaqueValue, bool IsImplicit = false)
5137+
Expr *Ready, Expr *Suspend, Expr *Resume, bool IsSuspendNoThrow,
5138+
OpaqueValueExpr *OpaqueValue, OpaqueValueExpr *OpaqueFramePtr,
5139+
bool IsImplicit = false)
51305140
: CoroutineSuspendExpr(CoawaitExprClass, CoawaitLoc, Operand, Common,
5131-
Ready, Suspend, Resume, OpaqueValue) {
5141+
Ready, Suspend, Resume, IsSuspendNoThrow,
5142+
OpaqueValue, OpaqueFramePtr) {
51325143
CoawaitBits.IsImplicit = IsImplicit;
51335144
}
51345145

@@ -5206,10 +5217,11 @@ class CoyieldExpr : public CoroutineSuspendExpr {
52065217

52075218
public:
52085219
CoyieldExpr(SourceLocation CoyieldLoc, Expr *Operand, Expr *Common,
5209-
Expr *Ready, Expr *Suspend, Expr *Resume,
5210-
OpaqueValueExpr *OpaqueValue)
5220+
Expr *Ready, Expr *Suspend, Expr *Resume, bool IsSuspendNoThrow,
5221+
OpaqueValueExpr *OpaqueValue, OpaqueValueExpr *OpaqueFramePtr)
52115222
: CoroutineSuspendExpr(CoyieldExprClass, CoyieldLoc, Operand, Common,
5212-
Ready, Suspend, Resume, OpaqueValue) {}
5223+
Ready, Suspend, Resume, IsSuspendNoThrow,
5224+
OpaqueValue, OpaqueFramePtr) {}
52135225
CoyieldExpr(SourceLocation CoyieldLoc, QualType Ty, Expr *Operand,
52145226
Expr *Common)
52155227
: CoroutineSuspendExpr(CoyieldExprClass, CoyieldLoc, Ty, Operand,

clang/lib/CodeGen/CGCoroutine.cpp

Lines changed: 132 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -212,9 +212,10 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co
212212
bool ignoreResult, bool forLValue) {
213213
auto *E = S.getCommonExpr();
214214

215-
auto Binder =
215+
auto CommonBinder =
216216
CodeGenFunction::OpaqueValueMappingData::bind(CGF, S.getOpaqueValue(), E);
217-
auto UnbindOnExit = llvm::make_scope_exit([&] { Binder.unbind(CGF); });
217+
auto UnbindCommonOnExit =
218+
llvm::make_scope_exit([&] { CommonBinder.unbind(CGF); });
218219

219220
auto Prefix = buildSuspendPrefixStr(Coro, Kind);
220221
BasicBlock *ReadyBlock = CGF.createBasicBlock(Prefix + Twine(".ready"));
@@ -232,16 +233,57 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co
232233
auto *NullPtr = llvm::ConstantPointerNull::get(CGF.CGM.Int8PtrTy);
233234
auto *SaveCall = Builder.CreateCall(CoroSave, {NullPtr});
234235

235-
CGF.CurCoro.InSuspendBlock = true;
236-
auto *SuspendRet = CGF.EmitScalarExpr(S.getSuspendExpr());
237-
CGF.CurCoro.InSuspendBlock = false;
236+
auto SuspendHelper = CodeGenFunction(CGF.CGM).generateAwaitSuspendHelper(
237+
CGF.CurFn->getName(), Prefix, S);
238238

239-
if (SuspendRet != nullptr && SuspendRet->getType()->isIntegerTy(1)) {
240-
// Veto suspension if requested by bool returning await_suspend.
241-
BasicBlock *RealSuspendBlock =
242-
CGF.createBasicBlock(Prefix + Twine(".suspend.bool"));
243-
CGF.Builder.CreateCondBr(SuspendRet, RealSuspendBlock, ReadyBlock);
244-
CGF.EmitBlock(RealSuspendBlock);
239+
llvm::CallBase *SuspendRet = nullptr;
240+
241+
{
242+
CGF.CurCoro.InSuspendBlock = true;
243+
244+
auto FramePtrBinder = CodeGenFunction::OpaqueValueMappingData::bind(
245+
CGF, S.getOpaqueFramePtr(), S.getOpaqueFramePtr()->getSourceExpr());
246+
auto UnbindFramePtrOnExit =
247+
llvm::make_scope_exit([&] { FramePtrBinder.unbind(CGF); });
248+
249+
SmallVector<llvm::Value *, 3> SuspendHelperCallArgs;
250+
SuspendHelperCallArgs.push_back(
251+
CGF.getOrCreateOpaqueLValueMapping(S.getOpaqueValue()).getPointer(CGF));
252+
SuspendHelperCallArgs.push_back(
253+
CGF.getOrCreateOpaqueRValueMapping(S.getOpaqueFramePtr())
254+
.getScalarVal());
255+
SuspendHelperCallArgs.push_back(SuspendHelper);
256+
257+
auto IID = llvm::Intrinsic::coro_await_suspend;
258+
if (S.getSuspendExpr()->getType()->isBooleanType())
259+
IID = llvm::Intrinsic::coro_await_suspend_bool;
260+
else if (S.getSuspendExpr()->getType()->isVoidPointerType())
261+
IID = llvm::Intrinsic::coro_await_suspend_handle;
262+
263+
llvm::Function *AwaitSuspendIntrinsic = CGF.CGM.getIntrinsic(IID);
264+
// FIXME: add call attributes?
265+
if (S.isSuspendNoThrow()) {
266+
SuspendRet = CGF.EmitNounwindRuntimeCall(AwaitSuspendIntrinsic,
267+
SuspendHelperCallArgs);
268+
} else {
269+
SuspendRet =
270+
CGF.EmitCallOrInvoke(AwaitSuspendIntrinsic, SuspendHelperCallArgs);
271+
}
272+
273+
CGF.CurCoro.InSuspendBlock = false;
274+
}
275+
276+
if (SuspendRet != nullptr) {
277+
if (SuspendRet->getType()->isIntegerTy(1)) {
278+
// Veto suspension if requested by bool returning await_suspend.
279+
BasicBlock *RealSuspendBlock =
280+
CGF.createBasicBlock(Prefix + Twine(".suspend.bool"));
281+
CGF.Builder.CreateCondBr(SuspendRet, RealSuspendBlock, ReadyBlock);
282+
CGF.EmitBlock(RealSuspendBlock);
283+
} else if (SuspendRet->getType()->isPointerTy()) {
284+
auto ResumeIntrinsic = CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_resume);
285+
Builder.CreateCall(ResumeIntrinsic, SuspendRet);
286+
}
245287
}
246288

247289
// Emit the suspend point.
@@ -338,6 +380,85 @@ static QualType getCoroutineSuspendExprReturnType(const ASTContext &Ctx,
338380
}
339381
#endif
340382

383+
llvm::Function *
384+
CodeGenFunction::generateAwaitSuspendHelper(Twine const &CoroName,
385+
Twine const &SuspendPointName,
386+
CoroutineSuspendExpr const &S) {
387+
std::string FuncName = "__await_suspend_helper_";
388+
FuncName += CoroName.str();
389+
FuncName += SuspendPointName.str();
390+
391+
ASTContext &C = getContext();
392+
393+
FunctionArgList args;
394+
395+
ImplicitParamDecl AwaiterDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);
396+
ImplicitParamDecl FrameDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);
397+
QualType ReturnTy = S.getSuspendExpr()->getType();
398+
399+
if (ReturnTy->isBooleanType()) {
400+
ReturnTy = C.BoolTy;
401+
} else if (ReturnTy->isVoidPointerType()) {
402+
ReturnTy = C.VoidPtrTy;
403+
} else {
404+
ReturnTy = C.VoidTy;
405+
}
406+
407+
args.push_back(&AwaiterDecl);
408+
args.push_back(&FrameDecl);
409+
410+
const CGFunctionInfo &FI =
411+
CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, args);
412+
413+
llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI);
414+
415+
llvm::Function *Fn = llvm::Function::Create(
416+
LTy, llvm::GlobalValue::PrivateLinkage, FuncName, &CGM.getModule());
417+
418+
Fn->addParamAttr(0, llvm::Attribute::AttrKind::NonNull);
419+
Fn->addParamAttr(0, llvm::Attribute::AttrKind::NoUndef);
420+
421+
Fn->addParamAttr(1, llvm::Attribute::AttrKind::NoUndef);
422+
423+
Fn->setMustProgress();
424+
Fn->addFnAttr(llvm::Attribute::AttrKind::AlwaysInline);
425+
426+
if (S.isSuspendNoThrow()) {
427+
Fn->addFnAttr(llvm::Attribute::AttrKind::NoUnwind);
428+
}
429+
430+
StartFunction(GlobalDecl(), ReturnTy, Fn, FI, args);
431+
432+
llvm::Value *AwaiterPtr = Builder.CreateLoad(GetAddrOfLocalVar(&AwaiterDecl));
433+
auto AwaiterLValue =
434+
MakeNaturalAlignAddrLValue(AwaiterPtr, AwaiterDecl.getType());
435+
436+
// FIXME: mark as aliasing with awaiter?
437+
// FIXME: TBAA?
438+
// FIXME: emit in a better way (maybe egenerate AST in SemaCoroutine)?
439+
auto FramePtrRValue =
440+
RValue::get(Builder.CreateLoad(GetAddrOfLocalVar(&FrameDecl)));
441+
442+
auto AwaiterBinder = CodeGenFunction::OpaqueValueMappingData::bind(
443+
*this, S.getOpaqueValue(), AwaiterLValue);
444+
auto FramePtrBinder = CodeGenFunction::OpaqueValueMappingData::bind(
445+
*this, S.getOpaqueFramePtr(), FramePtrRValue);
446+
447+
auto *SuspendRet = EmitScalarExpr(S.getSuspendExpr());
448+
449+
auto UnbindCommonOnExit =
450+
llvm::make_scope_exit([&] { AwaiterBinder.unbind(*this); });
451+
auto UnbindFramePtrOnExit =
452+
llvm::make_scope_exit([&] { FramePtrBinder.unbind(*this); });
453+
if (SuspendRet != nullptr) {
454+
Fn->addRetAttr(llvm::Attribute::AttrKind::NoUndef);
455+
Builder.CreateStore(SuspendRet, ReturnValue);
456+
}
457+
458+
FinishFunction();
459+
return Fn;
460+
}
461+
341462
LValue
342463
CodeGenFunction::EmitCoawaitLValue(const CoawaitExpr *E) {
343464
assert(getCoroutineSuspendExprReturnType(getContext(), E)->isReferenceType() &&

clang/lib/CodeGen/CodeGenFunction.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,10 @@ class CodeGenFunction : public CodeGenTypeCache {
350350
return isCoroutine() && CurCoro.InSuspendBlock;
351351
}
352352

353+
llvm::Function *generateAwaitSuspendHelper(Twine const &CoroName,
354+
Twine const &SuspendPointName,
355+
CoroutineSuspendExpr const &S);
356+
353357
/// CurGD - The GlobalDecl for the current function being compiled.
354358
GlobalDecl CurGD;
355359

0 commit comments

Comments
 (0)