Skip to content

Commit 5c30195

Browse files
aslasl
committed
Introduce llvm.coro.end.results intrinsics and use it to (optionally) return values
from `retcon.once` coroutines.
1 parent 1bf9db2 commit 5c30195

File tree

128 files changed

+436
-321
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

128 files changed

+436
-321
lines changed

‎clang/lib/CodeGen/CGCoroutine.cpp

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -403,8 +403,11 @@ struct CallCoroEnd final : public EHScopeStack::Cleanup {
403403
llvm::Function *CoroEndFn = CGM.getIntrinsic(llvm::Intrinsic::coro_end);
404404
// See if we have a funclet bundle to associate coro.end with. (WinEH)
405405
auto Bundles = getBundlesForCoroEnd(CGF);
406-
auto *CoroEnd = CGF.Builder.CreateCall(
407-
CoroEndFn, {NullPtr, CGF.Builder.getTrue()}, Bundles);
406+
auto *CoroEnd =
407+
CGF.Builder.CreateCall(CoroEndFn,
408+
{NullPtr, CGF.Builder.getTrue(),
409+
llvm::ConstantTokenNone::get(CoroEndFn->getContext())},
410+
Bundles);
408411
if (Bundles.empty()) {
409412
// Otherwise, (landingpad model), create a conditional branch that leads
410413
// either to a cleanup block or a block with EH resume instruction.
@@ -755,7 +758,9 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
755758
// Emit coro.end before getReturnStmt (and parameter destructors), since
756759
// resume and destroy parts of the coroutine should not include them.
757760
llvm::Function *CoroEnd = CGM.getIntrinsic(llvm::Intrinsic::coro_end);
758-
Builder.CreateCall(CoroEnd, {NullPtr, Builder.getFalse()});
761+
Builder.CreateCall(CoroEnd,
762+
{NullPtr, Builder.getFalse(),
763+
llvm::ConstantTokenNone::get(CoroEnd->getContext())});
759764

760765
if (Stmt *Ret = S.getReturnStmt()) {
761766
// Since we already emitted the return value above, so we shouldn't
@@ -824,7 +829,11 @@ RValue CodeGenFunction::EmitCoroutineIntrinsic(const CallExpr *E,
824829
}
825830
for (const Expr *Arg : E->arguments())
826831
Args.push_back(EmitScalarExpr(Arg));
827-
832+
// @llvm.coro.end takes a token parameter. Add token 'none' as the last
833+
// argument.
834+
if (IID == llvm::Intrinsic::coro_end)
835+
Args.push_back(llvm::ConstantTokenNone::get(getLLVMContext()));
836+
828837
llvm::Function *F = CGM.getIntrinsic(IID);
829838
llvm::CallInst *Call = Builder.CreateCall(F, Args);
830839

‎clang/test/CodeGenCoroutines/coro-builtins.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ void f(int n) {
3737
// CHECK-NEXT: call ptr @llvm.coro.free(token %[[COROID]], ptr %[[FRAME]])
3838
__builtin_coro_free(__builtin_coro_frame());
3939

40-
// CHECK-NEXT: call i1 (ptr, i1, ...) @llvm.coro.end(ptr %[[FRAME]], i1 false)
40+
// CHECK-NEXT: call i1 @llvm.coro.end(ptr %[[FRAME]], i1 false, token none)
4141
__builtin_coro_end(__builtin_coro_frame(), 0);
4242

4343
// CHECK-NEXT: call i8 @llvm.coro.suspend(token none, i1 true)

‎clang/test/CodeGenCoroutines/coro-eh-cleanup.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ coro_t f() {
6060

6161
// CHECK: [[COROENDBB]]:
6262
// CHECK-NEXT: %[[CLPAD:.+]] = cleanuppad within none
63-
// CHECK-NEXT: call i1 (ptr, i1, ...) @llvm.coro.end(ptr null, i1 true) [ "funclet"(token %[[CLPAD]]) ]
63+
// CHECK-NEXT: call i1 @llvm.coro.end(ptr null, i1 true, token none) [ "funclet"(token %[[CLPAD]]) ]
6464
// CHECK-NEXT: cleanupret from %[[CLPAD]] unwind label
6565

6666
// CHECK-LPAD: @_Z1fv(
@@ -76,7 +76,7 @@ coro_t f() {
7676
// CHECK-LPAD: to label %{{.+}} unwind label %[[UNWINDBB:.+]]
7777

7878
// CHECK-LPAD: [[UNWINDBB]]:
79-
// CHECK-LPAD: %[[I1RESUME:.+]] = call i1 (ptr, i1, ...) @llvm.coro.end(ptr null, i1 true)
79+
// CHECK-LPAD: %[[I1RESUME:.+]] = call i1 @llvm.coro.end(ptr null, i1 true, token none)
8080
// CHECK-LPAD: br i1 %[[I1RESUME]], label %[[EHRESUME:.+]], label
8181
// CHECK-LPAD: [[EHRESUME]]:
8282
// CHECK-LPAD-NEXT: %[[exn:.+]] = load ptr, ptr %exn.slot, align 8

‎clang/test/CodeGenCoroutines/coro-lambda.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,4 +55,4 @@ void f() {
5555
// CHECK: alloca %"struct.Task::promise_type"
5656
// CHECK: call token @llvm.coro.id(
5757
// CHECK: call i8 @llvm.coro.suspend(
58-
// CHECK: call i1 (ptr, i1, ...) @llvm.coro.end(
58+
// CHECK: call i1 @llvm.coro.end(

‎llvm/docs/Coroutines.rst

Lines changed: 56 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,7 @@ The LLVM IR for this coroutine looks like this:
303303
call void @free(ptr %mem)
304304
br label %suspend
305305
suspend:
306-
%unused = call i1 (ptr, i1, ...) @llvm.coro.end(ptr %hdl, i1 false)
306+
%unused = call i1 @llvm.coro.end(ptr %hdl, i1 false, token none)
307307
ret ptr %hdl
308308
}
309309
@@ -630,7 +630,7 @@ store the current value produced by a coroutine.
630630
call void @free(ptr %mem)
631631
br label %suspend
632632
suspend:
633-
%unused = call i1 (ptr, i1, ...) @llvm.coro.end(ptr %hdl, i1 false)
633+
%unused = call i1 @llvm.coro.end(ptr %hdl, i1 false, token none)
634634
ret ptr %hdl
635635
}
636636
@@ -1326,7 +1326,7 @@ A frontend should emit function attribute `presplitcoroutine` for the coroutine.
13261326
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
13271327
::
13281328

1329-
declare i1 @llvm.coro.end(ptr <handle>, i1 <unwind>, ...)
1329+
declare i1 @llvm.coro.end(ptr <handle>, i1 <unwind>, token <result.token>)
13301330

13311331
Overview:
13321332
"""""""""
@@ -1347,20 +1347,11 @@ The second argument should be `true` if this coro.end is in the block that is
13471347
part of the unwind sequence leaving the coroutine body due to an exception and
13481348
`false` otherwise.
13491349

1350-
Other arguments can only be specified for unique-suspend returned-continuation
1351-
coroutines where they will be normal returns of a coroutine continuation
1352-
function. The number of arguments must match the return type of the continuation
1353-
function:
1350+
Non-trivial (non-none) token argument can only be specified for unique-suspend
1351+
returned-continuation coroutines where it must be a token value produced by
1352+
'``llvm.coro.end.results``' intrinsic.
13541353

1355-
- if the return type of the continuation function is ``void`` there must be no
1356-
extra argumets
1357-
1358-
- if the return type of the continuation function is a ``struct``, the arguments
1359-
will be element types of that ``struct`` in order;
1360-
1361-
- otherwise, it is just the return value of the continuation function.
1362-
1363-
No extra arguments are allowed for coro.end calls in unwind sections
1354+
Only none token is allowed for coro.end calls in unwind sections
13641355

13651356
Semantics:
13661357
""""""""""
@@ -1393,7 +1384,7 @@ For landingpad based exception model, it is expected that frontend uses the
13931384
.. code-block:: llvm
13941385
13951386
ehcleanup:
1396-
%InResumePart = call i1 (ptr, i1, ...) @llvm.coro.end(ptr null, i1 true)
1387+
%InResumePart = call i1 @llvm.coro.end(ptr null, i1 true, token none)
13971388
br i1 %InResumePart, label %eh.resume, label %cleanup.cont
13981389
13991390
cleanup.cont:
@@ -1418,7 +1409,7 @@ referring to an enclosing cleanuppad as follows:
14181409
14191410
ehcleanup:
14201411
%tok = cleanuppad within none []
1421-
%unused = call i1 (ptr, i1, ...) @llvm.coro.end(ptr null, i1 true) [ "funclet"(token %tok) ]
1412+
%unused = call i1 @llvm.coro.end(ptr null, i1 true, token none) [ "funclet"(token %tok) ]
14221413
cleanupret from %tok unwind label %RestOfTheCleanup
14231414
14241415
The `CoroSplit` pass, if the funclet bundle is present, will insert
@@ -1443,6 +1434,53 @@ The following table summarizes the handling of `coro.end`_ intrinsic.
14431434
| | Landingpad | mark coroutine as done | mark coroutine done |
14441435
+------------+-------------+------------------------+---------------------------------+
14451436

1437+
.. _coro.end.results:
1438+
1439+
'llvm.coro.end.results' Intrinsic
1440+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1441+
::
1442+
1443+
declare token @llvm.coro.end.results(...)
1444+
1445+
Overview:
1446+
"""""""""
1447+
1448+
The '``llvm.coro.end.results``' intrinsic captures values to be returned from
1449+
unique-suspend returned-continuation coroutines.
1450+
1451+
Arguments:
1452+
""""""""""
1453+
1454+
The number of arguments must match the return type of the continuation function:
1455+
1456+
- if the return type of the continuation function is ``void`` there must be no
1457+
arguments
1458+
1459+
- if the return type of the continuation function is a ``struct``, the arguments
1460+
will be of element types of that ``struct`` in order;
1461+
1462+
- otherwise, it is just the return value of the continuation function.
1463+
1464+
.. code-block:: llvm
1465+
1466+
define {ptr, ptr} @g(ptr %buffer, ptr %ptr, i8 %val) presplitcoroutine {
1467+
entry:
1468+
%id = call token @llvm.coro.id.retcon.once(i32 8, i32 8, ptr %buffer,
1469+
ptr @prototype,
1470+
ptr @allocate, ptr @deallocate)
1471+
%hdl = call ptr @llvm.coro.begin(token %id, ptr null)
1472+
1473+
...
1474+
1475+
cleanup:
1476+
%tok = call token (...) @llvm.coro.end.results(i8 %val)
1477+
call i1 @llvm.coro.end(ptr %hdl, i1 0, token %tok)
1478+
unreachable
1479+
1480+
...
1481+
1482+
declare i8 @prototype(ptr, i1 zeroext)
1483+
14461484
14471485
'llvm.coro.end.async' Intrinsic
14481486
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

‎llvm/include/llvm/IR/Intrinsics.td

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1643,7 +1643,8 @@ def int_coro_free : Intrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_ptr_ty],
16431643
[IntrReadMem, IntrArgMemOnly,
16441644
ReadOnly<ArgIndex<1>>,
16451645
NoCapture<ArgIndex<1>>]>;
1646-
def int_coro_end : Intrinsic<[llvm_i1_ty], [llvm_ptr_ty, llvm_i1_ty, llvm_vararg_ty], []>;
1646+
def int_coro_end : Intrinsic<[llvm_i1_ty], [llvm_ptr_ty, llvm_i1_ty, llvm_token_ty], []>;
1647+
def int_coro_end_results : Intrinsic<[llvm_token_ty], [llvm_vararg_ty]>;
16471648
def int_coro_end_async
16481649
: Intrinsic<[llvm_i1_ty], [llvm_ptr_ty, llvm_i1_ty, llvm_vararg_ty], []>;
16491650

‎llvm/lib/IR/AutoUpgrade.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -951,7 +951,7 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
951951
F->arg_begin()->getType());
952952
return true;
953953
}
954-
if (Name.equals("coro.end") && !F->isVarArg()) {
954+
if (Name.equals("coro.end") && F->arg_size() == 2) {
955955
rename(F);
956956
NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::coro_end);
957957
return true;
@@ -4214,7 +4214,8 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
42144214
}
42154215

42164216
case Intrinsic::coro_end: {
4217-
SmallVector<Value *, 2> Args(CI->args());
4217+
SmallVector<Value *, 3> Args(CI->args());
4218+
Args.push_back(ConstantTokenNone::get(CI->getContext()));
42184219
NewCall = Builder.CreateCall(NewFn, Args);
42194220
break;
42204221
}

‎llvm/lib/Transforms/Coroutines/CoroFrame.cpp

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3046,8 +3046,7 @@ void coro::buildCoroutineFrame(
30463046
// Collect the spills for arguments and other not-materializable values.
30473047
for (Argument &A : F.args())
30483048
for (User *U : A.users())
3049-
if (Checker.isDefinitionAcrossSuspend(A, U) ||
3050-
isa<CoroEndInst>(U))
3049+
if (Checker.isDefinitionAcrossSuspend(A, U))
30513050
FrameData.Spills[&A].push_back(cast<Instruction>(U));
30523051

30533052
const DominatorTree DT(F);

‎llvm/lib/Transforms/Coroutines/CoroInstr.h

Lines changed: 39 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -611,15 +611,53 @@ class LLVM_LIBRARY_VISIBILITY CoroAlignInst : public IntrinsicInst {
611611
}
612612
};
613613

614+
/// This represents the llvm.end.results instruction.
615+
class LLVM_LIBRARY_VISIBILITY CoroEndResults : public IntrinsicInst {
616+
public:
617+
op_iterator retval_begin() { return arg_begin(); }
618+
const_op_iterator retval_begin() const { return arg_begin(); }
619+
620+
op_iterator retval_end() { return arg_end(); }
621+
const_op_iterator retval_end() const { return arg_end(); }
622+
623+
iterator_range<op_iterator> return_values() {
624+
return make_range(retval_begin(), retval_end());
625+
}
626+
iterator_range<const_op_iterator> return_values() const {
627+
return make_range(retval_begin(), retval_end());
628+
}
629+
630+
unsigned numReturns() const {
631+
return std::distance(retval_begin(), retval_end());
632+
}
633+
634+
// Methods to support type inquiry through isa, cast, and dyn_cast:
635+
static bool classof(const IntrinsicInst *I) {
636+
return I->getIntrinsicID() == Intrinsic::coro_end_results;
637+
}
638+
static bool classof(const Value *V) {
639+
return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
640+
}
641+
};
642+
614643
class LLVM_LIBRARY_VISIBILITY AnyCoroEndInst : public IntrinsicInst {
615-
enum { FrameArg, UnwindArg };
644+
enum { FrameArg, UnwindArg, TokenArg };
616645

617646
public:
618647
bool isFallthrough() const { return !isUnwind(); }
619648
bool isUnwind() const {
620649
return cast<Constant>(getArgOperand(UnwindArg))->isOneValue();
621650
}
622651

652+
bool hasResults() const {
653+
return !isa<ConstantTokenNone>(getArgOperand(TokenArg));
654+
}
655+
656+
CoroEndResults *getResults() const {
657+
assert(hasResults());
658+
return cast<CoroEndResults>(getArgOperand(TokenArg));
659+
}
660+
623661
// Methods to support type inquiry through isa, cast, and dyn_cast:
624662
static bool classof(const IntrinsicInst *I) {
625663
auto ID = I->getIntrinsicID();
@@ -633,23 +671,6 @@ class LLVM_LIBRARY_VISIBILITY AnyCoroEndInst : public IntrinsicInst {
633671
/// This represents the llvm.coro.end instruction.
634672
class LLVM_LIBRARY_VISIBILITY CoroEndInst : public AnyCoroEndInst {
635673
public:
636-
op_iterator retval_begin() { return std::next(arg_begin(), 2); }
637-
const_op_iterator retval_begin() const { return std::next(arg_begin(), 2); }
638-
639-
op_iterator retval_end() { return arg_end(); }
640-
const_op_iterator retval_end() const { return arg_end(); }
641-
642-
iterator_range<op_iterator> return_values() {
643-
return make_range(retval_begin(), retval_end());
644-
}
645-
iterator_range<const_op_iterator> return_values() const {
646-
return make_range(retval_begin(), retval_end());
647-
}
648-
649-
unsigned numReturns() const {
650-
return std::distance(retval_begin(), retval_end());
651-
}
652-
653674
// Methods to support type inquiry through isa, cast, and dyn_cast:
654675
static bool classof(const IntrinsicInst *I) {
655676
return I->getIntrinsicID() == Intrinsic::coro_end;

‎llvm/lib/Transforms/Coroutines/CoroSplit.cpp

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ static void replaceFallthroughCoroEnd(AnyCoroEndInst *End,
234234
switch (Shape.ABI) {
235235
// The cloned functions in switch-lowering always return void.
236236
case coro::ABI::Switch:
237-
assert(cast<CoroEndInst>(End)->numReturns() == 0 &&
237+
assert(!cast<CoroEndInst>(End)->hasResults() &&
238238
"switch coroutine should not return any values");
239239
// coro.end doesn't immediately end the coroutine in the main function
240240
// in this lowering, because we need to deallocate the coroutine.
@@ -257,30 +257,40 @@ static void replaceFallthroughCoroEnd(AnyCoroEndInst *End,
257257
maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
258258
auto *CoroEnd = cast<CoroEndInst>(End);
259259
auto *RetTy = Shape.getResumeFunctionType()->getReturnType();
260-
unsigned NumReturns = CoroEnd->numReturns();
260+
261+
if (!CoroEnd->hasResults()) {
262+
assert(RetTy->isVoidTy());
263+
Builder.CreateRetVoid();
264+
break;
265+
}
266+
267+
auto *CoroResults = CoroEnd->getResults();
268+
unsigned NumReturns = CoroResults->numReturns();
261269

262270
if (auto *RetStructTy = dyn_cast<StructType>(RetTy)) {
263271
assert(RetStructTy->getNumElements() == NumReturns &&
264272
"numbers of returns should match resume function singature");
265273
Value *ReturnValue = UndefValue::get(RetStructTy);
266274
unsigned Idx = 0;
267-
for (Value *RetValEl : CoroEnd->return_values())
275+
for (Value *RetValEl : CoroResults->return_values())
268276
ReturnValue = Builder.CreateInsertValue(ReturnValue, RetValEl, Idx++);
269277
Builder.CreateRet(ReturnValue);
270278
} else if (NumReturns == 0) {
271279
assert(RetTy->isVoidTy());
272280
Builder.CreateRetVoid();
273281
} else {
274282
assert(NumReturns == 1);
275-
Builder.CreateRet(*CoroEnd->retval_begin());
283+
Builder.CreateRet(*CoroResults->retval_begin());
276284
}
285+
CoroResults->replaceAllUsesWith(ConstantTokenNone::get(CoroResults->getContext()));
286+
CoroResults->eraseFromParent();
277287
break;
278288
}
279289

280290
// In non-unique continuation lowering, we signal completion by returning
281291
// a null continuation.
282292
case coro::ABI::Retcon: {
283-
assert(cast<CoroEndInst>(End)->numReturns() == 0 &&
293+
assert(!cast<CoroEndInst>(End)->hasResults() &&
284294
"retcon coroutine should not return any values");
285295
maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
286296
auto RetTy = Shape.getResumeFunctionType()->getReturnType();

‎llvm/test/Assembler/auto_upgrade_intrinsics.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ entry:
5050
declare i1 @llvm.coro.end(ptr, i1)
5151
define void @test.coro.end(ptr %ptr) {
5252
; CHECK-LABEL: @test.coro.end(
53-
; CHECK: call i1 (ptr, i1, ...) @llvm.coro.end(ptr %ptr, i1 false)
53+
; CHECK: call i1 @llvm.coro.end(ptr %ptr, i1 false, token none)
5454
call i1 @llvm.coro.end(ptr %ptr, i1 false)
5555
ret void
5656
}

‎llvm/test/Transforms/Coroutines/ArgAddr.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ coro_Cleanup:
4545
br label %coro_Suspend
4646

4747
coro_Suspend:
48-
call i1 (ptr, i1, ...) @llvm.coro.end(ptr null, i1 false)
48+
call i1 @llvm.coro.end(ptr null, i1 false, token none)
4949
ret ptr %1
5050
}
5151

@@ -69,7 +69,7 @@ declare i32 @llvm.coro.size.i32()
6969
declare ptr @llvm.coro.begin(token, ptr)
7070
declare i8 @llvm.coro.suspend(token, i1)
7171
declare ptr @llvm.coro.free(token, ptr)
72-
declare i1 @llvm.coro.end(ptr, i1, ...)
72+
declare i1 @llvm.coro.end(ptr, i1, token)
7373

7474
declare void @llvm.coro.resume(ptr)
7575
declare void @llvm.coro.destroy(ptr)

0 commit comments

Comments
 (0)