diff --git a/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h b/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h index ea90fa89e04ea..cd2ae61334d0f 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h +++ b/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h @@ -19,6 +19,8 @@ #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/CFG.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/Support/BranchProbability.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -82,6 +84,38 @@ template class CFGMST { return true; } + void handleCoroSuspendEdge(Edge *E) { + // We must not add instrumentation to the BB representing the + // "suspend" path, else CoroSplit won't be able to lower + // llvm.coro.suspend to a tail call. We do want profiling info for + // the other branches (resume/destroy). So we do 2 things: + // 1. we prefer instrumenting those other edges by setting the weight + // of the "suspend" edge to max, and + // 2. we mark the edge as "Removed" to guarantee it is not considered + // for instrumentation. That could technically happen: + // (from test/Transforms/Coroutines/coro-split-musttail.ll) + // + // %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + // switch i8 %suspend, label %exit [ + // i8 0, label %await.ready + // i8 1, label %exit + // ] + const BasicBlock *EdgeTarget = E->DestBB; + if (!EdgeTarget) + return; + assert(E->SrcBB); + const Function *F = EdgeTarget->getParent(); + if (!F->isPresplitCoroutine()) + return; + + const Instruction *TI = E->SrcBB->getTerminator(); + if (auto *SWInst = dyn_cast(TI)) + if (auto *Intrinsic = dyn_cast(SWInst->getCondition())) + if (Intrinsic->getIntrinsicID() == Intrinsic::coro_suspend && + SWInst->getDefaultDest() == EdgeTarget) + E->Removed = true; + } + // Traverse the CFG using a stack. Find all the edges and assign the weight. // Edges with large weight will be put into MST first so they are less likely // to be instrumented. @@ -133,6 +167,7 @@ template class CFGMST { Weight++; auto *E = &addEdge(&BB, TargetBB, Weight); E->IsCritical = Critical; + handleCoroSuspendEdge(E); LLVM_DEBUG(dbgs() << " Edge: from " << BB.getName() << " to " << TargetBB->getName() << " w=" << Weight << "\n"); diff --git a/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail.ll b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail.ll new file mode 100644 index 0000000000000..a7321833d7484 --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail.ll @@ -0,0 +1,63 @@ +; Tests that instrumentation doesn't interfere with lowering (coro-split). +; It should convert coro.resume followed by a suspend to a musttail call. + +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s + +define void @f() #0 { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %alloc = call ptr @malloc(i64 16) #3 + %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc) + + %save = call token @llvm.coro.save(ptr null) + %addr1 = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) + call fastcc void %addr1(ptr null) + + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + switch i8 %suspend, label %exit [ + i8 0, label %await.ready + i8 1, label %exit + ] +await.ready: + %save2 = call token @llvm.coro.save(ptr null) + %addr2 = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) + call fastcc void %addr2(ptr null) + + %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false) + switch i8 %suspend2, label %exit [ + i8 0, label %exit + i8 1, label %exit + ] +exit: + call i1 @llvm.coro.end(ptr null, i1 false, token none) + ret void +} + +; Verify that in the initial function resume is not marked with musttail. +; CHECK-LABEL: @f( +; CHECK: %[[addr1:.+]] = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) +; CHECK-NOT: musttail call fastcc void %[[addr1]](ptr null) + +; Verify that in the resume part resume call is marked with musttail. +; CHECK-LABEL: @f.resume( +; CHECK: %[[addr2:.+]] = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) +; CHECK: call void @llvm.instrprof +; CHECK-NEXT: musttail call fastcc void %[[addr2]](ptr null) +; CHECK-NEXT: ret void + +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1 +declare i1 @llvm.coro.alloc(token) #2 +declare i64 @llvm.coro.size.i64() #3 +declare ptr @llvm.coro.begin(token, ptr writeonly) #2 +declare token @llvm.coro.save(ptr) #2 +declare ptr @llvm.coro.frame() #3 +declare i8 @llvm.coro.suspend(token, i1) #2 +declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1 +declare i1 @llvm.coro.end(ptr, i1, token) #2 +declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1 +declare ptr @malloc(i64) + +attributes #0 = { presplitcoroutine } +attributes #1 = { argmemonly nounwind readonly } +attributes #2 = { nounwind } +attributes #3 = { nounwind readnone } diff --git a/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail1.ll b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail1.ll new file mode 100644 index 0000000000000..6098dee9a5803 --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail1.ll @@ -0,0 +1,97 @@ +; Tests that instrumentation doesn't interfere with lowering (coro-split). +; It should convert coro.resume followed by a suspend to a musttail call. + +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s + +define void @f() #0 { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %alloc = call ptr @malloc(i64 16) #3 + %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc) + + %save = call token @llvm.coro.save(ptr null) + %addr1 = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) + call fastcc void %addr1(ptr null) + + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + switch i8 %suspend, label %exit [ + i8 0, label %await.suspend + i8 1, label %exit + ] +await.suspend: + %save2 = call token @llvm.coro.save(ptr null) + %br0 = call i8 @switch_result() + switch i8 %br0, label %unreach [ + i8 0, label %await.resume3 + i8 1, label %await.resume1 + i8 2, label %await.resume2 + ] +await.resume1: + %hdl = call ptr @g() + %addr2 = call ptr @llvm.coro.subfn.addr(ptr %hdl, i8 0) + call fastcc void %addr2(ptr %hdl) + br label %final.suspend +await.resume2: + %hdl2 = call ptr @h() + %addr3 = call ptr @llvm.coro.subfn.addr(ptr %hdl2, i8 0) + call fastcc void %addr3(ptr %hdl2) + br label %final.suspend +await.resume3: + %addr4 = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) + call fastcc void %addr4(ptr null) + br label %final.suspend +final.suspend: + %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false) + switch i8 %suspend2, label %exit [ + i8 0, label %pre.exit + i8 1, label %exit + ] +pre.exit: + br label %exit +exit: + call i1 @llvm.coro.end(ptr null, i1 false, token none) + ret void +unreach: + unreachable +} + +; Verify that in the initial function resume is not marked with musttail. +; CHECK-LABEL: @f( +; CHECK: %[[addr1:.+]] = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) +; CHECK-NOT: musttail call fastcc void %[[addr1]](ptr null) + +; Verify that in the resume part resume call is marked with musttail. +; CHECK-LABEL: @f.resume( +; CHECK: %[[hdl:.+]] = call ptr @g() +; CHECK-NEXT: %[[addr2:.+]] = call ptr @llvm.coro.subfn.addr(ptr %[[hdl]], i8 0) +; CHECK: musttail call fastcc void %[[addr2]](ptr %[[hdl]]) +; CHECK-NEXT: ret void +; CHECK: %[[hdl2:.+]] = call ptr @h() +; CHECK-NEXT: %[[addr3:.+]] = call ptr @llvm.coro.subfn.addr(ptr %[[hdl2]], i8 0) +; CHECK: musttail call fastcc void %[[addr3]](ptr %[[hdl2]]) +; CHECK-NEXT: ret void +; CHECK: %[[addr4:.+]] = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) +; CHECK: musttail call fastcc void %[[addr4]](ptr null) +; CHECK-NEXT: ret void + + + +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1 +declare i1 @llvm.coro.alloc(token) #2 +declare i64 @llvm.coro.size.i64() #3 +declare ptr @llvm.coro.begin(token, ptr writeonly) #2 +declare token @llvm.coro.save(ptr) #2 +declare ptr @llvm.coro.frame() #3 +declare i8 @llvm.coro.suspend(token, i1) #2 +declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1 +declare i1 @llvm.coro.end(ptr, i1, token) #2 +declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1 +declare ptr @malloc(i64) +declare i8 @switch_result() +declare ptr @g() +declare ptr @h() + +attributes #0 = { presplitcoroutine } +attributes #1 = { argmemonly nounwind readonly } +attributes #2 = { nounwind } +attributes #3 = { nounwind readnone } diff --git a/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail10.ll b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail10.ll new file mode 100644 index 0000000000000..f43b10ebf42e5 --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail10.ll @@ -0,0 +1,55 @@ +; Tests that instrumentation doesn't interfere with lowering (coro-split). +; It should convert coro.resume followed by a suspend to a musttail call. + +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s + +target triple = "wasm64-unknown-unknown" + +define void @f() #0 { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %alloc = call ptr @malloc(i64 16) #3 + %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc) + + %save = call token @llvm.coro.save(ptr null) + %addr1 = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) + call fastcc void %addr1(ptr null) + + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + switch i8 %suspend, label %exit [ + i8 0, label %await.ready + i8 1, label %exit + ] +await.ready: + %save2 = call token @llvm.coro.save(ptr null) + %addr2 = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) + call fastcc void %addr2(ptr null) + + %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false) + switch i8 %suspend2, label %exit [ + i8 0, label %exit + i8 1, label %exit + ] +exit: + call i1 @llvm.coro.end(ptr null, i1 false, token none) + ret void +} + +; CHECK: musttail call + +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1 +declare i1 @llvm.coro.alloc(token) #2 +declare i64 @llvm.coro.size.i64() #3 +declare ptr @llvm.coro.begin(token, ptr writeonly) #2 +declare token @llvm.coro.save(ptr) #2 +declare ptr @llvm.coro.frame() #3 +declare i8 @llvm.coro.suspend(token, i1) #2 +declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1 +declare i1 @llvm.coro.end(ptr, i1, token) #2 +declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1 +declare ptr @malloc(i64) + +attributes #0 = { presplitcoroutine "target-features"="+tail-call" } +attributes #1 = { argmemonly nounwind readonly } +attributes #2 = { nounwind } +attributes #3 = { nounwind readnone } diff --git a/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail11.ll b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail11.ll new file mode 100644 index 0000000000000..fc5bb9a1b20b3 --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail11.ll @@ -0,0 +1,55 @@ +; Tests that instrumentation doesn't interfere with lowering (coro-split). +; It should convert coro.resume followed by a suspend to a musttail call. + +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s + +target triple = "wasm32-unknown-unknown" + +define void @f() #0 { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %alloc = call ptr @malloc(i64 16) #3 + %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc) + + %save = call token @llvm.coro.save(ptr null) + %addr1 = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) + call fastcc void %addr1(ptr null) + + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + switch i8 %suspend, label %exit [ + i8 0, label %await.ready + i8 1, label %exit + ] +await.ready: + %save2 = call token @llvm.coro.save(ptr null) + %addr2 = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) + call fastcc void %addr2(ptr null) + + %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false) + switch i8 %suspend2, label %exit [ + i8 0, label %exit + i8 1, label %exit + ] +exit: + call i1 @llvm.coro.end(ptr null, i1 false, token none) + ret void +} + +; CHECK: musttail call + +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1 +declare i1 @llvm.coro.alloc(token) #2 +declare i64 @llvm.coro.size.i64() #3 +declare ptr @llvm.coro.begin(token, ptr writeonly) #2 +declare token @llvm.coro.save(ptr) #2 +declare ptr @llvm.coro.frame() #3 +declare i8 @llvm.coro.suspend(token, i1) #2 +declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1 +declare i1 @llvm.coro.end(ptr, i1, token) #2 +declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1 +declare ptr @malloc(i64) + +attributes #0 = { presplitcoroutine "target-features"="+tail-call" } +attributes #1 = { argmemonly nounwind readonly } +attributes #2 = { nounwind } +attributes #3 = { nounwind readnone } diff --git a/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail12.ll b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail12.ll new file mode 100644 index 0000000000000..634d0106a2e6a --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail12.ll @@ -0,0 +1,85 @@ +; Tests that instrumentation doesn't interfere with lowering (coro-split). +; It should convert coro.resume followed by a suspend to a musttail call. + +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s + +declare void @fakeresume1(ptr) +declare void @print() + +define void @f(i1 %cond) #0 { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %alloc = call ptr @malloc(i64 16) #3 + %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc) + + %save = call token @llvm.coro.save(ptr null) + + %init_suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + switch i8 %init_suspend, label %coro.end [ + i8 0, label %await.ready + i8 1, label %coro.end + ] +await.ready: + %save2 = call token @llvm.coro.save(ptr null) + br i1 %cond, label %then, label %else + +then: + call fastcc void @fakeresume1(ptr align 8 null) + br label %merge + +else: + br label %merge + +merge: + %v0 = phi i1 [0, %then], [1, %else] + br label %compare + +compare: + %cond.cmp = icmp eq i1 %v0, 0 + br i1 %cond.cmp, label %ready, label %prepare + +prepare: + call void @print() + br label %ready + +ready: + %suspend = call i8 @llvm.coro.suspend(token %save2, i1 true) + %switch = icmp ult i8 %suspend, 2 + br i1 %switch, label %cleanup, label %coro.end + +cleanup: + %free.handle = call ptr @llvm.coro.free(token %id, ptr %vFrame) + %.not = icmp eq ptr %free.handle, null + br i1 %.not, label %coro.end, label %coro.free + +coro.free: + call void @delete(ptr nonnull %free.handle) #2 + br label %coro.end + +coro.end: + call i1 @llvm.coro.end(ptr null, i1 false, token none) + ret void +} + +; CHECK-LABEL: @f.resume( +; CHECK-NOT: } +; CHECK: call void @print() + + +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1 +declare i1 @llvm.coro.alloc(token) #2 +declare i64 @llvm.coro.size.i64() #3 +declare ptr @llvm.coro.begin(token, ptr writeonly) #2 +declare token @llvm.coro.save(ptr) #2 +declare ptr @llvm.coro.frame() #3 +declare i8 @llvm.coro.suspend(token, i1) #2 +declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1 +declare i1 @llvm.coro.end(ptr, i1, token) #2 +declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1 +declare ptr @malloc(i64) +declare void @delete(ptr nonnull) #2 + +attributes #0 = { presplitcoroutine } +attributes #1 = { argmemonly nounwind readonly } +attributes #2 = { nounwind } +attributes #3 = { nounwind readnone } diff --git a/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail13.ll b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail13.ll new file mode 100644 index 0000000000000..2f9a14c901071 --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail13.ll @@ -0,0 +1,76 @@ +; Tests that instrumentation doesn't interfere with lowering (coro-split). +; It should convert coro.resume followed by a suspend to a musttail call. + +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s + +declare void @fakeresume1(ptr) +declare void @may_throw(ptr) +declare void @print() + +define void @f(i1 %cond) #0 personality i32 3 { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %alloc = call ptr @malloc(i64 16) #3 + %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc) + + %save = call token @llvm.coro.save(ptr null) + + %init_suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + switch i8 %init_suspend, label %coro.end [ + i8 0, label %await.ready + i8 1, label %coro.end + ] +await.ready: + call fastcc void @fakeresume1(ptr align 8 null) + invoke void @may_throw(ptr null) + to label %ready unwind label %lpad + +ready: + %save2 = call token @llvm.coro.save(ptr null) + %suspend = call i8 @llvm.coro.suspend(token %save2, i1 true) + %switch = icmp ult i8 %suspend, 2 + br i1 %switch, label %cleanup, label %coro.end + +cleanup: + %free.handle = call ptr @llvm.coro.free(token %id, ptr %vFrame) + %.not = icmp eq ptr %free.handle, null + br i1 %.not, label %coro.end, label %coro.free + +lpad: + %lpval = landingpad { ptr, i32 } + cleanup + + %need.resume = call i1 @llvm.coro.end(ptr null, i1 true, token none) + resume { ptr, i32 } %lpval + +coro.free: + call void @delete(ptr nonnull %free.handle) #2 + br label %coro.end + +coro.end: + call i1 @llvm.coro.end(ptr null, i1 false, token none) + ret void +} + +; CHECK-LABEL: @f.resume( +; CHECK-NOT: musttail call fastcc void @fakeresume1( +; CHECK: } + + +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1 +declare i1 @llvm.coro.alloc(token) #2 +declare i64 @llvm.coro.size.i64() #3 +declare ptr @llvm.coro.begin(token, ptr writeonly) #2 +declare token @llvm.coro.save(ptr) #2 +declare ptr @llvm.coro.frame() #3 +declare i8 @llvm.coro.suspend(token, i1) #2 +declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1 +declare i1 @llvm.coro.end(ptr, i1, token) #2 +declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1 +declare ptr @malloc(i64) +declare void @delete(ptr nonnull) #2 + +attributes #0 = { presplitcoroutine } +attributes #1 = { argmemonly nounwind readonly } +attributes #2 = { nounwind } +attributes #3 = { nounwind readnone } diff --git a/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail2.ll b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail2.ll new file mode 100644 index 0000000000000..61b61a200e704 --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail2.ll @@ -0,0 +1,68 @@ +; Tests that instrumentation doesn't interfere with lowering (coro-split). +; It should convert coro.resume followed by a suspend to a musttail call. + +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s + +define void @fakeresume1(ptr) { +entry: + ret void; +} + +define void @fakeresume2(ptr align 8) { +entry: + ret void; +} + +define void @g() #0 { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %alloc = call ptr @malloc(i64 16) #3 + %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc) + + %save = call token @llvm.coro.save(ptr null) + call fastcc void @fakeresume1(ptr null) + + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + switch i8 %suspend, label %exit [ + i8 0, label %await.ready + i8 1, label %exit + ] +await.ready: + %save2 = call token @llvm.coro.save(ptr null) + call fastcc void @fakeresume2(ptr align 8 null) + + %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false) + switch i8 %suspend2, label %exit [ + i8 0, label %exit + i8 1, label %exit + ] +exit: + call i1 @llvm.coro.end(ptr null, i1 false, token none) + ret void +} + +; Verify that in the initial function resume is not marked with musttail. +; CHECK-LABEL: @g( +; CHECK-NOT: musttail call fastcc void @fakeresume1(ptr null) + +; Verify that in the resume part resume call is marked with musttail. +; CHECK-LABEL: @g.resume( +; CHECK: musttail call fastcc void @fakeresume2(ptr align 8 null) +; CHECK-NEXT: ret void + +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1 +declare i1 @llvm.coro.alloc(token) #2 +declare i64 @llvm.coro.size.i64() #3 +declare ptr @llvm.coro.begin(token, ptr writeonly) #2 +declare token @llvm.coro.save(ptr) #2 +declare ptr @llvm.coro.frame() #3 +declare i8 @llvm.coro.suspend(token, i1) #2 +declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1 +declare i1 @llvm.coro.end(ptr, i1, token) #2 +declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1 +declare ptr @malloc(i64) + +attributes #0 = { presplitcoroutine } +attributes #1 = { argmemonly nounwind readonly } +attributes #2 = { nounwind } +attributes #3 = { nounwind readnone } diff --git a/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail3.ll b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail3.ll new file mode 100644 index 0000000000000..82176b8085e6c --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail3.ll @@ -0,0 +1,91 @@ +; Tests that instrumentation doesn't interfere with lowering (coro-split). +; It should convert coro.resume followed by a suspend to a musttail call. + +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s + +define void @f() #0 { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %alloc = call ptr @malloc(i64 16) #3 + %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc) + + %save = call token @llvm.coro.save(ptr null) + %addr1 = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) + call fastcc void %addr1(ptr null) + + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + %cmp = icmp eq i8 %suspend, 0 + br i1 %cmp, label %await.suspend, label %exit +await.suspend: + %save2 = call token @llvm.coro.save(ptr null) + %br0 = call i8 @switch_result() + switch i8 %br0, label %unreach [ + i8 0, label %await.resume3 + i8 1, label %await.resume1 + i8 2, label %await.resume2 + ] +await.resume1: + %hdl = call ptr @g() + %addr2 = call ptr @llvm.coro.subfn.addr(ptr %hdl, i8 0) + call fastcc void %addr2(ptr %hdl) + br label %final.suspend +await.resume2: + %hdl2 = call ptr @h() + %addr3 = call ptr @llvm.coro.subfn.addr(ptr %hdl2, i8 0) + call fastcc void %addr3(ptr %hdl2) + br label %final.suspend +await.resume3: + %addr4 = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) + call fastcc void %addr4(ptr null) + br label %final.suspend +final.suspend: + %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false) + %cmp2 = icmp eq i8 %suspend2, 0 + br i1 %cmp2, label %pre.exit, label %exit +pre.exit: + br label %exit +exit: + call i1 @llvm.coro.end(ptr null, i1 false, token none) + ret void +unreach: + unreachable +} + +; Verify that in the initial function resume is not marked with musttail. +; CHECK-LABEL: @f( +; CHECK: %[[addr1:.+]] = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) +; CHECK-NOT: musttail call fastcc void %[[addr1]](ptr null) + +; Verify that in the resume part resume call is marked with musttail. +; CHECK-LABEL: @f.resume( +; CHECK: %[[hdl:.+]] = call ptr @g() +; CHECK-NEXT: %[[addr2:.+]] = call ptr @llvm.coro.subfn.addr(ptr %[[hdl]], i8 0) +; CHECK: musttail call fastcc void %[[addr2]](ptr %[[hdl]]) +; CHECK-NEXT: ret void +; CHECK: %[[hdl2:.+]] = call ptr @h() +; CHECK-NEXT: %[[addr3:.+]] = call ptr @llvm.coro.subfn.addr(ptr %[[hdl2]], i8 0) +; CHECK: musttail call fastcc void %[[addr3]](ptr %[[hdl2]]) +; CHECK-NEXT: ret void +; CHECK: %[[addr4:.+]] = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) +; CHECK: musttail call fastcc void %[[addr4]](ptr null) +; CHECK-NEXT: ret void + +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1 +declare i1 @llvm.coro.alloc(token) #2 +declare i64 @llvm.coro.size.i64() #3 +declare ptr @llvm.coro.begin(token, ptr writeonly) #2 +declare token @llvm.coro.save(ptr) #2 +declare ptr @llvm.coro.frame() #3 +declare i8 @llvm.coro.suspend(token, i1) #2 +declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1 +declare i1 @llvm.coro.end(ptr, i1, token) #2 +declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1 +declare ptr @malloc(i64) +declare i8 @switch_result() +declare ptr @g() +declare ptr @h() + +attributes #0 = { presplitcoroutine } +attributes #1 = { argmemonly nounwind readonly } +attributes #2 = { nounwind } +attributes #3 = { nounwind readnone } diff --git a/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail4.ll b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail4.ll new file mode 100644 index 0000000000000..be70fc4b51f1d --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail4.ll @@ -0,0 +1,66 @@ +; Tests that instrumentation doesn't interfere with lowering (coro-split). +; It should convert coro.resume followed by a suspend to a musttail call. + +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s + +define void @fakeresume1(ptr) { +entry: + ret void; +} + +define void @f() #0 { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %alloc = call ptr @malloc(i64 16) #3 + %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc) + + %save = call token @llvm.coro.save(ptr null) + + %init_suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + switch i8 %init_suspend, label %coro.end [ + i8 0, label %await.ready + i8 1, label %coro.end + ] +await.ready: + %save2 = call token @llvm.coro.save(ptr null) + + call fastcc void @fakeresume1(ptr align 8 null) + %suspend = call i8 @llvm.coro.suspend(token %save2, i1 true) + %switch = icmp ult i8 %suspend, 2 + br i1 %switch, label %cleanup, label %coro.end + +cleanup: + %free.handle = call ptr @llvm.coro.free(token %id, ptr %vFrame) + %.not = icmp eq ptr %free.handle, null + br i1 %.not, label %coro.end, label %coro.free + +coro.free: + call void @delete(ptr nonnull %free.handle) #2 + br label %coro.end + +coro.end: + call i1 @llvm.coro.end(ptr null, i1 false, token none) + ret void +} + +; CHECK-LABEL: @f.resume( +; CHECK: musttail call fastcc void @fakeresume1( +; CHECK-NEXT: ret void + +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1 +declare i1 @llvm.coro.alloc(token) #2 +declare i64 @llvm.coro.size.i64() #3 +declare ptr @llvm.coro.begin(token, ptr writeonly) #2 +declare token @llvm.coro.save(ptr) #2 +declare ptr @llvm.coro.frame() #3 +declare i8 @llvm.coro.suspend(token, i1) #2 +declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1 +declare i1 @llvm.coro.end(ptr, i1, token) #2 +declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1 +declare ptr @malloc(i64) +declare void @delete(ptr nonnull) #2 + +attributes #0 = { presplitcoroutine } +attributes #1 = { argmemonly nounwind readonly } +attributes #2 = { nounwind } +attributes #3 = { nounwind readnone } diff --git a/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail5.ll b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail5.ll new file mode 100644 index 0000000000000..3e5bddd8e1311 --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail5.ll @@ -0,0 +1,63 @@ +; Tests that instrumentation doesn't interfere with lowering (coro-split). +; It should convert coro.resume followed by a suspend to a musttail call. + +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s + +declare void @fakeresume1(ptr align 8) + +define void @g() #0 { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %alloc = call ptr @malloc(i64 16) #3 + %alloc.var = alloca i8 + call void @llvm.lifetime.start.p0(i64 1, ptr %alloc.var) + %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc) + + %save = call token @llvm.coro.save(ptr null) + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + + switch i8 %suspend, label %exit [ + i8 0, label %await.suspend + i8 1, label %exit + ] +await.suspend: + %save2 = call token @llvm.coro.save(ptr null) + call fastcc void @fakeresume1(ptr align 8 null) + %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false) + switch i8 %suspend2, label %exit [ + i8 0, label %await.ready + i8 1, label %exit + ] +await.ready: + call void @consume(ptr %alloc.var) + call void @llvm.lifetime.end.p0(i64 1, ptr %alloc.var) + br label %exit +exit: + call i1 @llvm.coro.end(ptr null, i1 false, token none) + ret void +} + +; Verify that in the resume part resume call is marked with musttail. +; CHECK-LABEL: @g.resume( +; CHECK: musttail call fastcc void @fakeresume1(ptr align 8 null) +; CHECK-NEXT: ret void + +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1 +declare i1 @llvm.coro.alloc(token) #2 +declare i64 @llvm.coro.size.i64() #3 +declare ptr @llvm.coro.begin(token, ptr writeonly) #2 +declare token @llvm.coro.save(ptr) #2 +declare ptr @llvm.coro.frame() #3 +declare i8 @llvm.coro.suspend(token, i1) #2 +declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1 +declare i1 @llvm.coro.end(ptr, i1, token) #2 +declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1 +declare ptr @malloc(i64) +declare void @consume(ptr) +declare void @llvm.lifetime.start.p0(i64, ptr nocapture) +declare void @llvm.lifetime.end.p0(i64, ptr nocapture) + +attributes #0 = { presplitcoroutine } +attributes #1 = { argmemonly nounwind readonly } +attributes #2 = { nounwind } +attributes #3 = { nounwind readnone } diff --git a/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail6.ll b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail6.ll new file mode 100644 index 0000000000000..5d068872fcace --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail6.ll @@ -0,0 +1,113 @@ +; Tests that instrumentation doesn't interfere with lowering (coro-split). +; It should convert coro.resume followed by a suspend to a musttail call. + +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s + +declare void @fakeresume1(ptr align 8) + +define void @g() #0 { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %alloc = call ptr @malloc(i64 16) #3 + %alloc.var = alloca i64 + call void @llvm.lifetime.start.p0(i64 1, ptr %alloc.var) + %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc) + + %save = call token @llvm.coro.save(ptr null) + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + + switch i8 %suspend, label %exit [ + i8 0, label %await.suspend + i8 1, label %exit + ] +await.suspend: + %save2 = call token @llvm.coro.save(ptr null) + call fastcc void @fakeresume1(ptr align 8 null) + %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false) + switch i8 %suspend2, label %exit [ + i8 0, label %await.ready + i8 1, label %exit + ] +await.ready: + call void @consume(ptr %alloc.var) + call void @llvm.lifetime.end.p0(i64 1, ptr %alloc.var) + br label %exit +exit: + call i1 @llvm.coro.end(ptr null, i1 false, token none) + ret void +} + +; Verify that in the resume part resume call is marked with musttail. +; CHECK-LABEL: @g.resume( +; CHECK: musttail call fastcc void @fakeresume1(ptr align 8 null) +; CHECK-NEXT: ret void + +; It has a cleanup bb. +define void @f() #0 { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %alloc = call ptr @malloc(i64 16) #3 + %alloc.var = alloca i64 + call void @llvm.lifetime.start.p0(i64 1, ptr %alloc.var) + %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc) + + %save = call token @llvm.coro.save(ptr null) + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + + switch i8 %suspend, label %exit [ + i8 0, label %await.suspend + i8 1, label %exit + ] +await.suspend: + %save2 = call token @llvm.coro.save(ptr null) + call fastcc void @fakeresume1(ptr align 8 null) + %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false) + switch i8 %suspend2, label %exit [ + i8 0, label %await.ready + i8 1, label %cleanup + ] +await.ready: + call void @consume(ptr %alloc.var) + call void @llvm.lifetime.end.p0(i64 1, ptr %alloc.var) + br label %exit + +cleanup: + %free.handle = call ptr @llvm.coro.free(token %id, ptr %vFrame) + %.not = icmp eq ptr %free.handle, null + br i1 %.not, label %exit, label %coro.free + +coro.free: + call void @delete(ptr nonnull %free.handle) #2 + br label %exit + +exit: + call i1 @llvm.coro.end(ptr null, i1 false, token none) + ret void +} + +; FIXME: The fakeresume1 here should be marked as musttail. +; Verify that in the resume part resume call is marked with musttail. +; CHECK-LABEL: @f.resume( +; CHECK: musttail call fastcc void @fakeresume1(ptr align 8 null) +; CHECK-NEXT: ret void + +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1 +declare i1 @llvm.coro.alloc(token) #2 +declare i64 @llvm.coro.size.i64() #3 +declare ptr @llvm.coro.begin(token, ptr writeonly) #2 +declare token @llvm.coro.save(ptr) #2 +declare ptr @llvm.coro.frame() #3 +declare i8 @llvm.coro.suspend(token, i1) #2 +declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1 +declare i1 @llvm.coro.end(ptr, i1, token) #2 +declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1 +declare ptr @malloc(i64) +declare void @delete(ptr nonnull) #2 +declare void @consume(ptr) +declare void @llvm.lifetime.start.p0(i64, ptr nocapture) +declare void @llvm.lifetime.end.p0(i64, ptr nocapture) + +attributes #0 = { presplitcoroutine } +attributes #1 = { argmemonly nounwind readonly } +attributes #2 = { nounwind } +attributes #3 = { nounwind readnone } diff --git a/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail7.ll b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail7.ll new file mode 100644 index 0000000000000..6ea81c6ff0b09 --- /dev/null +++ b/llvm/test/Instrumentation/InstrProfiling/Coro/coro-split-musttail7.ll @@ -0,0 +1,116 @@ +; Tests that instrumentation doesn't interfere with lowering (coro-split). +; It should convert coro.resume followed by a suspend to a musttail call. + +; The difference between this and coro-split-musttail5.ll and coro-split-musttail5.ll +; is that this contains dead instruction generated during the transformation, +; which makes the optimization harder. +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s + +declare void @fakeresume1(ptr align 8) + +define void @g() #0 { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %alloc = call ptr @malloc(i64 16) #3 + %alloc.var = alloca i64 + call void @llvm.lifetime.start.p0(i64 1, ptr %alloc.var) + %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc) + + %save = call token @llvm.coro.save(ptr null) + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + + switch i8 %suspend, label %exit [ + i8 0, label %await.suspend + i8 1, label %exit + ] +await.suspend: + %save2 = call token @llvm.coro.save(ptr null) + call fastcc void @fakeresume1(ptr align 8 null) + %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false) + switch i8 %suspend2, label %exit [ + i8 0, label %await.ready + i8 1, label %exit + ] +await.ready: + call void @consume(ptr %alloc.var) + call void @llvm.lifetime.end.p0(i64 1, ptr %alloc.var) + br label %exit +exit: + call i1 @llvm.coro.end(ptr null, i1 false, token none) + ret void +} + +; Verify that in the resume part resume call is marked with musttail. +; CHECK-LABEL: @g.resume( +; CHECK: musttail call fastcc void @fakeresume1(ptr align 8 null) +; CHECK-NEXT: ret void + +; It has a cleanup bb. +define void @f() #0 { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %alloc = call ptr @malloc(i64 16) #3 + %alloc.var = alloca i64 + call void @llvm.lifetime.start.p0(i64 1, ptr %alloc.var) + %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc) + + %save = call token @llvm.coro.save(ptr null) + %suspend = call i8 @llvm.coro.suspend(token %save, i1 false) + + switch i8 %suspend, label %exit [ + i8 0, label %await.suspend + i8 1, label %exit + ] +await.suspend: + %save2 = call token @llvm.coro.save(ptr null) + call fastcc void @fakeresume1(ptr align 8 null) + %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false) + switch i8 %suspend2, label %exit [ + i8 0, label %await.ready + i8 1, label %cleanup + ] +await.ready: + call void @consume(ptr %alloc.var) + call void @llvm.lifetime.end.p0(i64 1, ptr %alloc.var) + br label %exit + +cleanup: + %free.handle = call ptr @llvm.coro.free(token %id, ptr %vFrame) + %.not = icmp eq ptr %free.handle, null + br i1 %.not, label %exit, label %coro.free + +coro.free: + call void @delete(ptr nonnull %free.handle) #2 + br label %exit + +exit: + call i1 @llvm.coro.end(ptr null, i1 false, token none) + ret void +} + +; FIXME: The fakeresume1 here should be marked as musttail. +; Verify that in the resume part resume call is marked with musttail. +; CHECK-LABEL: @f.resume( +; CHECK: musttail call fastcc void @fakeresume1(ptr align 8 null) +; CHECK-NEXT: ret void + +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1 +declare i1 @llvm.coro.alloc(token) #2 +declare i64 @llvm.coro.size.i64() #3 +declare ptr @llvm.coro.begin(token, ptr writeonly) #2 +declare token @llvm.coro.save(ptr) #2 +declare ptr @llvm.coro.frame() #3 +declare i8 @llvm.coro.suspend(token, i1) #2 +declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1 +declare i1 @llvm.coro.end(ptr, i1, token) #2 +declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1 +declare ptr @malloc(i64) +declare void @delete(ptr nonnull) #2 +declare void @consume(ptr) +declare void @llvm.lifetime.start.p0(i64, ptr nocapture) +declare void @llvm.lifetime.end.p0(i64, ptr nocapture) + +attributes #0 = { presplitcoroutine } +attributes #1 = { argmemonly nounwind readonly } +attributes #2 = { nounwind } +attributes #3 = { nounwind readnone } diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail.ll index 0406135687904..825e44471db27 100644 --- a/llvm/test/Transforms/Coroutines/coro-split-musttail.ll +++ b/llvm/test/Transforms/Coroutines/coro-split-musttail.ll @@ -1,6 +1,7 @@ ; Tests that coro-split will convert coro.resume followed by a suspend to a ; musttail call. -; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s +; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK,NOPGO %s +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK,PGO %s define void @f() #0 { entry: @@ -40,7 +41,9 @@ exit: ; Verify that in the resume part resume call is marked with musttail. ; CHECK-LABEL: @f.resume( ; CHECK: %[[addr2:.+]] = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) -; CHECK-NEXT: musttail call fastcc void %[[addr2]](ptr null) +; NOPGO-NEXT: musttail call fastcc void %[[addr2]](ptr null) +; PGO: call void @llvm.instrprof +; PGO-NEXT: musttail call fastcc void %[[addr2]](ptr null) ; CHECK-NEXT: ret void declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1 diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail1.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail1.ll index cd1635b93d2cc..d0d11fc4495e4 100644 --- a/llvm/test/Transforms/Coroutines/coro-split-musttail1.ll +++ b/llvm/test/Transforms/Coroutines/coro-split-musttail1.ll @@ -1,6 +1,7 @@ ; Tests that coro-split will convert coro.resume followed by a suspend to a ; musttail call. -; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s +; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK,NOPGO %s +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK,PGO %s define void @f() #0 { entry: @@ -63,14 +64,17 @@ unreach: ; CHECK-LABEL: @f.resume( ; CHECK: %[[hdl:.+]] = call ptr @g() ; CHECK-NEXT: %[[addr2:.+]] = call ptr @llvm.coro.subfn.addr(ptr %[[hdl]], i8 0) -; CHECK-NEXT: musttail call fastcc void %[[addr2]](ptr %[[hdl]]) +; NOPGO-NEXT: musttail call fastcc void %[[addr2]](ptr %[[hdl]]) +; PGO: musttail call fastcc void %[[addr2]](ptr %[[hdl]]) ; CHECK-NEXT: ret void ; CHECK: %[[hdl2:.+]] = call ptr @h() ; CHECK-NEXT: %[[addr3:.+]] = call ptr @llvm.coro.subfn.addr(ptr %[[hdl2]], i8 0) -; CHECK-NEXT: musttail call fastcc void %[[addr3]](ptr %[[hdl2]]) +; NOPGO-NEXT: musttail call fastcc void %[[addr3]](ptr %[[hdl2]]) +; PGO: musttail call fastcc void %[[addr3]](ptr %[[hdl2]]) ; CHECK-NEXT: ret void ; CHECK: %[[addr4:.+]] = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) -; CHECK-NEXT: musttail call fastcc void %[[addr4]](ptr null) +; NOPGO-NEXT: musttail call fastcc void %[[addr4]](ptr null) +; PGO: musttail call fastcc void %[[addr4]](ptr null) ; CHECK-NEXT: ret void diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail10.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail10.ll index 9d73c8bbc57b8..cdd58b2a084fc 100644 --- a/llvm/test/Transforms/Coroutines/coro-split-musttail10.ll +++ b/llvm/test/Transforms/Coroutines/coro-split-musttail10.ll @@ -1,6 +1,7 @@ ; Tests that we would convert coro.resume to a musttail call if the target is ; Wasm64 with tail-call support. ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s target triple = "wasm64-unknown-unknown" diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail11.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail11.ll index 9bc5b4f0c65d9..da5d868280e96 100644 --- a/llvm/test/Transforms/Coroutines/coro-split-musttail11.ll +++ b/llvm/test/Transforms/Coroutines/coro-split-musttail11.ll @@ -1,6 +1,7 @@ ; Tests that we would convert coro.resume to a musttail call if the target is ; Wasm32 with tail-call support. ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s target triple = "wasm32-unknown-unknown" diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail12.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail12.ll index e7f4bcb9b0ff2..5baec378876bb 100644 --- a/llvm/test/Transforms/Coroutines/coro-split-musttail12.ll +++ b/llvm/test/Transforms/Coroutines/coro-split-musttail12.ll @@ -1,5 +1,6 @@ ; Tests that coro-split won't convert the cmp instruction prematurely. ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s declare void @fakeresume1(ptr) declare void @print() diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail13.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail13.ll index 2384f9382685b..0290e42339e2a 100644 --- a/llvm/test/Transforms/Coroutines/coro-split-musttail13.ll +++ b/llvm/test/Transforms/Coroutines/coro-split-musttail13.ll @@ -1,5 +1,6 @@ ; Tests that coro-split won't fall in infinite loop when simplify the terminators leading to ret. ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s declare void @fakeresume1(ptr) declare void @may_throw(ptr) diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail2.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail2.ll index 38fc12815c033..2f27f79480ab1 100644 --- a/llvm/test/Transforms/Coroutines/coro-split-musttail2.ll +++ b/llvm/test/Transforms/Coroutines/coro-split-musttail2.ll @@ -1,6 +1,7 @@ ; Tests that coro-split will convert coro.resume followed by a suspend to a ; musttail call. ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s define void @fakeresume1(ptr) { entry: diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail3.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail3.ll index b777f000e33a6..4778e3dcaf995 100644 --- a/llvm/test/Transforms/Coroutines/coro-split-musttail3.ll +++ b/llvm/test/Transforms/Coroutines/coro-split-musttail3.ll @@ -1,6 +1,7 @@ ; Tests that coro-split will convert coro.resume followed by a suspend to a ; musttail call. -; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s +; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK,NOPGO %s +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK,PGO %s define void @f() #0 { entry: @@ -59,14 +60,17 @@ unreach: ; CHECK-LABEL: @f.resume( ; CHECK: %[[hdl:.+]] = call ptr @g() ; CHECK-NEXT: %[[addr2:.+]] = call ptr @llvm.coro.subfn.addr(ptr %[[hdl]], i8 0) -; CHECK-NEXT: musttail call fastcc void %[[addr2]](ptr %[[hdl]]) +; NOPGO-NEXT: musttail call fastcc void %[[addr2]](ptr %[[hdl]]) +; PGO: musttail call fastcc void %[[addr2]](ptr %[[hdl]]) ; CHECK-NEXT: ret void ; CHECK: %[[hdl2:.+]] = call ptr @h() ; CHECK-NEXT: %[[addr3:.+]] = call ptr @llvm.coro.subfn.addr(ptr %[[hdl2]], i8 0) -; CHECK-NEXT: musttail call fastcc void %[[addr3]](ptr %[[hdl2]]) +; NOPGO-NEXT: musttail call fastcc void %[[addr3]](ptr %[[hdl2]]) +; PGO: musttail call fastcc void %[[addr3]](ptr %[[hdl2]]) ; CHECK-NEXT: ret void ; CHECK: %[[addr4:.+]] = call ptr @llvm.coro.subfn.addr(ptr null, i8 0) -; CHECK-NEXT: musttail call fastcc void %[[addr4]](ptr null) +; NOPGO-NEXT: musttail call fastcc void %[[addr4]](ptr null) +; PGO: musttail call fastcc void %[[addr4]](ptr null) ; CHECK-NEXT: ret void diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail4.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail4.ll index 1e0fcdb87a72d..00ee422ce5863 100644 --- a/llvm/test/Transforms/Coroutines/coro-split-musttail4.ll +++ b/llvm/test/Transforms/Coroutines/coro-split-musttail4.ll @@ -1,6 +1,7 @@ ; Tests that coro-split will convert a call before coro.suspend to a musttail call ; while the user of the coro.suspend is a icmpinst. ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s define void @fakeresume1(ptr) { entry: diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail5.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail5.ll index d19606491335e..9afc79abbe88c 100644 --- a/llvm/test/Transforms/Coroutines/coro-split-musttail5.ll +++ b/llvm/test/Transforms/Coroutines/coro-split-musttail5.ll @@ -1,6 +1,7 @@ ; Tests that sinked lifetime markers wouldn't provent optimization ; to convert a resuming call to a musttail call. ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s declare void @fakeresume1(ptr align 8) diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail6.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail6.ll index eea711861c488..9c2b1ece1624b 100644 --- a/llvm/test/Transforms/Coroutines/coro-split-musttail6.ll +++ b/llvm/test/Transforms/Coroutines/coro-split-musttail6.ll @@ -4,6 +4,7 @@ ; an extra bitcast instruction in the path, which makes it harder to ; optimize. ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s declare void @fakeresume1(ptr align 8) diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll index c32fe9b0ee304..860032bd3cf8e 100644 --- a/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll +++ b/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll @@ -4,6 +4,7 @@ ; is that this contains dead instruction generated during the transformation, ; which makes the optimization harder. ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s +; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s declare void @fakeresume1(ptr align 8)