Multiplatform benchmarks baseline (#4350)

Update kotlinx-benchmark-based setup in kotlinx-coroutines-core and add a basic zero-contention benchmark for locks

Author: qwwdfsad

buildSrc/build.gradle.kts

@@ -63,7 +63,7 @@ dependencies {
         exclude(group = "org.jetbrains.kotlin", module = "kotlin-stdlib-jdk7")
         exclude(group = "org.jetbrains.kotlin", module = "kotlin-stdlib")
     }
-    implementation("org.jetbrains.kotlinx:kotlinx-benchmark-plugin:0.4.9")
+    implementation("org.jetbrains.kotlinx:kotlinx-benchmark-plugin:${version("benchmarks")}")
     implementation("org.jetbrains.kotlinx:kotlinx-knit:${version("knit")}")
     implementation("org.jetbrains.kotlinx:atomicfu-gradle-plugin:${version("atomicfu")}")
 }

gradle.properties

@@ -4,6 +4,8 @@ group=org.jetbrains.kotlinx
 kotlin_version=2.1.0
 # DO NOT rename this property without adapting kotlinx.train build chain:
 atomicfu_version=0.26.1
+benchmarks_version=0.4.13
+benchmarks_jmh_version=1.37
 
 # Dependencies
 junit_version=4.12

kotlinx-coroutines-core/benchmarks/README.md

@@ -0,0 +1,21 @@
+## kotlinx-coroutines-core benchmarks
+
+Multiplatform benchmarks for kotlinx-coroutines-core.
+
+This source-set contains benchmarks that leverage `internal` API (e.g. `suspendCancellableCoroutineReusable`) or
+that are multiplatform (-> only supported with `kotlinx-benchmarks` which is less convenient than `jmh` plugin).
+For JVM-only non-internal benchmarks, consider using `benchmarks` top-level project.
+
+### Usage
+
+```
+// JVM only
+./gradlew :kotlinx-coroutines-core:jvmBenchmarkBenchmarkJar
+java -jar kotlinx-coroutines-core/build/benchmarks/jvmBenchmark/jars/kotlinx-coroutines-core-jvmBenchmark-jmh-*-JMH.jar
+
+// Native, OS X
+./gradlew :kotlinx-coroutines-core:macosArm64BenchmarkBenchmark
+
+// Figure out what to use
+./gradlew :kotlinx-coroutines-core:tasks | grep -i bench
+```

kotlinx-coroutines-core/jvmBenchmark/kotlin/kotlinx/coroutines/BenchmarkUtils.kt renamed to kotlinx-coroutines-core/benchmarks/jvm/kotlin/kotlinx/coroutines/BenchmarkUtils.kt

@@ -76,7 +76,7 @@ open class SemaphoreBenchmark {
 
 enum class SemaphoreBenchDispatcherCreator(val create: (parallelism: Int) -> CoroutineDispatcher) {
     FORK_JOIN({ parallelism -> ForkJoinPool(parallelism).asCoroutineDispatcher() }),
-    DEFAULT({ parallelism -> ExperimentalCoroutineDispatcher(corePoolSize = parallelism, maxPoolSize = parallelism) })
+    DEFAULT({ parallelism -> CoroutineScheduler(corePoolSize = parallelism, maxPoolSize = parallelism).asCoroutineDispatcher() })
 }
 
 private const val WORK_INSIDE = 50

kotlinx-coroutines-core/jvmBenchmark/kotlin/kotlinx/coroutines/SemaphoreBenchmark.kt renamed to kotlinx-coroutines-core/benchmarks/jvm/kotlin/kotlinx/coroutines/SemaphoreBenchmark.kt

@@ -133,7 +133,7 @@ open class ChannelProducerConsumerBenchmark {
 
 enum class DispatcherCreator(val create: (parallelism: Int) -> CoroutineDispatcher) {
     FORK_JOIN({ parallelism ->  ForkJoinPool(parallelism).asCoroutineDispatcher() }),
-    DEFAULT({ parallelism -> ExperimentalCoroutineDispatcher(corePoolSize = parallelism, maxPoolSize = parallelism) })
+    DEFAULT({ parallelism -> CoroutineScheduler(corePoolSize = parallelism, maxPoolSize = parallelism).asCoroutineDispatcher() })
 }
 
 enum class ChannelCreator(private val capacity: Int) {

kotlinx-coroutines-core/jvmBenchmark/kotlin/kotlinx/coroutines/channels/ChannelProducerConsumerBenchmark.kt renamed to kotlinx-coroutines-core/benchmarks/jvm/kotlin/kotlinx/coroutines/channels/ChannelProducerConsumerBenchmark.kt

@@ -0,0 +1,24 @@
+package kotlinx.coroutines
+
+import kotlinx.coroutines.*
+import kotlinx.coroutines.flow.*
+import kotlinx.benchmark.*
+
+// Stresses out 'syncrhonozed' codepath in MutableSharedFlow
+@State(Scope.Benchmark)
+@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS)
+@OutputTimeUnit(BenchmarkTimeUnit.MICROSECONDS)
+@BenchmarkMode(Mode.AverageTime)
+open class SharedFlowBaseline {
+    private var size: Int = 10_000
+
+    @Benchmark
+    fun baseline() = runBlocking {
+        val flow = MutableSharedFlow<Unit>()
+        launch {
+            repeat(size) { flow.emit(Unit) }
+        }
+
+        flow.take(size).collect {  }
+    }
+}

kotlinx-coroutines-core/jvmBenchmark/kotlin/kotlinx/coroutines/channels/SelectBenchmark.kt renamed to kotlinx-coroutines-core/benchmarks/jvm/kotlin/kotlinx/coroutines/channels/SelectBenchmark.kt

@@ -1,5 +1,7 @@
 import org.gradle.api.tasks.testing.*
 import org.gradle.kotlin.dsl.*
+import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
+import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
 import org.jetbrains.kotlin.gradle.plugin.mpp.*
 import org.jetbrains.kotlin.gradle.targets.native.tasks.*
 import org.jetbrains.kotlin.gradle.tasks.*
@@ -59,6 +61,8 @@ kotlin {
             }
         }
     }
+    setupBenchmarkSourceSets(sourceSets)
+
     /*
      * Configure two test runs for Native:
      * 1) Main thread
@@ -84,13 +88,45 @@ kotlin {
     jvm {
         // For animal sniffer
         withJava()
-        compilations.create("benchmark") { associateWith(this@jvm.compilations.getByName("main")) }
     }
 }
 
-benchmark {
-    targets {
-        register("jvmBenchmark")
+private fun KotlinMultiplatformExtension.setupBenchmarkSourceSets(ss: NamedDomainObjectContainer<KotlinSourceSet>) {
+    // Forgive me, Father, for I have sinned.
+    // Really, that is needed to have benchmark sourcesets be the part of the project, not a separate project
+    val benchmarkMain by ss.creating {
+        dependencies {
+            implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:${version("benchmarks")}")
+        }
+        // For each source set we have to manually set path to the sources, otherwise lookup will fail
+        kotlin.srcDir("benchmarks/main/kotlin")
+    }
+
+    @Suppress("UnusedVariable")
+    val jvmBenchmark by ss.creating {
+        // For each source set we have to manually set path to the sources, otherwise lookup will fail
+        kotlin.srcDir("benchmarks/jvm/kotlin")
+    }
+
+    targets.matching {
+        it.name != "metadata"
+            // Doesn't work, don't want to figure it out for now
+            && !it.name.contains("wasm")
+            && !it.name.contains("js")
+    }.all {
+        compilations.create("benchmark") {
+            associateWith(this@all.compilations.getByName("main"))
+            defaultSourceSet {
+                dependencies {
+                    implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:${version("benchmarks")}")
+                }
+                dependsOn(benchmarkMain)
+            }
+        }
+    }
+
+    targets.matching { it.name != "metadata" }.all {
+        benchmark.targets.register("${name}Benchmark")
     }
 }
 
@@ -131,10 +167,12 @@ val allMetadataJar by tasks.getting(Jar::class) { setupManifest(this) }
 
 fun setupManifest(jar: Jar) {
     jar.manifest {
-        attributes(mapOf(
-            "Premain-Class" to "kotlinx.coroutines.debug.internal.AgentPremain",
-            "Can-Retransform-Classes" to "true",
-        ))
+        attributes(
+            mapOf(
+                "Premain-Class" to "kotlinx.coroutines.debug.internal.AgentPremain",
+                "Can-Retransform-Classes" to "true",
+            )
+        )
     }
 }
 
@@ -187,9 +225,11 @@ fun Test.configureJvmForLincheck(segmentSize: Int = 1) {
     minHeapSize = "1g"
     maxHeapSize = "4g" // we may need more space for building an interleaving tree in the model checking mode
     // https://github.com/JetBrains/lincheck#java-9
-    jvmArgs = listOf("--add-opens", "java.base/jdk.internal.misc=ALL-UNNAMED",   // required for transformation
+    jvmArgs = listOf(
+        "--add-opens", "java.base/jdk.internal.misc=ALL-UNNAMED",   // required for transformation
         "--add-exports", "java.base/sun.security.action=ALL-UNNAMED",
-        "--add-exports", "java.base/jdk.internal.util=ALL-UNNAMED") // in the model checking mode
+        "--add-exports", "java.base/jdk.internal.util=ALL-UNNAMED"
+    ) // in the model checking mode
     // Adjust internal algorithmic parameters to increase the testing quality instead of performance.
     systemProperty("kotlinx.coroutines.semaphore.segmentSize", segmentSize)
     systemProperty("kotlinx.coroutines.semaphore.maxSpinCycles", 1) // better for the model checking mode
@@ -215,6 +255,9 @@ kover {
             // lincheck has NPE error on `ManagedStrategyStateHolder` class
             excludedClasses.addAll("org.jetbrains.kotlinx.lincheck.*")
         }
+        sources {
+            excludedSourceSets.addAll("benchmark")
+        }
     }
 
     reports {