[jb]: monitor low memory notifications and gc overhead

Author: akosyakov

components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodClientProjectSessionTracker.kt

@@ -17,6 +17,7 @@ import com.intellij.openapi.diagnostic.thisLogger
 import com.intellij.openapi.fileEditor.FileEditorManagerEvent
 import com.intellij.openapi.fileEditor.FileEditorManagerListener
 import com.intellij.openapi.fileTypes.LanguageFileType
+import com.intellij.openapi.util.LowMemoryWatcher
 import com.intellij.remoteDev.util.onTerminationOrNow
 import com.intellij.util.application
 import com.jetbrains.rd.util.lifetime.Lifetime
@@ -58,6 +59,17 @@ class GitpodClientProjectSessionTracker(
         }
     }
 
+    init {
+        val lowMemoryWatcher = ClientId.withClientId(session.clientId) {
+            LowMemoryWatcher.register {
+                manager.incLowMemoryCounter()
+            }
+        }
+        lifetime.onTerminationOrNow {
+            lowMemoryWatcher.stop()
+        }
+    }
+
     private fun isExposedServedPort(port: Status.PortsStatus?) : Boolean {
         if (port === null) {
             return false

components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodManager.kt

@@ -15,6 +15,7 @@ import com.intellij.openapi.extensions.PluginId
 import com.intellij.remoteDev.util.onTerminationOrNow
 import com.intellij.util.application
 import com.jetbrains.rd.util.lifetime.Lifetime
+import com.sun.management.GarbageCollectionNotificationInfo
 import git4idea.config.GitVcsApplicationSettings
 import io.gitpod.gitpodprotocol.api.GitpodClient
 import io.gitpod.gitpodprotocol.api.GitpodServerLauncher
@@ -28,6 +29,7 @@ import io.grpc.ManagedChannelBuilder
 import io.grpc.stub.ClientCallStreamObserver
 import io.grpc.stub.ClientResponseObserver
 import io.prometheus.client.CollectorRegistry
+import io.prometheus.client.Counter
 import io.prometheus.client.Gauge
 import io.prometheus.client.exporter.PushGateway
 import kotlinx.coroutines.GlobalScope
@@ -37,13 +39,20 @@ import kotlinx.coroutines.guava.asDeferred
 import kotlinx.coroutines.isActive
 import kotlinx.coroutines.launch
 import org.jetbrains.ide.BuiltInServerManager
+import java.lang.management.ManagementFactory
 import java.net.URI
 import java.net.http.HttpClient
 import java.net.http.HttpRequest
 import java.net.http.HttpResponse
 import java.time.Duration
 import java.util.concurrent.CancellationException
 import java.util.concurrent.CompletableFuture
+import java.util.concurrent.TimeUnit
+import javax.management.ListenerNotFoundException
+import javax.management.Notification
+import javax.management.NotificationEmitter
+import javax.management.NotificationListener
+import javax.management.openmbean.CompositeData
 import javax.websocket.DeploymentException
 
 @Service
@@ -64,13 +73,65 @@ class GitpodManager : Disposable {
         lifetime.terminate()
     }
 
+
+    private val registry = CollectorRegistry()
+
+    private val lowMemoryCounter = Counter.build()
+        .name("gitpod_jb_backend_low_memory_counter")
+        .help("Low memory notification counter")
+        .labelNames("product", "qualifier")
+        .register(registry)
+    fun incLowMemoryCounter() {
+        lowMemoryCounter.labels(backendKind, backendQualifier).inc()
+    }
+
+    private val gcOverheadGauge = Gauge.build()
+        .name("gitpod_jb_backend_gc_overhead")
+        .help("An approximation of the percent of CPU time used by GC activities in the range [0..1]")
+        .labelNames("product", "qualifier")
+        .register(registry)
+    init {
+        val startTime = System.nanoTime()
+        var gcPauseTime = 0L
+        val listener = object : NotificationListener {
+            override fun handleNotification(notification: Notification, ref: Any?) {
+                val notificationInfo = GarbageCollectionNotificationInfo.from(notification.userData as CompositeData)
+                if ("No GC" == notificationInfo.gcCause || "Shenandoah Cycles" == notificationInfo.gcName || "ZGC Cycles" == notificationInfo.gcName) {
+                    // see https://github.com/micrometer-metrics/micrometer/blob/b617100e92f55584f3207a29c8e0338d73042b73/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemory.java#L37
+                    return
+                }
+                gcPauseTime += notificationInfo.gcInfo.duration
+                val totalTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime)
+                val gcOverhead = gcPauseTime / totalTime.toDouble()
+                gcOverheadGauge.labels(backendKind, backendQualifier).set(gcOverhead)
+            }
+        }
+        for (gcBean in ManagementFactory.getGarbageCollectorMXBeans()) {
+            if (gcBean !is NotificationEmitter) {
+                continue
+            }
+            gcBean.addNotificationListener(listener,
+                { notification: Notification ->
+                    (notification.type == GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION)
+                }, null
+            )
+            lifetime.onTerminationOrNow {
+                try {
+                    gcBean.removeNotificationListener(listener)
+                } catch (ignore: ListenerNotFoundException) {
+                }
+            }
+        }
+    }
+
     init {
-        val monitoringJob =  GlobalScope.launch {
+        val monitoringJob = GlobalScope.launch {
             if (application.isHeadlessEnvironment) {
                 return@launch
             }
             val pg = PushGateway("localhost:22999")
-            val registry = CollectorRegistry()
+            // prod: count(max_over_time(gitpod_jb_backend_memory_used_bytes/gitpod_jb_backend_memory_max_bytes)[5m:] > 0.9)
+            // prev: count(max_over_time(gitpod_jb_backend_memory_used_bytes[5m:])/max_over_time(gitpod_jb_backend_memory_max_bytes[5m:]) > 0.8)
             val allocatedGauge = Gauge.build()
                     .name("gitpod_jb_backend_memory_max_bytes")
                     .help("Total allocated memory of JB backend in bytes.")
@@ -81,7 +142,8 @@ class GitpodManager : Disposable {
                     .help("Used memory of JB backend in bytes.")
                     .labelNames("product", "qualifier")
                     .register(registry)
-            while(isActive) {
+
+            while (isActive) {
                 val totalMemory = Runtime.getRuntime().totalMemory()
                 allocatedGauge.labels(backendKind, backendQualifier).set(totalMemory.toDouble())
                 val usedMemory = (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())
@@ -108,7 +170,7 @@ class GitpodManager : Disposable {
                     .connectTimeout(Duration.ofSeconds(5))
                     .build()
                 val httpRequest = HttpRequest.newBuilder()
-                    .uri(URI.create("http://localhost:24000/gatewayLink?backendPort=${backendPort}"))
+                    .uri(URI.create("http://localhost:24000/gatewayLink?backendPort=$backendPort"))
                     .GET()
                     .build()
                 val response =
@@ -260,14 +322,14 @@ class GitpodManager : Disposable {
                         tokenResponse.token
                 )
             } finally {
-                Thread.currentThread().contextClassLoader = originalClassLoader;
+                Thread.currentThread().contextClassLoader = originalClassLoader
             }
         }
 
         val minReconnectionDelay = 2 * 1000L
         val maxReconnectionDelay = 30 * 1000L
-        val reconnectionDelayGrowFactor = 1.5;
-        var reconnectionDelay = minReconnectionDelay;
+        val reconnectionDelayGrowFactor = 1.5
+        var reconnectionDelay = minReconnectionDelay
         val gitpodHost = info.gitpodApi.host
         var closeReason: Any = "cancelled"
         try {