[jb]: monitor low memory notifications

components/ide/jetbrains/backend-plugin/launch-dev-server.sh

@@ -49,4 +49,6 @@ export GIT_EDITOR="$EDITOR --wait"
 export GP_PREVIEW_BROWSER="$IDEA_CLI_DEV_PATH preview"
 export GP_EXTERNAL_BROWSER="$IDEA_CLI_DEV_PATH preview"
 
+export JETBRAINS_GITPOD_BACKEND_KIND=intellij
+
 $TEST_BACKEND_DIR/bin/remote-dev-server.sh run $TEST_DIR

components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodCLIService.kt

@@ -10,27 +10,48 @@ import com.intellij.ide.CommandLineProcessor
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.client.ClientSession
 import com.intellij.openapi.client.ClientSessionsManager
+import com.intellij.openapi.components.service
 import com.intellij.openapi.diagnostic.thisLogger
 import com.intellij.openapi.project.Project
+import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream
 import com.intellij.openapi.util.io.FileUtilRt
 import com.intellij.util.application
+import io.netty.buffer.Unpooled
 import io.netty.channel.ChannelHandlerContext
 import io.netty.handler.codec.http.FullHttpRequest
 import io.netty.handler.codec.http.QueryStringDecoder
+import io.prometheus.client.exporter.common.TextFormat
 import org.jetbrains.ide.RestService
+import org.jetbrains.io.response
+import java.io.OutputStreamWriter
 import java.nio.file.InvalidPathException
 import java.nio.file.Path
 
 @Suppress("UnstableApiUsage")
 class GitpodCLIService : RestService() {
 
+    private val manager = service<GitpodManager>()
+
     override fun getServiceName() = SERVICE_NAME
 
     override fun execute(urlDecoder: QueryStringDecoder, request: FullHttpRequest, context: ChannelHandlerContext): String? {
+        val operation = getStringParameter("op", urlDecoder)
         if (application.isHeadlessEnvironment) {
             return "not supported in headless mode"
         }
-        val operation = getStringParameter("op", urlDecoder)
+        /**
+         * prod: curl http://localhost:63342/api/gitpod/cli?op=metrics
+         * dev:  curl http://localhost:63343/api/gitpod/cli?op=metrics
+         */
+        if (operation == "metrics") {
+            val out = BufferExposingByteArrayOutputStream()
+            val writer = OutputStreamWriter(out)
+            TextFormat.write004(writer, manager.registry.metricFamilySamples())
+            writer.close()
+            val response = response(TextFormat.CONTENT_TYPE_004, Unpooled.wrappedBuffer(out.internalBuffer, 0, out.size()))
+            sendResponse(request, context, response)
+            return null
+        }
         if (operation == "open") {
             val fileStr = getStringParameter("file", urlDecoder)
             if (fileStr.isNullOrBlank()) {

components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodManager.kt

@@ -12,6 +12,7 @@ import com.intellij.openapi.Disposable
 import com.intellij.openapi.components.Service
 import com.intellij.openapi.diagnostic.thisLogger
 import com.intellij.openapi.extensions.PluginId
+import com.intellij.openapi.util.LowMemoryWatcher
 import com.intellij.remoteDev.util.onTerminationOrNow
 import com.intellij.util.application
 import com.jetbrains.rd.util.lifetime.Lifetime
@@ -28,6 +29,7 @@ import io.grpc.ManagedChannelBuilder
 import io.grpc.stub.ClientCallStreamObserver
 import io.grpc.stub.ClientResponseObserver
 import io.prometheus.client.CollectorRegistry
+import io.prometheus.client.Counter
 import io.prometheus.client.Gauge
 import io.prometheus.client.exporter.PushGateway
 import kotlinx.coroutines.GlobalScope
@@ -64,13 +66,29 @@ class GitpodManager : Disposable {
         lifetime.terminate()
     }
 
+    val registry = CollectorRegistry()
+
     init {
-        val monitoringJob =  GlobalScope.launch {
+        // Rate of low memory after GC notifications in the last 5 minutes:
+        // rate(gitpod_jb_backend_low_memory_after_gc_total[5m])
+        val lowMemoryCounter = Counter.build()
+            .name("gitpod_jb_backend_low_memory_after_gc")
+            .help("Low memory notifications after GC")
+            .labelNames("product", "qualifier")
+            .register(registry)
+        LowMemoryWatcher.register({
+            lowMemoryCounter.labels(backendKind, backendQualifier).inc()
+         }, LowMemoryWatcher.LowMemoryWatcherType.ONLY_AFTER_GC, this)
+    }
+
+    init {
+        val monitoringJob = GlobalScope.launch {
             if (application.isHeadlessEnvironment) {
                 return@launch
             }
-            val pg = PushGateway("localhost:22999")
-            val registry = CollectorRegistry()
+            val pg = if(devMode) null else PushGateway("localhost:22999")
+            // Heap usage at any time in the last 5 minutes:
+            // max_over_time(gitpod_jb_backend_memory_used_bytes[5m:])/max_over_time(gitpod_jb_backend_memory_max_bytes[5m:])
             val allocatedGauge = Gauge.build()
                     .name("gitpod_jb_backend_memory_max_bytes")
                     .help("Total allocated memory of JB backend in bytes.")
@@ -81,13 +99,14 @@ class GitpodManager : Disposable {
                     .help("Used memory of JB backend in bytes.")
                     .labelNames("product", "qualifier")
                     .register(registry)
-            while(isActive) {
+
+            while (isActive) {
                 val totalMemory = Runtime.getRuntime().totalMemory()
                 allocatedGauge.labels(backendKind, backendQualifier).set(totalMemory.toDouble())
                 val usedMemory = (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())
                 usedGauge.labels(backendKind, backendQualifier).set(usedMemory.toDouble())
                 try {
-                    pg.push(registry, "jb_backend")
+                    pg?.push(registry, "jb_backend")
                 } catch (t: Throwable) {
                     thisLogger().error("gitpod: failed to push monitoring metrics:", t)
                 }
@@ -108,7 +127,7 @@ class GitpodManager : Disposable {
                     .connectTimeout(Duration.ofSeconds(5))
                     .build()
                 val httpRequest = HttpRequest.newBuilder()
-                    .uri(URI.create("http://localhost:24000/gatewayLink?backendPort=${backendPort}"))
+                    .uri(URI.create("http://localhost:24000/gatewayLink?backendPort=$backendPort"))
                     .GET()
                     .build()
                 val response =
@@ -260,14 +279,14 @@ class GitpodManager : Disposable {
                         tokenResponse.token
                 )
             } finally {
-                Thread.currentThread().contextClassLoader = originalClassLoader;
+                Thread.currentThread().contextClassLoader = originalClassLoader
             }
         }
 
         val minReconnectionDelay = 2 * 1000L
         val maxReconnectionDelay = 30 * 1000L
-        val reconnectionDelayGrowFactor = 1.5;
-        var reconnectionDelay = minReconnectionDelay;
+        val reconnectionDelayGrowFactor = 1.5
+        var reconnectionDelay = minReconnectionDelay
         val gitpodHost = info.gitpodApi.host
         var closeReason: Any = "cancelled"
         try {

operations/observability/mixins/IDE/dashboards.libsonnet

@@ -10,6 +10,7 @@
     // 'my-new-dashboard.json': (import 'dashboards/components/new-component.json'),
     'gitpod-component-openvsx-proxy.json': (import 'dashboards/components/openvsx-proxy.json'),
     'gitpod-component-ssh-gateway.json': (import 'dashboards/components/ssh-gateway.json'),
+    'gitpod-component-jb.json': (import 'dashboards/components/jb.json'),
 
   },
 }

operations/observability/mixins/IDE/dashboards/components/jb.json

@@ -0,0 +1,232 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 63,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P1809F7CD0C75ACF3"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "notification/second",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 11,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [
+            "max",
+            "mean"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P1809F7CD0C75ACF3"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(gitpod_jb_backend_low_memory_after_gc_total[5m]))",
+          "legendFormat": "rate",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Total low memory after GC",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "P1809F7CD0C75ACF3"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "notification/second",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 11,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [
+            "max",
+            "mean"
+          ],
+          "displayMode": "table",
+          "placement": "right"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "P1809F7CD0C75ACF3"
+          },
+          "editorMode": "code",
+          "expr": "topk(10, sum(rate(gitpod_jb_backend_low_memory_after_gc_total[5m])) by (pod))",
+          "legendFormat": "{{pod}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Top low memory after GC",
+      "type": "timeseries"
+    }
+  ],
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "JetBrains Overview",
+  "uid": "oamBLUC7k",
+  "version": 8,
+  "weekStart": ""
+}