diff --git a/components/ide/jetbrains/backend-plugin/launch-dev-server.sh b/components/ide/jetbrains/backend-plugin/launch-dev-server.sh index 7c658fb4ac8717..c6e7b17b8f5a70 100755 --- a/components/ide/jetbrains/backend-plugin/launch-dev-server.sh +++ b/components/ide/jetbrains/backend-plugin/launch-dev-server.sh @@ -49,4 +49,6 @@ export GIT_EDITOR="$EDITOR --wait" export GP_PREVIEW_BROWSER="$IDEA_CLI_DEV_PATH preview" export GP_EXTERNAL_BROWSER="$IDEA_CLI_DEV_PATH preview" +export JETBRAINS_GITPOD_BACKEND_KIND=intellij + $TEST_BACKEND_DIR/bin/remote-dev-server.sh run $TEST_DIR diff --git a/components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodCLIService.kt b/components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodCLIService.kt index a6d4244d2dcd80..49fd84e2f88832 100644 --- a/components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodCLIService.kt +++ b/components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodCLIService.kt @@ -10,27 +10,48 @@ import com.intellij.ide.CommandLineProcessor import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.client.ClientSession import com.intellij.openapi.client.ClientSessionsManager +import com.intellij.openapi.components.service import com.intellij.openapi.diagnostic.thisLogger import com.intellij.openapi.project.Project +import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream import com.intellij.openapi.util.io.FileUtilRt import com.intellij.util.application +import io.netty.buffer.Unpooled import io.netty.channel.ChannelHandlerContext import io.netty.handler.codec.http.FullHttpRequest import io.netty.handler.codec.http.QueryStringDecoder +import io.prometheus.client.exporter.common.TextFormat import org.jetbrains.ide.RestService +import org.jetbrains.io.response +import java.io.OutputStreamWriter import java.nio.file.InvalidPathException import java.nio.file.Path @Suppress("UnstableApiUsage") class GitpodCLIService : RestService() { + private val manager = service() + override fun getServiceName() = SERVICE_NAME override fun execute(urlDecoder: QueryStringDecoder, request: FullHttpRequest, context: ChannelHandlerContext): String? { + val operation = getStringParameter("op", urlDecoder) if (application.isHeadlessEnvironment) { return "not supported in headless mode" } - val operation = getStringParameter("op", urlDecoder) + /** + * prod: curl http://localhost:63342/api/gitpod/cli?op=metrics + * dev: curl http://localhost:63343/api/gitpod/cli?op=metrics + */ + if (operation == "metrics") { + val out = BufferExposingByteArrayOutputStream() + val writer = OutputStreamWriter(out) + TextFormat.write004(writer, manager.registry.metricFamilySamples()) + writer.close() + val response = response(TextFormat.CONTENT_TYPE_004, Unpooled.wrappedBuffer(out.internalBuffer, 0, out.size())) + sendResponse(request, context, response) + return null + } if (operation == "open") { val fileStr = getStringParameter("file", urlDecoder) if (fileStr.isNullOrBlank()) { diff --git a/components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodManager.kt b/components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodManager.kt index f75fb169b2f926..dc9bb16f3d9e23 100644 --- a/components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodManager.kt +++ b/components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodManager.kt @@ -12,6 +12,7 @@ import com.intellij.openapi.Disposable import com.intellij.openapi.components.Service import com.intellij.openapi.diagnostic.thisLogger import com.intellij.openapi.extensions.PluginId +import com.intellij.openapi.util.LowMemoryWatcher import com.intellij.remoteDev.util.onTerminationOrNow import com.intellij.util.application import com.jetbrains.rd.util.lifetime.Lifetime @@ -28,6 +29,7 @@ import io.grpc.ManagedChannelBuilder import io.grpc.stub.ClientCallStreamObserver import io.grpc.stub.ClientResponseObserver import io.prometheus.client.CollectorRegistry +import io.prometheus.client.Counter import io.prometheus.client.Gauge import io.prometheus.client.exporter.PushGateway import kotlinx.coroutines.GlobalScope @@ -64,13 +66,29 @@ class GitpodManager : Disposable { lifetime.terminate() } + val registry = CollectorRegistry() + init { - val monitoringJob = GlobalScope.launch { + // Rate of low memory after GC notifications in the last 5 minutes: + // rate(gitpod_jb_backend_low_memory_after_gc_total[5m]) + val lowMemoryCounter = Counter.build() + .name("gitpod_jb_backend_low_memory_after_gc") + .help("Low memory notifications after GC") + .labelNames("product", "qualifier") + .register(registry) + LowMemoryWatcher.register({ + lowMemoryCounter.labels(backendKind, backendQualifier).inc() + }, LowMemoryWatcher.LowMemoryWatcherType.ONLY_AFTER_GC, this) + } + + init { + val monitoringJob = GlobalScope.launch { if (application.isHeadlessEnvironment) { return@launch } - val pg = PushGateway("localhost:22999") - val registry = CollectorRegistry() + val pg = if(devMode) null else PushGateway("localhost:22999") + // Heap usage at any time in the last 5 minutes: + // max_over_time(gitpod_jb_backend_memory_used_bytes[5m:])/max_over_time(gitpod_jb_backend_memory_max_bytes[5m:]) val allocatedGauge = Gauge.build() .name("gitpod_jb_backend_memory_max_bytes") .help("Total allocated memory of JB backend in bytes.") @@ -81,13 +99,14 @@ class GitpodManager : Disposable { .help("Used memory of JB backend in bytes.") .labelNames("product", "qualifier") .register(registry) - while(isActive) { + + while (isActive) { val totalMemory = Runtime.getRuntime().totalMemory() allocatedGauge.labels(backendKind, backendQualifier).set(totalMemory.toDouble()) val usedMemory = (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) usedGauge.labels(backendKind, backendQualifier).set(usedMemory.toDouble()) try { - pg.push(registry, "jb_backend") + pg?.push(registry, "jb_backend") } catch (t: Throwable) { thisLogger().error("gitpod: failed to push monitoring metrics:", t) } @@ -108,7 +127,7 @@ class GitpodManager : Disposable { .connectTimeout(Duration.ofSeconds(5)) .build() val httpRequest = HttpRequest.newBuilder() - .uri(URI.create("http://localhost:24000/gatewayLink?backendPort=${backendPort}")) + .uri(URI.create("http://localhost:24000/gatewayLink?backendPort=$backendPort")) .GET() .build() val response = @@ -260,14 +279,14 @@ class GitpodManager : Disposable { tokenResponse.token ) } finally { - Thread.currentThread().contextClassLoader = originalClassLoader; + Thread.currentThread().contextClassLoader = originalClassLoader } } val minReconnectionDelay = 2 * 1000L val maxReconnectionDelay = 30 * 1000L - val reconnectionDelayGrowFactor = 1.5; - var reconnectionDelay = minReconnectionDelay; + val reconnectionDelayGrowFactor = 1.5 + var reconnectionDelay = minReconnectionDelay val gitpodHost = info.gitpodApi.host var closeReason: Any = "cancelled" try { diff --git a/operations/observability/mixins/IDE/dashboards.libsonnet b/operations/observability/mixins/IDE/dashboards.libsonnet index 76db5d94bca728..c40a8a456502c1 100644 --- a/operations/observability/mixins/IDE/dashboards.libsonnet +++ b/operations/observability/mixins/IDE/dashboards.libsonnet @@ -10,6 +10,7 @@ // 'my-new-dashboard.json': (import 'dashboards/components/new-component.json'), 'gitpod-component-openvsx-proxy.json': (import 'dashboards/components/openvsx-proxy.json'), 'gitpod-component-ssh-gateway.json': (import 'dashboards/components/ssh-gateway.json'), + 'gitpod-component-jb.json': (import 'dashboards/components/jb.json'), }, } diff --git a/operations/observability/mixins/IDE/dashboards/components/jb.json b/operations/observability/mixins/IDE/dashboards/components/jb.json new file mode 100644 index 00000000000000..fcd13895841ff5 --- /dev/null +++ b/operations/observability/mixins/IDE/dashboards/components/jb.json @@ -0,0 +1,232 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 63, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "notification/second", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "code", + "expr": "sum(rate(gitpod_jb_backend_low_memory_after_gc_total[5m]))", + "legendFormat": "rate", + "range": true, + "refId": "A" + } + ], + "title": "Total low memory after GC", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "notification/second", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "code", + "expr": "topk(10, sum(rate(gitpod_jb_backend_low_memory_after_gc_total[5m])) by (pod))", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Top low memory after GC", + "type": "timeseries" + } + ], + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "JetBrains Overview", + "uid": "oamBLUC7k", + "version": 8, + "weekStart": "" +}