Skip to content

[jb]: monitor low memory notifications #10558

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 16, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,6 @@ export GIT_EDITOR="$EDITOR --wait"
export GP_PREVIEW_BROWSER="$IDEA_CLI_DEV_PATH preview"
export GP_EXTERNAL_BROWSER="$IDEA_CLI_DEV_PATH preview"

export JETBRAINS_GITPOD_BACKEND_KIND=intellij

$TEST_BACKEND_DIR/bin/remote-dev-server.sh run $TEST_DIR
Original file line number Diff line number Diff line change
Expand Up @@ -10,27 +10,48 @@ import com.intellij.ide.CommandLineProcessor
import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.client.ClientSession
import com.intellij.openapi.client.ClientSessionsManager
import com.intellij.openapi.components.service
import com.intellij.openapi.diagnostic.thisLogger
import com.intellij.openapi.project.Project
import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream
import com.intellij.openapi.util.io.FileUtilRt
import com.intellij.util.application
import io.netty.buffer.Unpooled
import io.netty.channel.ChannelHandlerContext
import io.netty.handler.codec.http.FullHttpRequest
import io.netty.handler.codec.http.QueryStringDecoder
import io.prometheus.client.exporter.common.TextFormat
import org.jetbrains.ide.RestService
import org.jetbrains.io.response
import java.io.OutputStreamWriter
import java.nio.file.InvalidPathException
import java.nio.file.Path

@Suppress("UnstableApiUsage")
class GitpodCLIService : RestService() {

private val manager = service<GitpodManager>()

override fun getServiceName() = SERVICE_NAME

override fun execute(urlDecoder: QueryStringDecoder, request: FullHttpRequest, context: ChannelHandlerContext): String? {
val operation = getStringParameter("op", urlDecoder)
if (application.isHeadlessEnvironment) {
return "not supported in headless mode"
}
val operation = getStringParameter("op", urlDecoder)
/**
* prod: curl http://localhost:63342/api/gitpod/cli?op=metrics
* dev: curl http://localhost:63343/api/gitpod/cli?op=metrics
*/
if (operation == "metrics") {
val out = BufferExposingByteArrayOutputStream()
val writer = OutputStreamWriter(out)
TextFormat.write004(writer, manager.registry.metricFamilySamples())
writer.close()
val response = response(TextFormat.CONTENT_TYPE_004, Unpooled.wrappedBuffer(out.internalBuffer, 0, out.size()))
sendResponse(request, context, response)
return null
}
if (operation == "open") {
val fileStr = getStringParameter("file", urlDecoder)
if (fileStr.isNullOrBlank()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import com.intellij.openapi.Disposable
import com.intellij.openapi.components.Service
import com.intellij.openapi.diagnostic.thisLogger
import com.intellij.openapi.extensions.PluginId
import com.intellij.openapi.util.LowMemoryWatcher
import com.intellij.remoteDev.util.onTerminationOrNow
import com.intellij.util.application
import com.jetbrains.rd.util.lifetime.Lifetime
Expand All @@ -28,6 +29,7 @@ import io.grpc.ManagedChannelBuilder
import io.grpc.stub.ClientCallStreamObserver
import io.grpc.stub.ClientResponseObserver
import io.prometheus.client.CollectorRegistry
import io.prometheus.client.Counter
import io.prometheus.client.Gauge
import io.prometheus.client.exporter.PushGateway
import kotlinx.coroutines.GlobalScope
Expand Down Expand Up @@ -64,13 +66,29 @@ class GitpodManager : Disposable {
lifetime.terminate()
}

val registry = CollectorRegistry()

init {
val monitoringJob = GlobalScope.launch {
// Rate of low memory after GC notifications in the last 5 minutes:
// rate(gitpod_jb_backend_low_memory_after_gc_total[5m])
val lowMemoryCounter = Counter.build()
.name("gitpod_jb_backend_low_memory_after_gc")
.help("Low memory notifications after GC")
.labelNames("product", "qualifier")
.register(registry)
LowMemoryWatcher.register({
lowMemoryCounter.labels(backendKind, backendQualifier).inc()
}, LowMemoryWatcher.LowMemoryWatcherType.ONLY_AFTER_GC, this)
}

init {
val monitoringJob = GlobalScope.launch {
if (application.isHeadlessEnvironment) {
return@launch
}
val pg = PushGateway("localhost:22999")
val registry = CollectorRegistry()
val pg = if(devMode) null else PushGateway("localhost:22999")
// Heap usage at any time in the last 5 minutes:
// max_over_time(gitpod_jb_backend_memory_used_bytes[5m:])/max_over_time(gitpod_jb_backend_memory_max_bytes[5m:])
val allocatedGauge = Gauge.build()
.name("gitpod_jb_backend_memory_max_bytes")
.help("Total allocated memory of JB backend in bytes.")
Expand All @@ -81,13 +99,14 @@ class GitpodManager : Disposable {
.help("Used memory of JB backend in bytes.")
.labelNames("product", "qualifier")
.register(registry)
while(isActive) {

while (isActive) {
val totalMemory = Runtime.getRuntime().totalMemory()
allocatedGauge.labels(backendKind, backendQualifier).set(totalMemory.toDouble())
val usedMemory = (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())
usedGauge.labels(backendKind, backendQualifier).set(usedMemory.toDouble())
try {
pg.push(registry, "jb_backend")
pg?.push(registry, "jb_backend")
} catch (t: Throwable) {
thisLogger().error("gitpod: failed to push monitoring metrics:", t)
}
Expand All @@ -108,7 +127,7 @@ class GitpodManager : Disposable {
.connectTimeout(Duration.ofSeconds(5))
.build()
val httpRequest = HttpRequest.newBuilder()
.uri(URI.create("http://localhost:24000/gatewayLink?backendPort=${backendPort}"))
.uri(URI.create("http://localhost:24000/gatewayLink?backendPort=$backendPort"))
.GET()
.build()
val response =
Expand Down Expand Up @@ -260,14 +279,14 @@ class GitpodManager : Disposable {
tokenResponse.token
)
} finally {
Thread.currentThread().contextClassLoader = originalClassLoader;
Thread.currentThread().contextClassLoader = originalClassLoader
}
}

val minReconnectionDelay = 2 * 1000L
val maxReconnectionDelay = 30 * 1000L
val reconnectionDelayGrowFactor = 1.5;
var reconnectionDelay = minReconnectionDelay;
val reconnectionDelayGrowFactor = 1.5
var reconnectionDelay = minReconnectionDelay
val gitpodHost = info.gitpodApi.host
var closeReason: Any = "cancelled"
try {
Expand Down
1 change: 1 addition & 0 deletions operations/observability/mixins/IDE/dashboards.libsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
// 'my-new-dashboard.json': (import 'dashboards/components/new-component.json'),
'gitpod-component-openvsx-proxy.json': (import 'dashboards/components/openvsx-proxy.json'),
'gitpod-component-ssh-gateway.json': (import 'dashboards/components/ssh-gateway.json'),
'gitpod-component-jb.json': (import 'dashboards/components/jb.json'),

},
}
232 changes: 232 additions & 0 deletions operations/observability/mixins/IDE/dashboards/components/jb.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,232 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 63,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "notification/second",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 11,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [
"max",
"mean"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"editorMode": "code",
"expr": "sum(rate(gitpod_jb_backend_low_memory_after_gc_total[5m]))",
"legendFormat": "rate",
"range": true,
"refId": "A"
}
],
"title": "Total low memory after GC",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "notification/second",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 11,
"w": 12,
"x": 12,
"y": 0
},
"id": 4,
"options": {
"legend": {
"calcs": [
"max",
"mean"
],
"displayMode": "table",
"placement": "right"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"editorMode": "code",
"expr": "topk(10, sum(rate(gitpod_jb_backend_low_memory_after_gc_total[5m])) by (pod))",
"legendFormat": "{{pod}}",
"range": true,
"refId": "A"
}
],
"title": "Top low memory after GC",
"type": "timeseries"
}
],
"schemaVersion": 36,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "JetBrains Overview",
"uid": "oamBLUC7k",
"version": 8,
"weekStart": ""
}