[JetBrains] Replace "Status" by "JetBrains Launcher" and run it in a separate image from IDEs

components/BUILD.yaml

@@ -41,6 +41,7 @@ packages:
       - components/ide/code-desktop:docker
       - components/ide/code-desktop:docker-insiders
       - components/ide/code:docker
+      - components/ide/jetbrains/launcher:docker
       - components/ide/jetbrains/backend-plugin:stable
       - components/ide/jetbrains/backend-plugin:latest
       - components/ide/jetbrains/image:goland
@@ -134,6 +135,7 @@ packages:
       - components/ws-manager:app
       - components/ide-metrics:app
       - components/ide-service:app
+      - components/ide/jetbrains/launcher:app
 scripts:
   - name: update-license-header
     description: Updates the license header in all source files

components/ide-service-api/go/config/ideconfig.go

@@ -52,13 +52,19 @@ type IDEOption struct {
 	// This is useful if this image points to a tag like `nightly` that will be updated regularly. When `resolveImageDigest` is `true`, we make sure that we resolve the tag regularly to the most recent image version.
 	ResolveImageDigest bool `json:"resolveImageDigest,omitempty"`
 	// PluginImage ref for the IDE image, this image ref always resolve to digest.
+	// DEPRECATED use ImageLayers instead
 	PluginImage string `json:"pluginImage,omitempty"`
 	// PluginLatestImage ref for the latest IDE image, this image ref always resolve to digest.
+	// DEPRECATED use LatestImageLayers instead
 	PluginLatestImage string `json:"pluginLatestImage,omitempty"`
 	// ImageVersion the semantic version of the IDE image.
 	ImageVersion string `json:"imageVersion,omitempty"`
 	// LatestImageVersion the semantic version of the latest IDE image.
 	LatestImageVersion string `json:"latestImageVersion,omitempty"`
+	// ImageLayers for additional ide layers and dependencies
+	ImageLayers []string `json:"imageLayers,omitempty"`
+	// LatestImageLayers for latest additional ide layers and dependencies
+	LatestImageLayers []string `json:"latestImageLayers,omitempty"`
 }
 
 type IDEClient struct {

components/ide-service/example-ide-config.json

@@ -26,8 +26,14 @@
         "logo": "https://ide.gitpod.io/image/ide-logo/golandLogo.svg",
         "image": "eu.gcr.io/gitpod-core-dev/build/ide/goland:commit-9a6c79a91b2b1f583d5bcb7f9f1ef54ee977e0df",
         "latestImage": "eu.gcr.io/gitpod-core-dev/build/ide/goland:latest@sha256:e07524e52089829dc8d3b38f7d18fb51b24f07aed7d8e4e6e447899687978d43",
-        "pluginImage": "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a",
-        "pluginLatestImage": "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a-latest"
+        "imageLayers": [
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a",
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-launcher:commit-b38092639d1783a1957894ddd4f492b3cdc9794a"
+        ],
+        "latestImageLayers": [
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a-latest",
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-launcher:commit-b38092639d1783a1957894ddd4f492b3cdc9794a"
+        ]
       },
       "intellij": {
         "orderKey": "04",
@@ -36,8 +42,14 @@
         "logo": "https://ide.gitpod.io/image/ide-logo/intellijIdeaLogo.svg",
         "image": "eu.gcr.io/gitpod-core-dev/build/ide/intellij:commit-9a6c79a91b2b1f583d5bcb7f9f1ef54ee977e0df",
         "latestImage": "eu.gcr.io/gitpod-core-dev/build/ide/intellij:latest@sha256:e07524e52089829dc8d3b38f7d18fb51b24f07aed7d8e4e6e447899687978d43",
-        "pluginImage": "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a",
-        "pluginLatestImage": "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a-latest"
+        "imageLayers": [
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a",
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-launcher:commit-b38092639d1783a1957894ddd4f492b3cdc9794a"
+        ],
+        "latestImageLayers": [
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a-latest",
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-launcher:commit-b38092639d1783a1957894ddd4f492b3cdc9794a"
+        ]
       },
       "phpstorm": {
         "orderKey": "07",
@@ -46,8 +58,14 @@
         "logo": "https://ide.gitpod.io/image/ide-logo/phpstormLogo.svg",
         "image": "eu.gcr.io/gitpod-core-dev/build/ide/phpstorm:commit-9a6c79a91b2b1f583d5bcb7f9f1ef54ee977e0df",
         "latestImage": "eu.gcr.io/gitpod-core-dev/build/ide/phpstorm:latest@sha256:e07524e52089829dc8d3b38f7d18fb51b24f07aed7d8e4e6e447899687978d43",
-        "pluginImage": "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a",
-        "pluginLatestImage": "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a-latest"
+        "imageLayers": [
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a",
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-launcher:commit-b38092639d1783a1957894ddd4f492b3cdc9794a"
+        ],
+        "latestImageLayers": [
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a-latest",
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-launcher:commit-b38092639d1783a1957894ddd4f492b3cdc9794a"
+        ]
       },
       "pycharm": {
         "orderKey": "06",
@@ -56,8 +74,14 @@
         "logo": "https://ide.gitpod.io/image/ide-logo/pycharmLogo.svg",
         "image": "eu.gcr.io/gitpod-core-dev/build/ide/pycharm:commit-9a6c79a91b2b1f583d5bcb7f9f1ef54ee977e0df",
         "latestImage": "eu.gcr.io/gitpod-core-dev/build/ide/pycharm:latest@sha256:e07524e52089829dc8d3b38f7d18fb51b24f07aed7d8e4e6e447899687978d43",
-        "pluginImage": "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a",
-        "pluginLatestImage": "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a-latest"
+        "imageLayers": [
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a",
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-launcher:commit-b38092639d1783a1957894ddd4f492b3cdc9794a"
+        ],
+        "latestImageLayers": [
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-b38092639d1783a1957894ddd4f492b3cdc9794a-latest",
+          "eu.gcr.io/gitpod-core-dev/build/ide/jb-launcher:commit-b38092639d1783a1957894ddd4f492b3cdc9794a"
+        ]
       }
     },
     "defaultIde": "code",

components/ide-service/pkg/server/server.go

@@ -188,20 +188,6 @@ type WorkspaceContext struct {
 	ReferrerIde string `json:"referrerIde,omitempty"`
 }
 
-var JetbrainsCode map[string]string
-
-func init() {
-	JetbrainsCode = make(map[string]string)
-	JetbrainsCode["intellij"] = "IIU"
-	JetbrainsCode["goland"] = "GO"
-	JetbrainsCode["pycharm"] = "PCP"
-	JetbrainsCode["phpstorm"] = "PS"
-	JetbrainsCode["rubymine"] = "RM"
-	JetbrainsCode["webstorm"] = "WS"
-	JetbrainsCode["rider"] = "RD"
-	JetbrainsCode["clion"] = "CL"
-}
-
 func (s *IDEServiceServer) resolveReferrerIDE(ideConfig *config.IDEConfig, wsCtx *WorkspaceContext, chosenIDEName string) (ideName string, ideOption *config.IDEOption) {
 	if wsCtx == nil || wsCtx.Referrer == "" {
 		return
@@ -264,166 +250,149 @@ func (s *IDEServiceServer) ResolveWorkspaceConfig(ctx context.Context, req *api.
 		WebImage:        defaultIde.Image,
 	}
 
-	// TODO: reconsider this
-	// if req.Type != api.WorkspaceType_REGULAR {
-	// 	return resp, nil
-	// }
-
 	var wsConfig *gitpodapi.GitpodConfig
-	var wsContext *WorkspaceContext
-	var ideSettings *IDESettings
 
-	if req.IdeSettings != "" {
-		if err := json.Unmarshal([]byte(req.IdeSettings), &ideSettings); err != nil {
-			log.WithError(err).WithField("ideSetting", req.IdeSettings).Error("failed to parse ide settings")
-		}
-	}
 	if req.WorkspaceConfig != "" {
 		if err := json.Unmarshal([]byte(req.WorkspaceConfig), &wsConfig); err != nil {
 			log.WithError(err).WithField("workspaceConfig", req.WorkspaceConfig).Error("failed to parse workspace config")
 		}
 	}
-	if req.Context != "" {
-		if err := json.Unmarshal([]byte(req.Context), &wsContext); err != nil {
-			log.WithError(err).WithField("context", req.Context).Error("failed to parse context")
-		}
-	}
 
-	userIdeName := ""
-	useLatest := false
+	if req.Type == api.WorkspaceType_REGULAR {
+		var ideSettings *IDESettings
+		var wsContext *WorkspaceContext
 
-	if ideSettings != nil {
-		userIdeName = ideSettings.DefaultIde
-		useLatest = ideSettings.UseLatestVersion
-	}
+		if req.IdeSettings != "" {
+			if err := json.Unmarshal([]byte(req.IdeSettings), &ideSettings); err != nil {
+				log.WithError(err).WithField("ideSetting", req.IdeSettings).Error("failed to parse ide settings")
+			}
+		}
 
-	chosenIDE := defaultIde
+		if req.Context != "" {
+			if err := json.Unmarshal([]byte(req.Context), &wsContext); err != nil {
+				log.WithError(err).WithField("context", req.Context).Error("failed to parse context")
+			}
+		}
 
-	getUserIDEImage := func(ideOption *config.IDEOption) string {
-		if useLatest && ideOption.LatestImage != "" {
-			return ideOption.LatestImage
+		userIdeName := ""
+		useLatest := false
+
+		if ideSettings != nil {
+			userIdeName = ideSettings.DefaultIde
+			useLatest = ideSettings.UseLatestVersion
 		}
 
-		return ideOption.Image
-	}
+		chosenIDE := defaultIde
 
-	getUserPluginImage := func(ideOption *config.IDEOption) string {
-		if useLatest && ideOption.PluginLatestImage != "" {
-			return ideOption.PluginLatestImage
+		getUserIDEImage := func(ideOption *config.IDEOption) string {
+			if useLatest && ideOption.LatestImage != "" {
+				return ideOption.LatestImage
+			}
+
+			return ideOption.Image
 		}
 
-		return ideOption.PluginImage
-	}
+		getUserImageLayers := func(ideOption *config.IDEOption) []string {
+			if useLatest {
+				return ideOption.LatestImageLayers
+			}
 
-	if userIdeName != "" {
-		if ide, ok := ideConfig.IdeOptions.Options[userIdeName]; ok {
-			chosenIDE = &ide
+			return ideOption.ImageLayers
+		}
 
-			// TODO: Currently this variable reflects the IDE selected in
-			// user's settings for backward compatibility but in the future
-			// we want to make it represent the actual IDE.
-			ideAlias := api.EnvironmentVariable{
-				Name:  "GITPOD_IDE_ALIAS",
-				Value: userIdeName,
+		if userIdeName != "" {
+			if ide, ok := ideConfig.IdeOptions.Options[userIdeName]; ok {
+				chosenIDE = &ide
+
+				// TODO: Currently this variable reflects the IDE selected in
+				// user's settings for backward compatibility but in the future
+				// we want to make it represent the actual IDE.
+				ideAlias := api.EnvironmentVariable{
+					Name:  "GITPOD_IDE_ALIAS",
+					Value: userIdeName,
+				}
+				resp.Envvars = append(resp.Envvars, &ideAlias)
 			}
-			resp.Envvars = append(resp.Envvars, &ideAlias)
 		}
-	}
 
-	// we always need WebImage for when the user chooses a desktop ide
-	resp.WebImage = getUserIDEImage(defaultIde)
+		// we always need WebImage for when the user chooses a desktop ide
+		resp.WebImage = getUserIDEImage(defaultIde)
 
-	var desktopImageLayer string
-	var desktopPluginImageLayer string
-	if chosenIDE.Type == config.IDETypeDesktop {
-		desktopImageLayer = getUserIDEImage(chosenIDE)
-		desktopPluginImageLayer = getUserPluginImage(chosenIDE)
-	} else {
-		resp.WebImage = getUserIDEImage(chosenIDE)
-	}
+		var desktopImageLayer string
+		var userImageLayers []string
+		if chosenIDE.Type == config.IDETypeDesktop {
+			desktopImageLayer = getUserIDEImage(chosenIDE)
+			userImageLayers = getUserImageLayers(chosenIDE)
+		} else {
+			resp.WebImage = getUserIDEImage(chosenIDE)
+		}
 
-	ideName, referrer := s.resolveReferrerIDE(ideConfig, wsContext, userIdeName)
-	if ideName != "" {
-		resp.RefererIde = ideName
-		desktopImageLayer = getUserIDEImage(referrer)
-		desktopPluginImageLayer = getUserPluginImage(referrer)
-	}
+		ideName, referrer := s.resolveReferrerIDE(ideConfig, wsContext, userIdeName)
+		if ideName != "" {
+			resp.RefererIde = ideName
+			desktopImageLayer = getUserIDEImage(referrer)
+			userImageLayers = getUserImageLayers(referrer)
+		}
 
-	if desktopImageLayer != "" {
-		resp.IdeImageLayers = append(resp.IdeImageLayers, desktopImageLayer)
-		if desktopPluginImageLayer != "" {
-			resp.IdeImageLayers = append(resp.IdeImageLayers, desktopPluginImageLayer)
+		if desktopImageLayer != "" {
+			resp.IdeImageLayers = append(resp.IdeImageLayers, desktopImageLayer)
+			resp.IdeImageLayers = append(resp.IdeImageLayers, userImageLayers...)
 		}
 	}
 
 	jbGW, ok := ideConfig.IdeOptions.Clients["jetbrains-gateway"]
 	if req.Type == api.WorkspaceType_PREBUILD && ok {
 		warmUpTask := ""
+		imageLayers := make(map[string]struct{})
 		for _, alias := range jbGW.DesktopIDEs {
 			prebuilds := getPrebuilds(wsConfig, alias)
 			if prebuilds != nil {
 				if prebuilds.Version != "latest" {
-					template := `
+					if ide, ok := ideConfig.IdeOptions.Options[alias]; ok {
+						for _, ideImageLayer := range ide.ImageLayers {
+							if _, ok := imageLayers[ideImageLayer]; !ok {
+								imageLayers[ideImageLayer] = struct{}{}
+								resp.IdeImageLayers = append(resp.IdeImageLayers, ideImageLayer)
+							}
+						}
+						resp.IdeImageLayers = append(resp.IdeImageLayers, ide.Image)
+						template := `
 echo 'warming up stable release of ${key}...'
-echo 'downloading stable ${key} backend...'
-mkdir /tmp/backend
-curl -sSLo /tmp/backend/backend.tar.gz "https://download.jetbrains.com/product?type=release&distribution=linux&code=${productCode}"
-tar -xf /tmp/backend/backend.tar.gz --strip-components=1 --directory /tmp/backend
-
-echo 'configuring JB system config and caches aligned with runtime...'
-printf '\nshared.indexes.download.auto.consent=true' >> "/tmp/backend/bin/idea.properties"
-unset JAVA_TOOL_OPTIONS
-export IJ_HOST_CONFIG_BASE_DIR=/workspace/.config/JetBrains
-export IJ_HOST_SYSTEM_BASE_DIR=/workspace/.cache/JetBrains
-
-echo 'running stable ${key} backend in warmup mode...'
-/tmp/backend/bin/remote-dev-server.sh warmup "$GITPOD_REPO_ROOT"
-
-echo 'removing stable ${key} backend...'
-rm -rf /tmp/backend
+JETBRAINS_BACKEND_QUALIFIER=stable /ide-desktop/jb-launcher warmup ${key}
 `
-					if code, ok := JetbrainsCode[alias]; ok {
 						template = strings.ReplaceAll(template, "${key}", alias)
-						template = strings.ReplaceAll(template, "${productCode}", code)
 						warmUpTask += template
 					}
 				}
 
 				if prebuilds.Version != "stable" {
-					template := `
+					if ide, ok := ideConfig.IdeOptions.Options[alias]; ok {
+						for _, latestIdeImageLayer := range ide.LatestImageLayers {
+							if _, ok := imageLayers[latestIdeImageLayer]; !ok {
+								imageLayers[latestIdeImageLayer] = struct{}{}
+								resp.IdeImageLayers = append(resp.IdeImageLayers, latestIdeImageLayer)
+							}
+						}
+						resp.IdeImageLayers = append(resp.IdeImageLayers, ide.LatestImage)
+						template := `
 echo 'warming up latest release of ${key}...'
-echo 'downloading latest ${key} backend...'
-mkdir /tmp/backend-latest
-curl -sSLo /tmp/backend-latest/backend-latest.tar.gz "https://download.jetbrains.com/product?type=release,eap,rc&distribution=linux&code=${productCode}"
-tar -xf /tmp/backend-latest/backend-latest.tar.gz --strip-components=1 --directory /tmp/backend-latest
-
-echo 'configuring JB system config and caches aligned with runtime...'
-printf '\nshared.indexes.download.auto.consent=true' >> "/tmp/backend-latest/bin/idea.properties"
-unset JAVA_TOOL_OPTIONS
-export IJ_HOST_CONFIG_BASE_DIR=/workspace/.config/JetBrains-latest
-export IJ_HOST_SYSTEM_BASE_DIR=/workspace/.cache/JetBrains-latest
-
-echo 'running ${key} backend in warmup mode...'
-/tmp/backend-latest/bin/remote-dev-server.sh warmup "$GITPOD_REPO_ROOT"
-
-echo 'removing latest ${key} backend...'
-rm -rf /tmp/backend-latest
+JETBRAINS_BACKEND_QUALIFIER=latest /ide-desktop/jb-launcher warmup ${key}
 `
-					if code, ok := JetbrainsCode[alias]; ok {
 						template = strings.ReplaceAll(template, "${key}", alias)
-						template = strings.ReplaceAll(template, "${productCode}", code)
 						warmUpTask += template
 					}
 				}
 			}
 		}
+
 		if warmUpTask != "" {
 			warmUpEncoded := new(bytes.Buffer)
 			enc := json.NewEncoder(warmUpEncoded)
 			enc.SetEscapeHTML(false)
 
 			err := enc.Encode(&[]gitpodapi.TaskConfig{{
 				Init: strings.TrimSpace(warmUpTask),
+				Name: "GITPOD_JB_WARMUP_TASK",
 			}})
 			if err != nil {
 				log.WithError(err).Error("cannot marshal warm up task")
@@ -432,6 +401,7 @@ rm -rf /tmp/backend-latest
 			resp.Tasks = warmUpEncoded.String()
 		}
 	}
+
 	return
 }