Solve "cannot download OTS" - by removing OTS for secrets

components/content-service-api/go/go.mod

@@ -4,8 +4,10 @@ go 1.18
 
 require (
 	github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
+	github.com/google/go-cmp v0.5.7
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.2
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
 	google.golang.org/grpc v1.45.0
 	google.golang.org/protobuf v1.28.0
 )
@@ -33,7 +35,6 @@ require (
 	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )

components/content-service-api/go/go.sum

@@ -120,6 +120,7 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=

components/content-service-api/go/initializer.go

@@ -0,0 +1,132 @@
+// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License-AGPL.txt in the project root for license information.
+
+package api
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"golang.org/x/xerrors"
+)
+
+// GetCheckoutLocationsFromInitializer returns a list of all checkout locations from the initializer
+func GetCheckoutLocationsFromInitializer(init *WorkspaceInitializer) []string {
+	var res []string
+	_ = WalkInitializer(nil, init, func(path []string, init *WorkspaceInitializer) error {
+		switch spec := init.Spec.(type) {
+		case *WorkspaceInitializer_Git:
+			res = append(res, spec.Git.CheckoutLocation)
+		case *WorkspaceInitializer_Backup:
+			res = append(res, spec.Backup.CheckoutLocation)
+
+		case *WorkspaceInitializer_Prebuild:
+			// walkInitializer will visit the Git initializer
+		}
+
+		return nil
+	})
+	return res
+}
+
+const extractedSecretPrefix = "extracted-secret/"
+
+// ExtractSecretsFromInitializer removes secrets to the initializer.
+// This is the counterpart of InjectSecretsToInitializer.
+func ExtractSecretsFromInitializer(init *WorkspaceInitializer) map[string]string {
+	res := make(map[string]string)
+
+	_ = WalkInitializer([]string{"initializer"}, init, func(path []string, init *WorkspaceInitializer) error {
+		git, ok := init.Spec.(*WorkspaceInitializer_Git)
+		if !ok {
+			return nil
+		}
+
+		pwd := git.Git.Config.AuthPassword
+		if pwd == "" || strings.HasPrefix(pwd, extractedSecretPrefix) {
+			return nil
+		}
+
+		name := strings.Join(path, ".")
+		res[name] = pwd
+		git.Git.Config.AuthPassword = extractedSecretPrefix + name
+
+		return nil
+	})
+
+	return res
+}
+
+// InjectSecretsToInitializer injects secrets to the initializer. This is the counterpart of ExtractSecretsFromInitializer.
+func InjectSecretsToInitializer(init *WorkspaceInitializer, secrets map[string][]byte) error {
+	return WalkInitializer([]string{"initializer"}, init, func(path []string, init *WorkspaceInitializer) error {
+		git, ok := init.Spec.(*WorkspaceInitializer_Git)
+		if !ok {
+			return nil
+		}
+
+		pwd := git.Git.Config.AuthPassword
+		if !strings.HasPrefix(pwd, extractedSecretPrefix) {
+			return nil
+		}
+
+		name := strings.TrimPrefix(pwd, extractedSecretPrefix)
+		val, ok := secrets[name]
+		if !ok {
+			return xerrors.Errorf("secret %s not found", name)
+		}
+
+		git.Git.Config.AuthPassword = string(val)
+
+		return nil
+	})
+}
+
+// WalkInitializer walks the initializer structure
+func WalkInitializer(path []string, init *WorkspaceInitializer, visitor func(path []string, init *WorkspaceInitializer) error) error {
+	if init == nil {
+		return nil
+	}
+
+	switch spec := init.Spec.(type) {
+	case *WorkspaceInitializer_Backup:
+		return visitor(append(path, "backup"), init)
+	case *WorkspaceInitializer_Composite:
+		path = append(path, "composite")
+		err := visitor(path, init)
+		if err != nil {
+			return err
+		}
+		for i, p := range spec.Composite.Initializer {
+			err := WalkInitializer(append(path, strconv.Itoa(i)), p, visitor)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	case *WorkspaceInitializer_Download:
+		return visitor(append(path, "download"), init)
+	case *WorkspaceInitializer_Empty:
+		return visitor(append(path, "empty"), init)
+	case *WorkspaceInitializer_Git:
+		return visitor(append(path, "git"), init)
+	case *WorkspaceInitializer_Prebuild:
+		child := append(path, "prebuild")
+		err := visitor(child, init)
+		if err != nil {
+			return err
+		}
+		for i, g := range spec.Prebuild.Git {
+			err = WalkInitializer(append(child, strconv.Itoa(i)), &WorkspaceInitializer{Spec: &WorkspaceInitializer_Git{Git: g}}, visitor)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+
+	default:
+		return fmt.Errorf("unsupported workspace initializer in walkInitializer - this is a bug in Gitpod")
+	}
+}

components/content-service-api/go/initializer_test.go

@@ -0,0 +1,219 @@
+// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License-AGPL.txt in the project root for license information.
+
+package api_test
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/gitpod-io/gitpod/content-service/api"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestGetCheckoutLocationsFromInitializer(t *testing.T) {
+	var init []*api.WorkspaceInitializer
+	init = append(init, &api.WorkspaceInitializer{
+		Spec: &api.WorkspaceInitializer_Git{
+			Git: &api.GitInitializer{
+				CheckoutLocation: "/foo",
+				CloneTaget:       "head",
+				Config: &api.GitConfig{
+					Authentication: api.GitAuthMethod_NO_AUTH,
+				},
+				RemoteUri:  "somewhere-else",
+				TargetMode: api.CloneTargetMode_LOCAL_BRANCH,
+			},
+		},
+	})
+	init = append(init, &api.WorkspaceInitializer{
+		Spec: &api.WorkspaceInitializer_Git{
+			Git: &api.GitInitializer{
+				CheckoutLocation: "/bar",
+				CloneTaget:       "head",
+				Config: &api.GitConfig{
+					Authentication: api.GitAuthMethod_NO_AUTH,
+				},
+				RemoteUri:  "somewhere-else",
+				TargetMode: api.CloneTargetMode_LOCAL_BRANCH,
+			},
+		},
+	})
+
+	tests := []struct {
+		Name        string
+		Initializer *api.WorkspaceInitializer
+		Expectation string
+	}{
+		{
+			Name: "single git initializer",
+			Initializer: &api.WorkspaceInitializer{
+				Spec: &api.WorkspaceInitializer_Git{
+					Git: &api.GitInitializer{
+						CheckoutLocation: "/foo",
+						CloneTaget:       "head",
+						Config: &api.GitConfig{
+							Authentication: api.GitAuthMethod_NO_AUTH,
+						},
+						RemoteUri:  "somewhere-else",
+						TargetMode: api.CloneTargetMode_LOCAL_BRANCH,
+					},
+				},
+			},
+			Expectation: "/foo",
+		},
+		{
+			Name: "multiple git initializer",
+			Initializer: &api.WorkspaceInitializer{
+				Spec: &api.WorkspaceInitializer_Composite{
+					Composite: &api.CompositeInitializer{
+						Initializer: init,
+					},
+				},
+			},
+			Expectation: "/foo,/bar",
+		},
+		{
+			Name: "backup initializer",
+			Initializer: &api.WorkspaceInitializer{
+				Spec: &api.WorkspaceInitializer_Backup{
+					Backup: &api.FromBackupInitializer{
+						CheckoutLocation: "/foobar",
+					},
+				},
+			},
+			Expectation: "/foobar",
+		},
+		{
+			Name: "prebuild initializer",
+			Initializer: &api.WorkspaceInitializer{
+				Spec: &api.WorkspaceInitializer_Prebuild{
+					Prebuild: &api.PrebuildInitializer{
+						Git: []*api.GitInitializer{
+							{CheckoutLocation: "/foo"},
+							{CheckoutLocation: "/bar"},
+						},
+					},
+				},
+			},
+			Expectation: "/foo,/bar",
+		},
+		{
+			Name: "nil initializer",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			locations := strings.Join(api.GetCheckoutLocationsFromInitializer(test.Initializer), ",")
+			if locations != test.Expectation {
+				t.Errorf("expected %s, got %s", test.Expectation, locations)
+			}
+		})
+	}
+}
+
+func TestExtractInjectSecretsFromInitializer(t *testing.T) {
+	tests := []struct {
+		Name        string
+		Input       *api.WorkspaceInitializer
+		Expectation map[string]string
+	}{
+		{
+			Name: "git initializer",
+			Input: &api.WorkspaceInitializer{
+				Spec: &api.WorkspaceInitializer_Git{
+					Git: &api.GitInitializer{
+						Config: &api.GitConfig{
+							AuthPassword: "foobar",
+						},
+					},
+				},
+			},
+			Expectation: map[string]string{
+				"initializer.git": "foobar",
+			},
+		},
+		{
+			Name: "no secret git initializer",
+			Input: &api.WorkspaceInitializer{
+				Spec: &api.WorkspaceInitializer_Git{
+					Git: &api.GitInitializer{
+						Config: &api.GitConfig{},
+					},
+				},
+			},
+			Expectation: map[string]string{},
+		},
+		{
+			Name: "prebuild initializer",
+			Input: &api.WorkspaceInitializer{
+				Spec: &api.WorkspaceInitializer_Prebuild{
+					Prebuild: &api.PrebuildInitializer{
+						Git: []*api.GitInitializer{
+							{
+								Config: &api.GitConfig{
+									AuthPassword: "foobar",
+								},
+							},
+							{
+								Config: &api.GitConfig{
+									AuthPassword: "some value",
+								},
+							},
+						},
+					},
+				},
+			},
+			Expectation: map[string]string{
+				"initializer.prebuild.0.git": "foobar",
+				"initializer.prebuild.1.git": "some value",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			act := api.ExtractSecretsFromInitializer(test.Input)
+			if diff := cmp.Diff(test.Expectation, act); diff != "" {
+				t.Errorf("unexpected ExtractSecretsFromInitializer (-want +got):\n%s", diff)
+			}
+
+			_ = api.WalkInitializer(nil, test.Input, func(path []string, init *api.WorkspaceInitializer) error {
+				git, ok := init.Spec.(*api.WorkspaceInitializer_Git)
+				if !ok {
+					return nil
+				}
+				if pwd := git.Git.Config.AuthPassword; pwd != "" && !strings.HasPrefix(pwd, "extracted-secret/") {
+					t.Errorf("expected authPassword to be extracted, but got %s at %s", pwd, filepath.Join(path...))
+				}
+
+				return nil
+			})
+
+			injection := make(map[string][]byte, len(act))
+			for k, v := range act {
+				injection[k] = []byte(v)
+			}
+
+			err := api.InjectSecretsToInitializer(test.Input, injection)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			_ = api.WalkInitializer(nil, test.Input, func(path []string, init *api.WorkspaceInitializer) error {
+				git, ok := init.Spec.(*api.WorkspaceInitializer_Git)
+				if !ok {
+					return nil
+				}
+				if pwd := git.Git.Config.AuthPassword; pwd != "" && strings.HasPrefix(pwd, "extracted-secret/") {
+					t.Errorf("expected authPassword to be injected, but got %s at %s", pwd, filepath.Join(path...))
+				}
+
+				return nil
+			})
+		})
+	}
+}