From dcf3c2b4f8ab220c01fe7f890a0f3d3cba48cb9a Mon Sep 17 00:00:00 2001 From: Aleksandar Aleksandrov Date: Thu, 3 Nov 2022 10:09:48 +0000 Subject: [PATCH] Refactor previewctl install-context --- dev/preview/previewctl/cmd/credentials.go | 154 +++++------ dev/preview/previewctl/cmd/install_context.go | 81 +++++- dev/preview/previewctl/cmd/list_previews.go | 4 +- dev/preview/previewctl/cmd/root.go | 20 +- dev/preview/previewctl/go.mod | 25 +- dev/preview/previewctl/go.sum | 52 ++-- dev/preview/previewctl/main.go | 2 +- dev/preview/previewctl/pkg/gcloud/config.go | 62 +---- dev/preview/previewctl/pkg/k8s/config.go | 86 +++--- dev/preview/previewctl/pkg/k8s/context.go | 101 +++++++ .../previewctl/pkg/k8s/context/gke/dev.go | 104 +++++++ .../pkg/k8s/context/gke/dev_test.go | 101 +++++++ .../pkg/k8s/context/harvester/harvester.go | 89 ++++++ .../k8s/context/harvester/harvester_test.go | 127 +++++++++ .../previewctl/pkg/k8s/context/k3s/k3s.go | 254 ++++++++++++++++++ .../pkg/k8s/context/k3s/k3s_test.go | 119 ++++++++ .../previewctl/pkg/k8s/context/loader.go | 15 ++ dev/preview/previewctl/pkg/k8s/harvester.go | 41 --- dev/preview/previewctl/pkg/k8s/pforward.go | 61 +++++ dev/preview/previewctl/pkg/k8s/service.go | 6 +- .../previewctl/pkg/k8s/service_test.go | 2 +- dev/preview/previewctl/pkg/k8s/vm.go | 8 +- dev/preview/previewctl/pkg/k8s/vm_test.go | 4 +- dev/preview/previewctl/pkg/preview/preview.go | 89 ++++-- dev/preview/previewctl/pkg/ssh/mock.go | 39 +++ dev/preview/previewctl/pkg/ssh/ssh.go | 80 ++++++ 26 files changed, 1410 insertions(+), 316 deletions(-) create mode 100644 dev/preview/previewctl/pkg/k8s/context.go create mode 100644 dev/preview/previewctl/pkg/k8s/context/gke/dev.go create mode 100644 dev/preview/previewctl/pkg/k8s/context/gke/dev_test.go create mode 100644 dev/preview/previewctl/pkg/k8s/context/harvester/harvester.go create mode 100644 dev/preview/previewctl/pkg/k8s/context/harvester/harvester_test.go create mode 100644 dev/preview/previewctl/pkg/k8s/context/k3s/k3s.go create mode 100644 dev/preview/previewctl/pkg/k8s/context/k3s/k3s_test.go create mode 100644 dev/preview/previewctl/pkg/k8s/context/loader.go delete mode 100644 dev/preview/previewctl/pkg/k8s/harvester.go create mode 100644 dev/preview/previewctl/pkg/k8s/pforward.go create mode 100644 dev/preview/previewctl/pkg/ssh/mock.go create mode 100644 dev/preview/previewctl/pkg/ssh/ssh.go diff --git a/dev/preview/previewctl/cmd/credentials.go b/dev/preview/previewctl/cmd/credentials.go index 645726fb69d5fc..ec619e1b00002d 100644 --- a/dev/preview/previewctl/cmd/credentials.go +++ b/dev/preview/previewctl/cmd/credentials.go @@ -6,151 +6,127 @@ package cmd import ( "context" - "fmt" + "os" + "path/filepath" "github.com/cockroachdb/errors" + kctx "github.com/gitpod-io/gitpod/previewctl/pkg/k8s/context" + "github.com/gitpod-io/gitpod/previewctl/pkg/k8s/context/gke" + "github.com/gitpod-io/gitpod/previewctl/pkg/k8s/context/harvester" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/client-go/util/homedir" - "github.com/gitpod-io/gitpod/previewctl/pkg/gcloud" kube "github.com/gitpod-io/gitpod/previewctl/pkg/k8s" ) var ( - serviceAccountPath string - kubeConfigSavePath string + DefaultKubeConfigPath = filepath.Join(homedir.HomeDir(), clientcmd.RecommendedHomeDir, clientcmd.RecommendedFileName) ) const ( - coreDevClusterName = "core-dev" - coreDevProjectID = "gitpod-core-dev" - coreDevClusterZone = "europe-west1-b" - coreDevDesiredContextName = "dev" + coreDevClusterName = "core-dev" + coreDevProjectID = "gitpod-core-dev" + coreDevClusterZone = "europe-west1-b" ) type getCredentialsOpts struct { - gcpClient *gcloud.Config - logger *logrus.Logger + logger *logrus.Logger - getCredentialsMap map[string]func(ctx context.Context) (*api.Config, error) - configMap map[string]*api.Config + serviceAccountPath string + kubeConfigSavePath string } func newGetCredentialsCommand(logger *logrus.Logger) *cobra.Command { - var err error - var client *gcloud.Config ctx := context.Background() opts := &getCredentialsOpts{ - logger: logger, - configMap: map[string]*api.Config{}, + logger: logger, } cmd := &cobra.Command{ Use: "get-credentials", Long: `previewctl get-credentials retrieves the kubernetes configs for core-dev and harvester clusters, -merges them with the default config, and outputs them either to stdout or to a file.`, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - client, err = gcloud.New(ctx, serviceAccountPath) +merges them with the default config, and saves them to the path in KUBECONFIG or the default path '~/.kube/config'"`, + RunE: func(cmd *cobra.Command, args []string) error { + configs, err := opts.getCredentials(ctx) if err != nil { return err } - opts.gcpClient = client - opts.getCredentialsMap = map[string]func(ctx context.Context) (*api.Config, error){ - "dev": opts.getCoreDevKubeConfig, - "harvester": opts.getHarvesterKubeConfig, - } - - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - for _, kc := range []string{coreDevDesiredContextName, "harvester"} { - if ok := hasAccess(logger, kc); !ok { - config, err := opts.getCredentialsMap[kc](ctx) - if err != nil { - return err - } - - opts.configMap[kc] = config - } - } - - return opts.mergeContexts() + opts.kubeConfigSavePath = getKubeConfigPath() + return kube.OutputContext(opts.kubeConfigSavePath, configs) }, } - cmd.PersistentFlags().StringVar(&serviceAccountPath, "gcp-service-account", "", "path to the GCP service account to use") - cmd.PersistentFlags().StringVar(&kubeConfigSavePath, "kube-save-path", "", "path to save the generated kubeconfig to") + cmd.PersistentFlags().StringVar(&opts.serviceAccountPath, "gcp-service-account", "", "path to the GCP service account to use") return cmd } -func hasAccess(logger *logrus.Logger, contextName string) bool { - config, err := kube.NewFromDefaultConfigWithContext(logger, contextName) - if err != nil { - if errors.Is(err, kube.ErrContextNotExists) { - return false - } - - logger.Fatal(err) - } - - return config.HasAccess() -} - -func (o *getCredentialsOpts) mergeContexts() error { - var err error - configs := make([]*api.Config, 0, len(o.configMap)) - - for _, config := range o.configMap { - configs = append(configs, config) - } +func (o *getCredentialsOpts) getCredentials(ctx context.Context) (*api.Config, error) { + gkeLoader, err := gke.New(ctx, gke.ConfigLoaderOpts{ + Logger: o.logger, + ServiceAccountPath: o.serviceAccountPath, + Name: coreDevClusterName, + ProjectID: coreDevProjectID, + Zone: coreDevClusterZone, + RenamedContextName: gke.DevContextName, + }) - finalConfig, err := kube.MergeWithDefaultConfig(configs...) if err != nil { - return err + return nil, errors.Wrap(err, "failed to instantiate gke loader") } - if kubeConfigSavePath != "" { - return clientcmd.WriteToFile(*finalConfig, kubeConfigSavePath) + loaderMap := map[string]kctx.Loader{ + gke.DevContextName: gkeLoader, + harvester.ContextName: &harvester.ConfigLoader{}, } - bytes, err := clientcmd.Write(*finalConfig) - if err != nil { - return err - } + for _, contextName := range []string{gke.DevContextName, harvester.ContextName} { + loader := loaderMap[contextName] + if kc, err := kube.NewFromDefaultConfigWithContext(o.logger, contextName); err == nil && kc.HasAccess(ctx) { + continue + } - fmt.Println(string(bytes)) + kc, err := loader.Load(ctx) + if err != nil { + return nil, err + } - return err -} + configs, err := kube.MergeContextsWithDefault(kc) + if err != nil { + return nil, err + } -func (o *getCredentialsOpts) getCoreDevKubeConfig(ctx context.Context) (*api.Config, error) { - coreDevConfig, err := o.gcpClient.GenerateConfig(ctx, coreDevClusterName, coreDevProjectID, coreDevClusterZone, coreDevDesiredContextName) - if err != nil { - return nil, err + // always save the context at the default path + err = kube.OutputContext(DefaultKubeConfigPath, configs) + if err != nil { + return nil, err + } } - return coreDevConfig, nil + return kube.MergeContextsWithDefault() } -func (o *getCredentialsOpts) getHarvesterKubeConfig(ctx context.Context) (*api.Config, error) { - coreDevClientConfig, err := clientcmd.NewNonInteractiveClientConfig(*o.configMap[coreDevDesiredContextName], coreDevDesiredContextName, nil, nil).ClientConfig() +func hasAccess(ctx context.Context, logger *logrus.Logger, contextName string) bool { + config, err := kube.NewFromDefaultConfigWithContext(logger, contextName) if err != nil { - return nil, err - } + if errors.Is(err, kube.ErrContextNotExists) { + return false + } - kubeConfig, err := kube.NewWithConfig(o.logger, coreDevClientConfig) - if err != nil { - return nil, err + logger.Fatal(err) } - harvesterConfig, err := kubeConfig.GetHarvesterKubeConfig(ctx) - if err != nil { - return nil, err + return config.HasAccess(ctx) +} + +func getKubeConfigPath() string { + if v := os.Getenv("KUBECONFIG"); v != "" { + DefaultKubeConfigPath = v } - return harvesterConfig, nil + return DefaultKubeConfigPath } diff --git a/dev/preview/previewctl/cmd/install_context.go b/dev/preview/previewctl/cmd/install_context.go index 8b1125bc4f65df..563cc696b60592 100644 --- a/dev/preview/previewctl/cmd/install_context.go +++ b/dev/preview/previewctl/cmd/install_context.go @@ -5,25 +5,56 @@ package cmd import ( + "context" + "fmt" + "io/fs" + "os" "time" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "k8s.io/client-go/util/homedir" + kube "github.com/gitpod-io/gitpod/previewctl/pkg/k8s" "github.com/gitpod-io/gitpod/previewctl/pkg/preview" ) -var ( - watch bool - timeout time.Duration -) +type installContextCmdOpts struct { + logger *logrus.Logger + + watch bool + timeout time.Duration + kubeConfigSavePath string + sshPrivateKeyPath string + + getCredentialsOpts *getCredentialsOpts +} -func installContextCmd(logger *logrus.Logger) *cobra.Command { +func newInstallContextCmd(logger *logrus.Logger) *cobra.Command { + ctx := context.Background() + + opts := installContextCmdOpts{ + logger: logger, + getCredentialsOpts: &getCredentialsOpts{ + logger: logger, + }, + } // Used to ensure that we only install contexts var lastSuccessfulPreviewEnvironment *preview.Preview = nil install := func(timeout time.Duration) error { + name, err := preview.GetName(branch) + if err != nil { + return err + } + + if hasAccess(ctx, logger, name) { + opts.logger.Debugf("Access to [%s] already configured and connections can be established", name) + return nil + } + p, err := preview.New(branch, logger) if err != nil { @@ -35,19 +66,44 @@ func installContextCmd(logger *logrus.Logger) *cobra.Command { return nil } - err = p.InstallContext(true, timeout) + err = p.InstallContext(ctx, preview.InstallCtxOpts{ + Wait: opts.watch, + Timeout: opts.timeout, + KubeSavePath: opts.kubeConfigSavePath, + SSHPrivateKeyPath: opts.sshPrivateKeyPath, + }) + if err == nil { lastSuccessfulPreviewEnvironment = p } + return err } cmd := &cobra.Command{ Use: "install-context", Short: "Installs the kubectl context of a preview environment.", - RunE: func(cmd *cobra.Command, args []string) error { + PreRunE: func(cmd *cobra.Command, args []string) error { + configs, err := opts.getCredentialsOpts.getCredentials(ctx) + if err != nil { + return err + } + + opts.kubeConfigSavePath = getKubeConfigPath() + + err = kube.OutputContext(opts.kubeConfigSavePath, configs) + if err != nil { + return err + } - if watch { + if _, err = os.Stat(opts.sshPrivateKeyPath); errors.Is(err, fs.ErrNotExist) { + return preview.InstallVMSSHKeys() + } + + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + if opts.watch { for range time.Tick(15 * time.Second) { // We're using a short timeout here to handle the scenario where someone switches // to a branch that doens't have a preview envrionment. In that case the default @@ -59,14 +115,17 @@ func installContextCmd(logger *logrus.Logger) *cobra.Command { } } } else { - return install(timeout) + return install(opts.timeout) } return nil }, } - cmd.Flags().BoolVar(&watch, "watch", false, "If watch is enabled, previewctl will keep trying to install the kube-context every 15 seconds.") - cmd.Flags().DurationVarP(&timeout, "timeout", "t", 10*time.Minute, "Timeout before considering the installation failed") + cmd.Flags().BoolVar(&opts.watch, "watch", false, "If watch is enabled, previewctl will keep trying to install the kube-context every 15 seconds.") + cmd.Flags().DurationVarP(&opts.timeout, "timeout", "t", 10*time.Minute, "Timeout before considering the installation failed") + cmd.PersistentFlags().StringVar(&opts.sshPrivateKeyPath, "private-key-path", fmt.Sprintf("%s/.ssh/vm_id_rsa", homedir.HomeDir()), "path to the private key used to authenticate with the VM") + cmd.PersistentFlags().StringVar(&opts.getCredentialsOpts.serviceAccountPath, "gcp-service-account", "", "path to the GCP service account to use") + return cmd } diff --git a/dev/preview/previewctl/cmd/list_previews.go b/dev/preview/previewctl/cmd/list_previews.go index 4ae041d6017b27..0aa30b192fe68b 100644 --- a/dev/preview/previewctl/cmd/list_previews.go +++ b/dev/preview/previewctl/cmd/list_previews.go @@ -5,6 +5,8 @@ package cmd import ( + "context" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -26,7 +28,7 @@ func listPreviewsCmd(logger *logrus.Logger) *cobra.Command { return err } - err = p.ListAllPreviews() + err = p.ListAllPreviews(context.Background()) if err != nil { logger.WithFields(logrus.Fields{"err": err}).Fatal("Failed to list previews.") } diff --git a/dev/preview/previewctl/cmd/root.go b/dev/preview/previewctl/cmd/root.go index 67f073e4fd260a..497a7edc2c4ef4 100644 --- a/dev/preview/previewctl/cmd/root.go +++ b/dev/preview/previewctl/cmd/root.go @@ -10,20 +10,32 @@ import ( ) var ( - branch = "" + branch = "" + logLevel = "" ) -func RootCmd(logger *logrus.Logger) *cobra.Command { +func NewRootCmd(logger *logrus.Logger) *cobra.Command { cmd := &cobra.Command{ - Use: "previewctl", + Use: "previewctl", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + lvl, err := logrus.ParseLevel(logLevel) + if err != nil { + return err + } + + logger.SetLevel(lvl) + + return nil + }, Short: "Your best friend when interacting with Preview Environments :)", Long: `previewctl is your best friend when interacting with Preview Environments :)`, } cmd.PersistentFlags().StringVar(&branch, "branch", "", "From which branch's preview previewctl should interact with. By default it will use the result of \"git rev-parse --abbrev-ref HEAD\"") + cmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", "The logger's log level") cmd.AddCommand( - installContextCmd(logger), + newInstallContextCmd(logger), newGetNameCmd(branch), listPreviewsCmd(logger), SSHPreviewCmd(logger), diff --git a/dev/preview/previewctl/go.mod b/dev/preview/previewctl/go.mod index 3d8504a1a9979c..bdfa174d08d86a 100644 --- a/dev/preview/previewctl/go.mod +++ b/dev/preview/previewctl/go.mod @@ -5,22 +5,24 @@ go 1.19 require ( github.com/cockroachdb/errors v1.9.0 github.com/imdario/mergo v0.3.13 - github.com/spf13/cobra v1.6.0 + github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.0 - google.golang.org/api v0.99.0 + golang.org/x/crypto v0.1.0 + google.golang.org/api v0.102.0 k8s.io/api v0.25.3 k8s.io/apimachinery v0.25.3 kubevirt.io/api v0.58.0 ) require ( - cloud.google.com/go/compute v1.10.0 // indirect + cloud.google.com/go/compute v1.12.1 // indirect + cloud.google.com/go/compute/metadata v0.2.1 // indirect github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect github.com/cockroachdb/redact v1.1.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect - github.com/getsentry/sentry-go v0.12.0 // indirect + github.com/getsentry/sentry-go v0.14.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect @@ -36,9 +38,10 @@ require ( github.com/googleapis/gax-go/v2 v2.6.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kr/pretty v0.3.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -47,16 +50,16 @@ require ( github.com/pborman/uuid v1.2.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rogpeppe/go-internal v1.8.1 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/net v0.0.0-20221017152216-f25eb7ecb193 // indirect - golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect + golang.org/x/net v0.1.0 // indirect + golang.org/x/oauth2 v0.1.0 // indirect golang.org/x/sys v0.1.0 // indirect - golang.org/x/term v0.0.0-20221017184919-83659145692c // indirect + golang.org/x/term v0.1.0 // indirect golang.org/x/text v0.4.0 // indirect golang.org/x/time v0.1.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20221018143404-92eef740a0dc // indirect + google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect google.golang.org/grpc v1.50.1 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect @@ -65,7 +68,7 @@ require ( k8s.io/apiextensions-apiserver v0.25.3 // indirect k8s.io/klog/v2 v2.80.1 // indirect k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect - k8s.io/utils v0.0.0-20221012122500-cfd413dd9e85 // indirect + k8s.io/utils v0.0.0-20221101230645-61b03e2f6476 // indirect kubevirt.io/containerized-data-importer-api v1.55.0 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect diff --git a/dev/preview/previewctl/go.sum b/dev/preview/previewctl/go.sum index f0574cf827da31..ffd388ffb8ad70 100644 --- a/dev/preview/previewctl/go.sum +++ b/dev/preview/previewctl/go.sum @@ -1,7 +1,11 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go/compute v1.10.0 h1:aoLIYaA1fX3ywihqpBk2APQKOo20nXsp1GEZQbx5Jk4= -cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= +cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/longrunning v0.1.1 h1:y50CXG4j0+qvEukslYFBCrzaXX0qpFbBzc3PchSu/LE= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= @@ -18,6 +22,7 @@ github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqR github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= @@ -63,6 +68,7 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -87,14 +93,15 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= -github.com/getsentry/sentry-go v0.12.0 h1:era7g0re5iY13bHSdN/xMkyV+5zZppjRVQhZrXCaEIk= github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= +github.com/getsentry/sentry-go v0.14.0 h1:rlOBkuFZRKKdUnKO+0U3JclRDQKlRu5vVQtkWSQvC70= +github.com/getsentry/sentry-go v0.14.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -230,8 +237,9 @@ github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -260,6 +268,7 @@ github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -317,8 +326,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -336,8 +346,8 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= -github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -390,6 +400,8 @@ golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -429,12 +441,12 @@ golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20221017152216-f25eb7ecb193 h1:3Moaxt4TfzNcQH6DWvlYKraN1ozhBXQHcgvXjRGeim0= -golang.org/x/net v0.0.0-20221017152216-f25eb7ecb193/go.mod h1:RpDiru2p0u2F0lLpEoqnP2+7xs0ifAuOcJ442g6GU2s= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.1.0 h1:isLCZuhj4v+tYv7eskaN4v/TM+A1begWWgyVJDdl1+Y= +golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -482,8 +494,8 @@ golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20221017184919-83659145692c h1:dveknrit5futqEmXAvd2I1BbZIDhxRijsyWHM86NlcA= -golang.org/x/term v0.0.0-20221017184919-83659145692c/go.mod h1:VTIZ7TEbF0BS9Sv9lPTvGbtW8i4z6GGbJBCM37uMCzY= +golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -520,8 +532,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -google.golang.org/api v0.99.0 h1:tsBtOIklCE2OFxhmcYSVqGwSAN/Y897srxmcvAQnwK8= -google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= +google.golang.org/api v0.102.0 h1:JxJl2qQ85fRMPNvlZY/enexbxpCjLwGhZUtgfGeQ51I= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= @@ -534,8 +546,8 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20221018143404-92eef740a0dc h1:JcyYf3ta+dc+R9bjMX0U+RE/7wsOZ2/2FQa4VhElZ4A= -google.golang.org/genproto v0.0.0-20221018143404-92eef740a0dc/go.mod h1:45EK0dUbEZ2NHjCeAd2LXmyjAgGUGrpGROgjhC3ADck= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c h1:QgY/XxIAIeccR+Ca/rDdKubLIU9rcJ3xfy1DC/Wd2Oo= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= @@ -625,8 +637,8 @@ k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhkl k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20221012122500-cfd413dd9e85 h1:cTdVh7LYu82xeClmfzGtgyspNh6UxpwLWGi8R4sspNo= -k8s.io/utils v0.0.0-20221012122500-cfd413dd9e85/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20221101230645-61b03e2f6476 h1:L14f2LWkOxG2rYsuSA3ltQnnST1vMfek/GUk+VemxD4= +k8s.io/utils v0.0.0-20221101230645-61b03e2f6476/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v0.58.0 h1:qeNeRtD6AIJ5WVJuRXajmmXtnrO5dYchy+hpCm6QwhE= kubevirt.io/api v0.58.0/go.mod h1:U0CQlZR0JoJCaC+Va0wz4dMOtYDdVywJ98OT1KmOkzI= kubevirt.io/containerized-data-importer-api v1.55.0 h1:IQNc8PYVq1cTwKNPEJza5xSlcnXeYVNt76M5kZ8X7xo= diff --git a/dev/preview/previewctl/main.go b/dev/preview/previewctl/main.go index 9649216277ef99..55cdbd58f336fc 100644 --- a/dev/preview/previewctl/main.go +++ b/dev/preview/previewctl/main.go @@ -18,7 +18,7 @@ func main() { TimestampFormat: "2006-01-02 15:04:05", }) - root := cmd.RootCmd(logger) + root := cmd.NewRootCmd(logger) if err := root.Execute(); err != nil { logger.WithFields(logrus.Fields{"err": err}).Fatal("command failed.") } diff --git a/dev/preview/previewctl/pkg/gcloud/config.go b/dev/preview/previewctl/pkg/gcloud/config.go index 36364ec1958506..e3b0d879b2af9d 100644 --- a/dev/preview/previewctl/pkg/gcloud/config.go +++ b/dev/preview/previewctl/pkg/gcloud/config.go @@ -6,17 +6,18 @@ package gcloud import ( "context" - "encoding/base64" - "fmt" "google.golang.org/api/container/v1" "google.golang.org/api/option" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - "k8s.io/client-go/tools/clientcmd/api" - - kube "github.com/gitpod-io/gitpod/previewctl/pkg/k8s" ) +var _ Client = (*Config)(nil) + +type Client interface { + GetCluster(ctx context.Context, name, projectID, zone string) (*container.Cluster, error) +} + type Config struct { gkeService *container.Service } @@ -40,54 +41,3 @@ func New(ctx context.Context, serviceAccountPath string) (*Config, error) { func (c *Config) GetCluster(ctx context.Context, name, projectID, zone string) (*container.Cluster, error) { return c.gkeService.Projects.Zones.Clusters.Get(projectID, zone, name).Context(ctx).Do() } - -func (c *Config) GenerateConfig(ctx context.Context, name, projectID, zone, renamedContext string) (*api.Config, error) { - cluster, err := c.GetCluster(ctx, name, projectID, zone) - if err != nil { - return nil, err - } - - ret := &api.Config{ - APIVersion: "v1", - Kind: "Config", - Clusters: map[string]*api.Cluster{}, // Clusters is a map of referencable names to cluster configs - AuthInfos: map[string]*api.AuthInfo{}, // AuthInfos is a map of referencable names to user configs - Contexts: map[string]*api.Context{}, // Contexts is a map of referencable names to context configs - } - - cert, err := base64.StdEncoding.DecodeString(cluster.MasterAuth.ClusterCaCertificate) - if err != nil { - return nil, fmt.Errorf("invalid certificate cluster=%s cert=%s: %w", name, cluster.MasterAuth.ClusterCaCertificate, err) - } - - ret.Clusters[name] = &api.Cluster{ - CertificateAuthorityData: cert, - Server: "https://" + cluster.Endpoint, - } - - // Just reuse the context name as an auth name. - ret.Contexts[name] = &api.Context{ - Cluster: name, - AuthInfo: name, - } - - // GCP specific configuration; use cloud platform scope. - ret.AuthInfos[name] = &api.AuthInfo{ - Exec: &api.ExecConfig{ - Command: "gke-gcloud-auth-plugin", - Args: nil, - Env: nil, - APIVersion: "client.authentication.k8s.io/v1beta1", - InstallHint: `Install gke-gcloud-auth-plugin for use with kubectl by following - https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke`, - ProvideClusterInfo: true, - InteractiveMode: api.IfAvailableExecInteractiveMode, - }, - } - - if renamedContext != "" { - return kube.RenameContext(ret, name, renamedContext) - } - - return ret, nil -} diff --git a/dev/preview/previewctl/pkg/k8s/config.go b/dev/preview/previewctl/pkg/k8s/config.go index 95f1b98c5db3d2..d730f37aa9e0c5 100644 --- a/dev/preview/previewctl/pkg/k8s/config.go +++ b/dev/preview/previewctl/pkg/k8s/config.go @@ -6,11 +6,9 @@ package k8s import ( "context" - "fmt" "strings" "github.com/cockroachdb/errors" - "github.com/imdario/mergo" "github.com/sirupsen/logrus" authorizationv1 "k8s.io/api/authorization/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,8 +24,11 @@ var ( ) type Config struct { - coreClient kubernetes.Interface - dynamicClient dynamic.Interface + CoreClient kubernetes.Interface + DynamicClient dynamic.Interface + + config *rest.Config + clientConfig *api.Config logger *logrus.Logger } @@ -37,14 +38,15 @@ func NewWithConfig(logger *logrus.Logger, config *rest.Config) (*Config, error) dynamicClient := dynamic.NewForConfigOrDie(config) return &Config{ - coreClient: coreClient, - dynamicClient: dynamicClient, + CoreClient: coreClient, + DynamicClient: dynamicClient, logger: logger, + config: config, }, nil } func NewFromDefaultConfigWithContext(logger *logrus.Logger, contextName string) (*Config, error) { - kconf, err := getKubernetesConfig(contextName) + kconf, err := GetKubernetesConfigFromContext(contextName) if err != nil { return nil, errors.Wrapf(err, "couldn't get [%s] kube context", contextName) } @@ -52,72 +54,56 @@ func NewFromDefaultConfigWithContext(logger *logrus.Logger, contextName string) coreClient := kubernetes.NewForConfigOrDie(kconf) dynamicClient := dynamic.NewForConfigOrDie(kconf) + clientConfig, err := GetClientConfigFromContext(contextName) + if err != nil { + return nil, err + } + return &Config{ - coreClient: coreClient, - dynamicClient: dynamicClient, + CoreClient: coreClient, + DynamicClient: dynamicClient, logger: logger, + config: kconf, + clientConfig: clientConfig, }, nil } -func getKubernetesConfig(context string) (*rest.Config, error) { +func GetClientConfigFromContext(context string) (*api.Config, error) { configLoadingRules := clientcmd.NewDefaultClientConfigLoadingRules() configOverrides := &clientcmd.ConfigOverrides{CurrentContext: context} - kconf, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(configLoadingRules, configOverrides).ClientConfig() + config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(configLoadingRules, configOverrides).RawConfig() if err != nil { - if strings.Contains(err.Error(), "does not exist") { - return nil, errors.Mark(err, ErrContextNotExists) - } return nil, err } - return kconf, err -} - -func RenameContext(config *api.Config, oldName, newName string) (*api.Config, error) { - kubeCtx, exists := config.Contexts[oldName] - if !exists { - return nil, fmt.Errorf("cannot rename %q, it's not in the provided context", oldName) - } - - if _, newExists := config.Contexts[newName]; newExists { - return nil, fmt.Errorf("cannot rename %q, it already exists in the provided context", oldName) - } - - config.Contexts[newName] = kubeCtx - delete(config.Contexts, oldName) - - if config.CurrentContext == oldName { - config.CurrentContext = newName + if _, ok := config.Contexts[context]; !ok { + return nil, ErrContextNotExists } - return config, nil + return &config, err } -func MergeWithDefaultConfig(configs ...*api.Config) (*api.Config, error) { - defaultConfig, err := clientcmd.NewDefaultClientConfigLoadingRules().Load() - if err != nil { - return nil, err - } +func GetKubernetesConfigFromContext(context string) (*rest.Config, error) { + configLoadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{CurrentContext: context} - mapConfig := api.NewConfig() - err = mergo.Merge(mapConfig, defaultConfig, mergo.WithOverride) + kconf, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(configLoadingRules, configOverrides).ClientConfig() if err != nil { + if strings.Contains(err.Error(), "does not exist") { + return nil, errors.Mark(err, ErrContextNotExists) + } return nil, err } - // If the same contexts exist in the default config, we'll override them with the configs we merge - for _, config := range configs { - err = mergo.Merge(mapConfig, config, mergo.WithOverride) - if err != nil { - return nil, err - } - } + return kconf, err +} - return mapConfig, nil +func (c *Config) ClientConfig() *api.Config { + return c.clientConfig } -func (c *Config) HasAccess() bool { +func (c *Config) HasAccess(ctx context.Context) bool { sar := &authorizationv1.SelfSubjectAccessReview{ Spec: authorizationv1.SelfSubjectAccessReviewSpec{ ResourceAttributes: &authorizationv1.ResourceAttributes{ @@ -128,7 +114,7 @@ func (c *Config) HasAccess() bool { }, } - _, err := c.coreClient.AuthorizationV1().SelfSubjectAccessReviews().Create(context.TODO(), sar, metav1.CreateOptions{}) + _, err := c.CoreClient.AuthorizationV1().SelfSubjectAccessReviews().Create(ctx, sar, metav1.CreateOptions{}) if err != nil { c.logger.Error(err) return false diff --git a/dev/preview/previewctl/pkg/k8s/context.go b/dev/preview/previewctl/pkg/k8s/context.go new file mode 100644 index 00000000000000..717d08c24f51f1 --- /dev/null +++ b/dev/preview/previewctl/pkg/k8s/context.go @@ -0,0 +1,101 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package k8s + +import ( + "fmt" + + "github.com/imdario/mergo" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/clientcmd/api" +) + +func RenameConfig(config *api.Config, oldName, newName string) (*api.Config, error) { + // TODO: https://github.com/gitpod-io/ops/issues/6524 + kubeCtx, exists := config.Contexts[oldName] + if !exists { + return nil, fmt.Errorf("cannot rename %q, it's not in the provided context", oldName) + } + + if _, newExists := config.Contexts[newName]; newExists { + return nil, fmt.Errorf("cannot rename %q, it already exists in the provided context", oldName) + } + + kubeCtx.Cluster = newName + kubeCtx.AuthInfo = newName + config.Contexts[newName] = kubeCtx + delete(config.Contexts, oldName) + + if config.CurrentContext == oldName { + config.CurrentContext = newName + } + + // we need to overwrite the cluster name and auth info + // as otherwise another context might use the wrong cluster/auth (e.g. if they are called default) + cluster, exists := config.Clusters[oldName] + if !exists { + return nil, fmt.Errorf("cannot rename %q, it's not in the provided context", oldName) + } + + if _, newExists := config.Clusters[newName]; newExists { + return nil, fmt.Errorf("cannot rename %q, it already exists in the provided context", oldName) + } + + config.Clusters[newName] = cluster + delete(config.Clusters, oldName) + + auth, exists := config.AuthInfos[oldName] + if !exists { + return nil, fmt.Errorf("cannot rename %q, it's not in the provided context", oldName) + } + + if _, newExists := config.AuthInfos[newName]; newExists { + return nil, fmt.Errorf("cannot rename %q, it already exists in the provided context", oldName) + } + + config.AuthInfos[newName] = auth + delete(config.AuthInfos, oldName) + + return config, nil +} + +func MergeContextsWithDefault(configs ...*api.Config) (*api.Config, error) { + // TODO: https://github.com/gitpod-io/ops/issues/6524 + defaultConfig, err := clientcmd.NewDefaultClientConfigLoadingRules().Load() + if err != nil { + return nil, err + } + + mapConfig := api.NewConfig() + err = mergo.Merge(mapConfig, defaultConfig, mergo.WithOverride) + if err != nil { + return nil, err + } + + // If the same contexts exist in the default config, we'll override them with the configs we merge + for _, config := range configs { + err = mergo.Merge(mapConfig, config, mergo.WithOverride) + if err != nil { + return nil, err + } + } + + return mapConfig, nil +} + +func OutputContext(kubeConfigSavePath string, config *api.Config) error { + if kubeConfigSavePath != "" { + return clientcmd.WriteToFile(*config, kubeConfigSavePath) + } + + bytes, err := clientcmd.Write(*config) + if err != nil { + return err + } + + fmt.Println(string(bytes)) + + return err +} diff --git a/dev/preview/previewctl/pkg/k8s/context/gke/dev.go b/dev/preview/previewctl/pkg/k8s/context/gke/dev.go new file mode 100644 index 00000000000000..af049076286de3 --- /dev/null +++ b/dev/preview/previewctl/pkg/k8s/context/gke/dev.go @@ -0,0 +1,104 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package gke + +import ( + "context" + "encoding/base64" + "fmt" + + "github.com/sirupsen/logrus" + "k8s.io/client-go/tools/clientcmd/api" + + "github.com/gitpod-io/gitpod/previewctl/pkg/gcloud" + "github.com/gitpod-io/gitpod/previewctl/pkg/k8s" + kctx "github.com/gitpod-io/gitpod/previewctl/pkg/k8s/context" +) + +var _ kctx.Loader = (*ConfigLoader)(nil) + +const ( + DevContextName = "dev" +) + +type ConfigLoader struct { + logger *logrus.Logger + + Client gcloud.Client + Opts ConfigLoaderOpts +} + +type ConfigLoaderOpts struct { + Logger *logrus.Logger + + Name string + ProjectID string + Zone string + ServiceAccountPath string + RenamedContextName string +} + +func New(ctx context.Context, opts ConfigLoaderOpts) (*ConfigLoader, error) { + client, err := gcloud.New(ctx, opts.ServiceAccountPath) + if err != nil { + return nil, err + } + + return &ConfigLoader{ + logger: opts.Logger, + Client: client, + Opts: opts, + }, nil +} + +func (k *ConfigLoader) Load(ctx context.Context) (*api.Config, error) { + name := k.Opts.Name + cluster, err := k.Client.GetCluster(ctx, k.Opts.Name, k.Opts.ProjectID, k.Opts.Zone) + if err != nil { + return nil, err + } + + ret := &api.Config{ + APIVersion: "v1", + Kind: "Config", + Clusters: map[string]*api.Cluster{}, // Clusters is a map of referencable names to cluster configs + AuthInfos: map[string]*api.AuthInfo{}, // AuthInfos is a map of referencable names to user configs + Contexts: map[string]*api.Context{}, // Contexts is a map of referencable names to context configs + } + + cert, err := base64.StdEncoding.DecodeString(cluster.MasterAuth.ClusterCaCertificate) + if err != nil { + return nil, fmt.Errorf("invalid certificate cluster=%s cert=%s: %w", name, cluster.MasterAuth.ClusterCaCertificate, err) + } + + ret.Clusters[name] = &api.Cluster{ + CertificateAuthorityData: cert, + Server: "https://" + cluster.Endpoint, + } + + // Just reuse the context name as an auth name. + ret.Contexts[name] = &api.Context{ + Cluster: name, + AuthInfo: name, + } + + // GCP specific configuration; use cloud platform scope. + ret.AuthInfos[name] = &api.AuthInfo{ + Exec: &api.ExecConfig{ + Command: "gke-gcloud-auth-plugin", + APIVersion: "client.authentication.k8s.io/v1beta1", + InstallHint: `Install gke-gcloud-auth-plugin for use with kubectl by following + https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke`, + ProvideClusterInfo: true, + InteractiveMode: api.IfAvailableExecInteractiveMode, + }, + } + + if k.Opts.RenamedContextName != "" { + return k8s.RenameConfig(ret, name, k.Opts.RenamedContextName) + } + + return ret, nil +} diff --git a/dev/preview/previewctl/pkg/k8s/context/gke/dev_test.go b/dev/preview/previewctl/pkg/k8s/context/gke/dev_test.go new file mode 100644 index 00000000000000..5f624fe34096a0 --- /dev/null +++ b/dev/preview/previewctl/pkg/k8s/context/gke/dev_test.go @@ -0,0 +1,101 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package gke + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/api/container/v1" + "k8s.io/client-go/tools/clientcmd/api" + + "github.com/gitpod-io/gitpod/previewctl/pkg/gcloud" +) + +func Test_Load(t *testing.T) { + type expStruct struct { + config *api.Config + err error + } + + type testCase struct { + name string + client *mockGetClusterClient + expected *expStruct + } + + testCases := []testCase{ + { + name: "Get config", + client: &mockGetClusterClient{ + cert: "dGVzdF9kYXRh", + }, + expected: &expStruct{ + config: &api.Config{ + APIVersion: "v1", + Kind: "Config", + Contexts: map[string]*api.Context{ + DevContextName: { + Cluster: DevContextName, + AuthInfo: DevContextName, + }, + }, + Clusters: map[string]*api.Cluster{ + DevContextName: { + CertificateAuthorityData: []byte("test_data"), + Server: "https://test", + }, + }, + AuthInfos: map[string]*api.AuthInfo{ + DevContextName: { + Exec: &api.ExecConfig{ + Command: "gke-gcloud-auth-plugin", + APIVersion: "client.authentication.k8s.io/v1beta1", + InstallHint: `Install gke-gcloud-auth-plugin for use with kubectl by following + https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke`, + ProvideClusterInfo: true, + InteractiveMode: api.IfAvailableExecInteractiveMode, + }, + }, + }, + }, + err: nil, + }, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + k := &ConfigLoader{ + Client: test.client, + Opts: ConfigLoaderOpts{ + Name: "test", + RenamedContextName: DevContextName, + }, + } + + config, err := k.Load(context.TODO()) + + assert.ErrorIs(t, test.expected.err, err) + assert.Equal(t, test.expected.config, config) + }) + } +} + +type mockGetClusterClient struct { + gcloud.Client + + cert string +} + +func (m *mockGetClusterClient) GetCluster(ctx context.Context, name, projectID, zone string) (*container.Cluster, error) { + return &container.Cluster{ + MasterAuth: &container.MasterAuth{ + ClusterCaCertificate: m.cert, + }, + Endpoint: name, + }, nil +} diff --git a/dev/preview/previewctl/pkg/k8s/context/harvester/harvester.go b/dev/preview/previewctl/pkg/k8s/context/harvester/harvester.go new file mode 100644 index 00000000000000..33879e22b515c4 --- /dev/null +++ b/dev/preview/previewctl/pkg/k8s/context/harvester/harvester.go @@ -0,0 +1,89 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package harvester + +import ( + "context" + + "github.com/cockroachdb/errors" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/clientcmd/api" + + "github.com/gitpod-io/gitpod/previewctl/pkg/k8s" + kctx "github.com/gitpod-io/gitpod/previewctl/pkg/k8s/context" + "github.com/gitpod-io/gitpod/previewctl/pkg/k8s/context/gke" +) + +const ( + ContextName = "harvester" + + harvesterConfigSecretName = "harvester-kubeconfig" + werftNamespace = "werft" +) + +var ( + ErrSecretDataNotFound = errors.New("secret data not found") +) + +var _ kctx.Loader = (*ConfigLoader)(nil) + +type ConfigLoader struct { + logger *logrus.Logger + + Client *k8s.Config +} + +type ConfigLoaderOpts struct { + Logger *logrus.Logger +} + +func New(ctx context.Context, opts ConfigLoaderOpts) (*ConfigLoader, error) { + client, err := k8s.NewFromDefaultConfigWithContext(opts.Logger, gke.DevContextName) + if err != nil { + return nil, err + } + + return &ConfigLoader{ + logger: opts.Logger, + Client: client, + }, nil +} + +func (k *ConfigLoader) setup() error { + client, err := k8s.NewFromDefaultConfigWithContext(k.logger, gke.DevContextName) + if err != nil { + return err + } + + k.Client = client + + return nil +} + +func (k *ConfigLoader) Load(ctx context.Context) (*api.Config, error) { + if k.Client == nil { + if err := k.setup(); err != nil { + return nil, err + } + } + + secret, err := k.Client.CoreClient.CoreV1().Secrets(werftNamespace).Get(ctx, harvesterConfigSecretName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + if _, ok := secret.Data["harvester-kubeconfig.yml"]; !ok { + return nil, ErrSecretDataNotFound + } + + config, err := clientcmd.Load(secret.Data["harvester-kubeconfig.yml"]) + if err != nil { + return nil, err + } + + return k8s.RenameConfig(config, "default", "harvester") +} diff --git a/dev/preview/previewctl/pkg/k8s/context/harvester/harvester_test.go b/dev/preview/previewctl/pkg/k8s/context/harvester/harvester_test.go new file mode 100644 index 00000000000000..137f5c98e6945a --- /dev/null +++ b/dev/preview/previewctl/pkg/k8s/context/harvester/harvester_test.go @@ -0,0 +1,127 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package harvester + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/clientcmd/api" + + "github.com/gitpod-io/gitpod/previewctl/pkg/k8s" +) + +func TestLoad(t *testing.T) { + type expStruct struct { + config *api.Config + err error + } + + testCases := []struct { + name string + objects []runtime.Object + expected expStruct + }{ + { + name: "secret not found", + objects: []runtime.Object{ + &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: harvesterConfigSecretName, + Namespace: werftNamespace, + }, + Data: map[string][]byte{}, + }, + }, + expected: expStruct{ + config: nil, + err: ErrSecretDataNotFound, + }, + }, + { + name: "harvester config", + objects: []runtime.Object{ + &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: harvesterConfigSecretName, + Namespace: werftNamespace, + }, + Data: map[string][]byte{ + "harvester-kubeconfig.yml": []byte(` +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: dGVzdF9kYXRh + server: https://test.kube.gitpod-dev.com:6443 + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + client-certificate-data: dGVzdF9kYXRh + client-key-data: dGVzdF9kYXRh`), + }, + }, + }, + expected: expStruct{ + config: &api.Config{ + Preferences: api.Preferences{ + Extensions: map[string]runtime.Object{}, + }, + Contexts: map[string]*api.Context{ + "harvester": { + Cluster: "harvester", + AuthInfo: "harvester", + Extensions: map[string]runtime.Object{}, + }, + }, + Clusters: map[string]*api.Cluster{ + "harvester": { + LocationOfOrigin: "", + Server: "https://test.kube.gitpod-dev.com:6443", + CertificateAuthorityData: []byte("test_data"), + Extensions: map[string]runtime.Object{}, + }, + }, + CurrentContext: "harvester", + AuthInfos: map[string]*api.AuthInfo{ + "harvester": { + ClientCertificateData: []byte("test_data"), + ClientKeyData: []byte("test_data"), + Extensions: map[string]runtime.Object{}, + }, + }, + Extensions: map[string]runtime.Object{}, + }, + err: nil, + }, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + c := &ConfigLoader{ + Client: &k8s.Config{CoreClient: fake.NewSimpleClientset(test.objects...)}, + } + + config, err := c.Load(context.TODO()) + + assert.ErrorIs(t, test.expected.err, err) + assert.Equal(t, test.expected.config, config) + }) + } +} diff --git a/dev/preview/previewctl/pkg/k8s/context/k3s/k3s.go b/dev/preview/previewctl/pkg/k8s/context/k3s/k3s.go new file mode 100644 index 00000000000000..cd9532949cd7a3 --- /dev/null +++ b/dev/preview/previewctl/pkg/k8s/context/k3s/k3s.go @@ -0,0 +1,254 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package k3s + +import ( + "bytes" + "context" + "fmt" + "math/rand" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/cockroachdb/errors" + "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/clientcmd/api" + + "github.com/gitpod-io/gitpod/previewctl/pkg/k8s" + kctx "github.com/gitpod-io/gitpod/previewctl/pkg/k8s/context" + pssh "github.com/gitpod-io/gitpod/previewctl/pkg/ssh" +) + +var _ kctx.Loader = (*ConfigLoader)(nil) + +const ( + k3sConfigPath = "/etc/rancher/k3s/k3s.yaml" + catK3sConfigCmd = "sudo cat /etc/rancher/k3s/k3s.yaml" + harvesterContextName = "harvester" +) + +var ( + ErrK3SConfigNotFound = errors.New("k3s config file not found") +) + +type ConfigLoader struct { + logger *logrus.Logger + + sshClientFactory pssh.ClientFactory + client pssh.Client + + configPath string + opts ConfigLoaderOpts + harvesterClient *k8s.Config +} + +type ConfigLoaderOpts struct { + Logger *logrus.Logger + + PreviewName string + PreviewNamespace string + SSHPrivateKeyPath string + SSHUser string +} + +func New(ctx context.Context, opts ConfigLoaderOpts) (*ConfigLoader, error) { + key, err := os.ReadFile(opts.SSHPrivateKeyPath) + if err != nil { + return nil, err + } + + signer, err := ssh.ParsePrivateKey(key) + if err != nil { + return nil, err + } + + k8sClient, err := k8s.NewFromDefaultConfigWithContext(opts.Logger, harvesterContextName) + if err != nil { + return nil, err + } + + config := &ConfigLoader{ + logger: opts.Logger, + harvesterClient: k8sClient, + sshClientFactory: &pssh.FactoryImplementation{ + SSHConfig: &ssh.ClientConfig{ + User: opts.SSHUser, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + }, + }, + configPath: k3sConfigPath, + opts: opts, + } + + return config, nil +} + +func (k *ConfigLoader) Load(ctx context.Context) (*api.Config, error) { + if k.client == nil { + stopChan, readyChan, errChan := make(chan struct{}, 1), make(chan struct{}, 1), make(chan error, 1) + go func() { + err := k.setup(ctx, stopChan, readyChan, errChan) + if err != nil { + k.logger.WithFields(logrus.Fields{"err": err}).Error("failed to setup port-forward and ssh connection to VM") + } + }() + + select { + case <-readyChan: + case err := <-errChan: + return nil, err + } + + defer func(client pssh.Client) { + // closing the stopChan will stop the port-forward + close(stopChan) + err := client.Close() + if err != nil { + k.logger.WithFields(logrus.Fields{"err": err}).Error("failed to close client") + return + } + }(k.client) + } + + return k.getContext(ctx) +} + +func (k *ConfigLoader) getContext(ctx context.Context) (*api.Config, error) { + stdout := new(bytes.Buffer) + stderr := new(bytes.Buffer) + + err := k.client.Run(ctx, catK3sConfigCmd, stdout, stderr) + if err != nil { + if strings.Contains(stderr.String(), "No such file or directory") { + return nil, ErrK3SConfigNotFound + } + + return nil, errors.Wrap(err, stderr.String()) + } + + c, err := clientcmd.NewClientConfigFromBytes(stdout.Bytes()) + if err != nil { + return nil, err + } + + rc, err := c.RawConfig() + if err != nil { + return nil, err + } + + k3sConfig, err := k8s.RenameConfig(&rc, "default", k.opts.PreviewName) + if err != nil { + return nil, err + } + + k3sConfig.Clusters[k.opts.PreviewName].Server = fmt.Sprintf("https://%s.kube.gitpod-dev.com:6443", k.opts.PreviewName) + + return &rc, nil +} + +func (k *ConfigLoader) setup(ctx context.Context, stopChan, readyChan chan struct{}, errChan chan error) error { + // pick a random port, so we avoid clashes if something else port-forwards to 2200 + randPort := strconv.Itoa(rand.Intn(2299-2201) + 2201) + // we use portForwardReadyChan to signal when we've started the port-forward + portForwardReadyChan := make(chan struct{}, 1) + go func() { + podName, err := k.getVMPodName(ctx, k.opts.PreviewName, k.opts.PreviewNamespace) + if err != nil { + errChan <- err + return + } + + err = k.harvesterClient.PortForward(ctx, k8s.PortForwardOpts{ + Name: podName, + Namespace: k.opts.PreviewNamespace, + Ports: []string{ + fmt.Sprintf("%s:2200", randPort), + }, + ReadyChan: portForwardReadyChan, + StopChan: stopChan, + ErrChan: errChan, + }) + + if err != nil { + errChan <- err + return + } + }() + + var once sync.Once + select { + case <-portForwardReadyChan: + once.Do(func() { + err := k.connectToHost(ctx, "127.0.0.1", randPort) + if err != nil { + k.logger.Error(err) + errChan <- err + return + } + readyChan <- struct{}{} + }) + case err := <-errChan: + return err + case <-time.After(time.Second * 50): + return errors.New("timed out waiting for port forward") + case <-ctx.Done(): + k.logger.Debug("context cancelled") + return ctx.Err() + } + + return nil +} + +func (k *ConfigLoader) connectToHost(ctx context.Context, host, port string) error { + client, err := k.sshClientFactory.Dial(ctx, host, port) + if err != nil { + return err + } + k.client = client + + return nil +} + +func (k *ConfigLoader) Close() error { + if err := k.client.Close(); err != nil { + return err + } + + k.client = nil + return nil +} + +func (k *ConfigLoader) getVMPodName(ctx context.Context, previewName, namespace string) (string, error) { + // TODO: replace this with a call to SVC.Proxy and get the pod name from there + labelSelector := metav1.LabelSelector{ + MatchLabels: map[string]string{ + "harvesterhci.io/vmName": previewName, + }, + } + + pods, err := k.harvesterClient.CoreClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labels.Set(labelSelector.MatchLabels).String(), + }) + + if err != nil { + return "", err + } + + if len(pods.Items) != 1 { + return "", errors.Newf("expected a single pod, got [%d]", len(pods.Items)) + } + + return pods.Items[0].Name, nil +} diff --git a/dev/preview/previewctl/pkg/k8s/context/k3s/k3s_test.go b/dev/preview/previewctl/pkg/k8s/context/k3s/k3s_test.go new file mode 100644 index 00000000000000..d4a7109a4e468a --- /dev/null +++ b/dev/preview/previewctl/pkg/k8s/context/k3s/k3s_test.go @@ -0,0 +1,119 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package k3s + +import ( + "context" + "testing" + + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/clientcmd/api" + + pssh "github.com/gitpod-io/gitpod/previewctl/pkg/ssh" +) + +func Test_LoadK3SConfig(t *testing.T) { + type k3sExpStruct struct { + config *api.Config + err error + } + type testCase struct { + name string + cmd pssh.MockCmd + expected *k3sExpStruct + } + + testCases := []testCase{ + { + name: "k3s config not found", + cmd: pssh.MockCmd{ + CMD: catK3sConfigCmd, + STDOUT: []byte(""), + STDERR: []byte("cat: /etc/rancher/k3s/k3s.yaml: No such file or directory"), + Err: errors.New("some error that will be irrelevant"), + }, + expected: &k3sExpStruct{ + config: nil, + err: ErrK3SConfigNotFound, + }, + }, + { + name: "returned config", + cmd: pssh.MockCmd{ + CMD: catK3sConfigCmd, + STDOUT: []byte(` +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: dGVzdF9kYXRh + server: https://default.kube.gitpod-dev.com:6443 + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + client-certificate-data: dGVzdF9kYXRh + client-key-data: dGVzdF9kYXRh +`), + STDERR: nil, + Err: nil, + }, + expected: &k3sExpStruct{ + config: &api.Config{ + Preferences: api.Preferences{ + Extensions: map[string]runtime.Object{}, + }, + Contexts: map[string]*api.Context{ + "k3s": { + Cluster: "k3s", + AuthInfo: "k3s", + Extensions: map[string]runtime.Object{}, + }, + }, + Clusters: map[string]*api.Cluster{ + "k3s": { + Server: "https://k3s.kube.gitpod-dev.com:6443", + CertificateAuthorityData: []byte("test_data"), + Extensions: map[string]runtime.Object{}, + }, + }, + CurrentContext: "k3s", + AuthInfos: map[string]*api.AuthInfo{ + "k3s": { + ClientCertificateData: []byte("test_data"), + ClientKeyData: []byte("test_data"), + Extensions: map[string]runtime.Object{}, + }, + }, + Extensions: map[string]runtime.Object{}, + }, + err: nil, + }, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + c := &pssh.MockClient{Command: test.cmd} + k := &ConfigLoader{client: c, opts: ConfigLoaderOpts{ + PreviewName: "k3s", + }} + + config, err := k.Load(context.TODO()) + + assert.ErrorIs(t, test.expected.err, err) + assert.Equal(t, test.expected.config, config) + }) + } +} diff --git a/dev/preview/previewctl/pkg/k8s/context/loader.go b/dev/preview/previewctl/pkg/k8s/context/loader.go new file mode 100644 index 00000000000000..31fd648e9d4f1c --- /dev/null +++ b/dev/preview/previewctl/pkg/k8s/context/loader.go @@ -0,0 +1,15 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package context + +import ( + "context" + + "k8s.io/client-go/tools/clientcmd/api" +) + +type Loader interface { + Load(context.Context) (*api.Config, error) +} diff --git a/dev/preview/previewctl/pkg/k8s/harvester.go b/dev/preview/previewctl/pkg/k8s/harvester.go deleted file mode 100644 index 22d85431139af6..00000000000000 --- a/dev/preview/previewctl/pkg/k8s/harvester.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) 2022 Gitpod GmbH. All rights reserved. -// Licensed under the GNU Affero General Public License (AGPL). -// See License-AGPL.txt in the project root for license information. - -package k8s - -import ( - "context" - - "github.com/cockroachdb/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/clientcmd/api" -) - -var ( - ErrSecretDataNotFound = errors.New("secret data not found") -) - -const ( - harvesterConfigSecretName = "harvester-kubeconfig" - werftNamespace = "werft" -) - -func (c *Config) GetHarvesterKubeConfig(ctx context.Context) (*api.Config, error) { - secret, err := c.coreClient.CoreV1().Secrets(werftNamespace).Get(ctx, harvesterConfigSecretName, metav1.GetOptions{}) - if err != nil { - return nil, err - } - - if _, ok := secret.Data["harvester-kubeconfig.yml"]; !ok { - return nil, ErrSecretDataNotFound - } - - config, err := clientcmd.Load(secret.Data["harvester-kubeconfig.yml"]) - if err != nil { - return nil, err - } - - return RenameContext(config, "default", "harvester") -} diff --git a/dev/preview/previewctl/pkg/k8s/pforward.go b/dev/preview/previewctl/pkg/k8s/pforward.go new file mode 100644 index 00000000000000..261bfd218ec647 --- /dev/null +++ b/dev/preview/previewctl/pkg/k8s/pforward.go @@ -0,0 +1,61 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package k8s + +import ( + "bytes" + "context" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/cockroachdb/errors" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/transport/spdy" +) + +type PortForwardOpts struct { + Name string + Namespace string + Ports []string + ReadyChan, StopChan chan struct{} + ErrChan chan error +} + +func (c *Config) PortForward(ctx context.Context, opts PortForwardOpts) error { + roundTripper, upgrader, err := spdy.RoundTripperFor(c.config) + if err != nil { + panic(err) + } + + path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/portforward", opts.Namespace, opts.Name) + hostIP := strings.TrimLeft(c.config.Host, "https://") + serverURL := url.URL{Scheme: "https", Path: path, Host: hostIP} + + dialer := spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, &serverURL) + + out, errOut := new(bytes.Buffer), new(bytes.Buffer) + forwarder, err := portforward.New(dialer, opts.Ports, opts.StopChan, opts.ReadyChan, out, errOut) + if err != nil { + return err + } + + go func() { + for range opts.ReadyChan { // Kubernetes will close this channel when it has something to tell us. + } + if len(errOut.String()) != 0 { + opts.ErrChan <- errors.New(errOut.String()) + } else if len(out.String()) != 0 { + c.logger.Debug(out.String()) + } + }() + + if err = forwarder.ForwardPorts(); err != nil { // Locks until stopChan is closed. + return err + } + + return nil +} diff --git a/dev/preview/previewctl/pkg/k8s/service.go b/dev/preview/previewctl/pkg/k8s/service.go index b2ad9c237372e8..c2d00f9fcbbc72 100644 --- a/dev/preview/previewctl/pkg/k8s/service.go +++ b/dev/preview/previewctl/pkg/k8s/service.go @@ -21,12 +21,12 @@ var ErrSvcNotReady = errors.New("proxy service not ready") const proxySvcName = "proxy" func (c *Config) GetProxyVMServiceStatus(ctx context.Context, namespace string) error { - _, err := c.coreClient.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + _, err := c.CoreClient.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) if err != nil { return errors.Wrap(ErrSvcNotReady, err.Error()) } - svc, err := c.coreClient.CoreV1().Services(namespace).Get(ctx, proxySvcName, metav1.GetOptions{}) + svc, err := c.CoreClient.CoreV1().Services(namespace).Get(ctx, proxySvcName, metav1.GetOptions{}) if err != nil { return errors.Wrap(ErrSvcNotReady, err.Error()) } @@ -39,7 +39,7 @@ func (c *Config) GetProxyVMServiceStatus(ctx context.Context, namespace string) } func (c *Config) WaitProxySvcReady(ctx context.Context, namespace string, doneCh chan struct{}) error { - kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(c.coreClient, time.Second*30, informers.WithNamespace(namespace)) + kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(c.CoreClient, time.Second*30, informers.WithNamespace(namespace)) svcInformer := kubeInformerFactory.Core().V1().Services().Informer() stopCh := make(chan struct{}) diff --git a/dev/preview/previewctl/pkg/k8s/service_test.go b/dev/preview/previewctl/pkg/k8s/service_test.go index a6795de6b17521..1e18cd69f670ed 100644 --- a/dev/preview/previewctl/pkg/k8s/service_test.go +++ b/dev/preview/previewctl/pkg/k8s/service_test.go @@ -80,7 +80,7 @@ func TestGetVMProxySvcStatus(t *testing.T) { for _, test := range testCases { t.Run(test.name, func(t *testing.T) { - c.coreClient = fake.NewSimpleClientset(test.objects...) + c.CoreClient = fake.NewSimpleClientset(test.objects...) err := c.GetProxyVMServiceStatus(context.TODO(), namespace) diff --git a/dev/preview/previewctl/pkg/k8s/vm.go b/dev/preview/previewctl/pkg/k8s/vm.go index 9bc74069c19821..09a2e80ac4e51d 100644 --- a/dev/preview/previewctl/pkg/k8s/vm.go +++ b/dev/preview/previewctl/pkg/k8s/vm.go @@ -92,7 +92,7 @@ func (c *Config) GetVMICreationTimestamp(ctx context.Context, name, namespace st } func (c *Config) WaitVMReady(ctx context.Context, name, namespace string, doneCh chan struct{}) error { - kubeInformerFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(c.dynamicClient, time.Second*30, namespace, nil) + kubeInformerFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(c.DynamicClient, time.Second*30, namespace, nil) vmInformer := kubeInformerFactory.ForResource(vmInstanceResource).Informer() vmInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -149,7 +149,7 @@ func (c *Config) WaitVMReady(ctx context.Context, name, namespace string, doneCh } func (c *Config) GetVMs(ctx context.Context) ([]string, error) { - virtualMachineClient := c.dynamicClient.Resource(vmResource).Namespace("") + virtualMachineClient := c.DynamicClient.Resource(vmResource).Namespace("") vmObjs, err := virtualMachineClient.List(ctx, metav1.ListOptions{}) if err != nil { return nil, err @@ -164,7 +164,7 @@ func (c *Config) GetVMs(ctx context.Context) ([]string, error) { } func (c *Config) getVMI(ctx context.Context, name, namespace string) (*virtv1.VirtualMachineInstance, error) { - _, err := c.coreClient.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + _, err := c.CoreClient.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) if err != nil && !kerrors.IsNotFound(err) { return nil, errors.Wrap(ErrVmNotReady, err.Error()) } @@ -174,7 +174,7 @@ func (c *Config) getVMI(ctx context.Context, name, namespace string) (*virtv1.Vi return nil, errors.Wrap(ErrVmNotReady, err.Error()) } - vmClient := c.dynamicClient.Resource(vmInstanceResource).Namespace(namespace) + vmClient := c.DynamicClient.Resource(vmInstanceResource).Namespace(namespace) res, err := vmClient.Get(ctx, name, metav1.GetOptions{}) if err != nil { return nil, errors.Wrap(ErrVmNotReady, err.Error()) diff --git a/dev/preview/previewctl/pkg/k8s/vm_test.go b/dev/preview/previewctl/pkg/k8s/vm_test.go index 723614a5ff7910..1536ce4d035e46 100644 --- a/dev/preview/previewctl/pkg/k8s/vm_test.go +++ b/dev/preview/previewctl/pkg/k8s/vm_test.go @@ -105,12 +105,12 @@ func TestGetVMStatus(t *testing.T) { for _, test := range testCases { t.Run(test.name, func(t *testing.T) { - c.coreClient = fake.NewSimpleClientset(test.objects...) + c.CoreClient = fake.NewSimpleClientset(test.objects...) scheme := runtime.NewScheme() _ = virtv1.AddToScheme(scheme) - c.dynamicClient = dfake.NewSimpleDynamicClient(scheme, test.dynObjects...) + c.DynamicClient = dfake.NewSimpleDynamicClient(scheme, test.dynObjects...) err := c.GetVMStatus(context.TODO(), name, namespace) if test.err != nil { diff --git a/dev/preview/previewctl/pkg/preview/preview.go b/dev/preview/previewctl/pkg/preview/preview.go index 78b6b52e8dbd05..64aeebfe16f004 100644 --- a/dev/preview/previewctl/pkg/preview/preview.go +++ b/dev/preview/previewctl/pkg/preview/preview.go @@ -18,8 +18,10 @@ import ( "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd/api" "github.com/gitpod-io/gitpod/previewctl/pkg/k8s" + "github.com/gitpod-io/gitpod/previewctl/pkg/k8s/context/k3s" ) var ( @@ -33,7 +35,8 @@ type Preview struct { name string namespace string - kubeClient *k8s.Config + harvesterClient *k8s.Config + configLoader *k3s.ConfigLoader logger *logrus.Entry @@ -57,55 +60,78 @@ func New(branch string, logger *logrus.Logger) (*Preview, error) { branch: branch, namespace: fmt.Sprintf("preview-%s", branch), name: branch, - kubeClient: harvesterConfig, + harvesterClient: harvesterConfig, logger: logEntry, vmiCreationTime: nil, }, nil } -func (p *Preview) InstallContext(wait bool, timeout time.Duration) error { - ctx, cancel := context.WithTimeout(context.Background(), timeout) +type InstallCtxOpts struct { + Wait bool + Timeout time.Duration + KubeSavePath string + SSHPrivateKeyPath string +} + +func (p *Preview) InstallContext(ctx context.Context, opts InstallCtxOpts) error { + // TODO: https://github.com/gitpod-io/ops/issues/6524 + if p.configLoader == nil { + configLoader, err := k3s.New(ctx, k3s.ConfigLoaderOpts{ + Logger: p.logger.Logger, + PreviewName: p.name, + PreviewNamespace: p.namespace, + SSHPrivateKeyPath: opts.SSHPrivateKeyPath, + SSHUser: "ubuntu", + }) + + if err != nil { + return err + } + + p.configLoader = configLoader + } + + ctx, cancel := context.WithTimeout(ctx, opts.Timeout) defer cancel() - p.logger.WithFields(logrus.Fields{"timeout": timeout}).Infof("Installing context") + p.logger.WithFields(logrus.Fields{"timeout": opts.Timeout}).Debug("Installing context") // we use this channel to signal when we've found an event in wait functions, so we know when we're done doneCh := make(chan struct{}) defer close(doneCh) - // TODO: fix this, as it's a bit ugly - err := p.kubeClient.GetVMStatus(ctx, p.name, p.namespace) + err := p.harvesterClient.GetVMStatus(ctx, p.name, p.namespace) if err != nil && !errors.Is(err, k8s.ErrVmNotReady) { return err - } else if errors.Is(err, k8s.ErrVmNotReady) && !wait { + } else if errors.Is(err, k8s.ErrVmNotReady) && !opts.Wait { return err - } else if errors.Is(err, k8s.ErrVmNotReady) && wait { - err = p.kubeClient.WaitVMReady(ctx, p.name, p.namespace, doneCh) + } else if errors.Is(err, k8s.ErrVmNotReady) && opts.Wait { + err = p.harvesterClient.WaitVMReady(ctx, p.name, p.namespace, doneCh) if err != nil { return err } } - err = p.kubeClient.GetProxyVMServiceStatus(ctx, p.namespace) + err = p.harvesterClient.GetProxyVMServiceStatus(ctx, p.namespace) if err != nil && !errors.Is(err, k8s.ErrSvcNotReady) { return err - } else if errors.Is(err, k8s.ErrSvcNotReady) && !wait { + } else if errors.Is(err, k8s.ErrSvcNotReady) && !opts.Wait { return err - } else if errors.Is(err, k8s.ErrSvcNotReady) && wait { - err = p.kubeClient.WaitProxySvcReady(ctx, p.namespace, doneCh) + } else if errors.Is(err, k8s.ErrSvcNotReady) && opts.Wait { + err = p.harvesterClient.WaitProxySvcReady(ctx, p.namespace, doneCh) if err != nil { return err } } - if wait { + if opts.Wait { for { select { case <-ctx.Done(): return ctx.Err() case <-time.Tick(5 * time.Second): p.logger.Infof("waiting for context install to succeed") - err = installContext(p.branch) + err = p.Install(ctx, opts) if err == nil { p.logger.Infof("Successfully installed context") return nil @@ -114,7 +140,7 @@ func (p *Preview) InstallContext(wait bool, timeout time.Duration) error { } } - return installContext(p.branch) + return p.Install(ctx, opts) } // Same compares two preview envrionments @@ -138,7 +164,7 @@ func ensureVMICreationTime(p *Preview) { defer cancel() if p.vmiCreationTime == nil { - creationTime, err := p.kubeClient.GetVMICreationTimestamp(ctx, p.name, p.namespace) + creationTime, err := p.harvesterClient.GetVMICreationTimestamp(ctx, p.name, p.namespace) p.vmiCreationTime = creationTime if err != nil { p.logger.WithFields(logrus.Fields{"err": err}).Infof("Failed to get creation time") @@ -146,8 +172,27 @@ func ensureVMICreationTime(p *Preview) { } } -func installContext(branch string) error { - return exec.Command("bash", "/workspace/gitpod/dev/preview/install-k3s-kubeconfig.sh", "-b", branch).Run() +func (p *Preview) Install(ctx context.Context, opts InstallCtxOpts) error { + cfg, err := p.GetPreviewContext(ctx) + if err != nil { + return err + } + + merged, err := k8s.MergeContextsWithDefault(cfg) + if err != nil { + return err + } + + return k8s.OutputContext(opts.KubeSavePath, merged) +} + +func (p *Preview) GetPreviewContext(ctx context.Context) (*api.Config, error) { + return p.configLoader.Load(ctx) +} + +func InstallVMSSHKeys() error { + // TODO: https://github.com/gitpod-io/ops/issues/6524 + return exec.Command("bash", "/workspace/gitpod/dev/preview/util/install-vm-ssh-keys.sh").Run() } func SSHPreview(branch string) error { @@ -207,8 +252,8 @@ func GetName(branch string) (string, error) { return sanitizedBranch, nil } -func (p *Preview) ListAllPreviews() error { - previews, err := p.kubeClient.GetVMs(context.Background()) +func (p *Preview) ListAllPreviews(ctx context.Context) error { + previews, err := p.harvesterClient.GetVMs(ctx) if err != nil { return err } diff --git a/dev/preview/previewctl/pkg/ssh/mock.go b/dev/preview/previewctl/pkg/ssh/mock.go new file mode 100644 index 00000000000000..f5b864f5ab42e6 --- /dev/null +++ b/dev/preview/previewctl/pkg/ssh/mock.go @@ -0,0 +1,39 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package ssh + +import ( + "context" + "io" + + "github.com/cockroachdb/errors" +) + +var _ Client = &MockClient{} + +type MockCmd struct { + CMD string + STDOUT []byte + STDERR []byte + Err error +} + +type MockClient struct { + Command MockCmd +} + +func (m MockClient) Close() error { + return nil +} + +func (m MockClient) Run(ctx context.Context, cmd string, stdout io.Writer, stderr io.Writer) error { + if m.Command.CMD != cmd { + return errors.New("command not found") + } + + _, _ = stdout.Write(m.Command.STDOUT) + _, _ = stderr.Write(m.Command.STDERR) + return m.Command.Err +} diff --git a/dev/preview/previewctl/pkg/ssh/ssh.go b/dev/preview/previewctl/pkg/ssh/ssh.go new file mode 100644 index 00000000000000..08d3c58ded14b6 --- /dev/null +++ b/dev/preview/previewctl/pkg/ssh/ssh.go @@ -0,0 +1,80 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package ssh + +import ( + "context" + "fmt" + "io" + "net" + + "golang.org/x/crypto/ssh" +) + +type Client interface { + io.Closer + + Run(ctx context.Context, cmd string, stdout io.Writer, stderr io.Writer) error +} + +type ClientFactory interface { + Dial(ctx context.Context, host, port string) (Client, error) +} + +type ClientImplementation struct { + Client *ssh.Client +} + +var _ Client = &ClientImplementation{} + +func (s *ClientImplementation) Run(ctx context.Context, cmd string, stdout io.Writer, stderr io.Writer) error { + sess, err := s.Client.NewSession() + if err != nil { + return err + } + + defer func(sess *ssh.Session) { + err := sess.Close() + if err != nil && err != io.EOF { + panic(err) + } + }(sess) + + sess.Stdout = stdout + sess.Stderr = stderr + + return sess.Run(cmd) +} + +func (s *ClientImplementation) Close() error { + return s.Client.Close() +} + +type FactoryImplementation struct { + SSHConfig *ssh.ClientConfig +} + +var _ ClientFactory = &FactoryImplementation{} + +func (f *FactoryImplementation) Dial(ctx context.Context, host, port string) (Client, error) { + addr := fmt.Sprintf("%s:%s", host, port) + d := net.Dialer{} + conn, err := d.DialContext(ctx, "tcp", addr) + if err != nil { + return nil, err + } + + var client *ssh.Client + c, chans, reqs, err := ssh.NewClientConn(conn, addr, f.SSHConfig) + if err != nil { + return nil, err + } + + client = ssh.NewClient(c, chans, reqs) + + return &ClientImplementation{ + Client: client, + }, nil +}