commands/.../test,pkg/test: add --up-local flag to test local

Author: AlexNPavel

CHANGELOG.md

@@ -7,6 +7,7 @@
 ### Added
 
 - A new command [`operator-sdk print-deps`](https://github.com/operator-framework/operator-sdk/blob/master/doc/sdk-cli-reference.md#print-deps) which prints Golang packages and versions expected by the current Operator SDK version. Supplying `--as-file` prints packages and versions in Gopkg.toml format. ([#772](https://github.com/operator-framework/operator-sdk/pull/772))
+- Add [`up-local`](https://github.com/operator-framework/operator-sdk/blob/master/doc/sdk-cli-reference.md#flags-9) flag to `test local` subcommand ([#XXX](https://github.com/operator-framework/operator-sdk/pull/XXX))
 
 ### Bug fixes
 

commands/operator-sdk/cmd/test/local.go

@@ -39,6 +39,7 @@ type testLocalConfig struct {
 	namespacedManPath string
 	goTestFlags       string
 	namespace         string
+	upLocal           bool
 }
 
 var tlConfig testLocalConfig
@@ -54,6 +55,7 @@ func NewTestLocalCmd() *cobra.Command {
 	testCmd.Flags().StringVar(&tlConfig.namespacedManPath, "namespaced-manifest", "", "Path to manifest for per-test, namespaced resources (e.g. RBAC and Operator manifest)")
 	testCmd.Flags().StringVar(&tlConfig.goTestFlags, "go-test-flags", "", "Additional flags to pass to go test")
 	testCmd.Flags().StringVar(&tlConfig.namespace, "namespace", "", "If non-empty, single namespace to run tests in")
+	testCmd.Flags().BoolVar(&tlConfig.upLocal, "up-local", false, "Enable running operator locally with go run instead of as an image in the cluster")
 
 	return testCmd
 }
@@ -63,6 +65,10 @@ func testLocalFunc(cmd *cobra.Command, args []string) {
 		log.Fatal("operator-sdk test local requires exactly 1 argument")
 	}
 
+	if tlConfig.upLocal && tlConfig.namespace == "" {
+		log.Fatal("must specify a namespace to run in when -up-local flag is set")
+	}
+
 	log.Info("Testing operator locally.")
 
 	// if no namespaced manifest path is given, combine deploy/service_account.yaml, deploy/role.yaml, deploy/role_binding.yaml and deploy/operator.yaml
@@ -72,28 +78,29 @@ func testLocalFunc(cmd *cobra.Command, args []string) {
 			log.Fatalf("could not create %s: (%v)", deployTestDir, err)
 		}
 		tlConfig.namespacedManPath = filepath.Join(deployTestDir, "namespace-manifests.yaml")
-
-		sa, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.ServiceAccountYamlFile))
-		if err != nil {
-			log.Warnf("could not find the serviceaccount manifest: (%v)", err)
-		}
-		role, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.RoleYamlFile))
-		if err != nil {
-			log.Warnf("could not find role manifest: (%v)", err)
-		}
-		roleBinding, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.RoleBindingYamlFile))
-		if err != nil {
-			log.Warnf("could not find role_binding manifest: (%v)", err)
-		}
-		operator, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.OperatorYamlFile))
-		if err != nil {
-			log.Fatalf("could not find operator manifest: (%v)", err)
-		}
 		combined := []byte{}
-		combined = combineManifests(combined, sa)
-		combined = combineManifests(combined, role)
-		combined = combineManifests(combined, roleBinding)
-		combined = append(combined, operator...)
+		if !tlConfig.upLocal {
+			sa, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.ServiceAccountYamlFile))
+			if err != nil {
+				log.Warnf("could not find the serviceaccount manifest: (%v)", err)
+			}
+			role, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.RoleYamlFile))
+			if err != nil {
+				log.Warnf("could not find role manifest: (%v)", err)
+			}
+			roleBinding, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.RoleBindingYamlFile))
+			if err != nil {
+				log.Warnf("could not find role_binding manifest: (%v)", err)
+			}
+			operator, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.OperatorYamlFile))
+			if err != nil {
+				log.Fatalf("could not find operator manifest: (%v)", err)
+			}
+			combined = combineManifests(combined, sa)
+			combined = combineManifests(combined, role)
+			combined = combineManifests(combined, roleBinding)
+			combined = append(combined, operator...)
+		}
 		err = ioutil.WriteFile(tlConfig.namespacedManPath, combined, os.FileMode(fileutil.DefaultFileMode))
 		if err != nil {
 			log.Fatalf("could not create temporary namespaced manifest file: (%v)", err)
@@ -154,6 +161,9 @@ func testLocalFunc(cmd *cobra.Command, args []string) {
 	if tlConfig.namespace != "" {
 		testArgs = append(testArgs, "-"+test.SingleNamespaceFlag, "-parallel=1")
 	}
+	if tlConfig.upLocal {
+		testArgs = append(testArgs, "-"+test.LocalOperatorFlag)
+	}
 	dc := exec.Command("go", testArgs...)
 	dc.Env = append(os.Environ(), fmt.Sprintf("%v=%v", test.TestNamespaceEnv, tlConfig.namespace))
 	dc.Dir = projutil.MustGetwd()

doc/sdk-cli-reference.md

@@ -257,6 +257,7 @@ Runs the tests locally
 * `--namespaced-manifest` string - path to manifest for per-test, namespaced resources (default: combines deploy/service_account.yaml, deploy/rbac.yaml, and deploy/operator.yaml)
 *  `--namespace` string - if non-empty, single namespace to run tests in (e.g. "operator-test") (default: "")
 * `--go-test-flags` string - extra arguments to pass to `go test` (e.g. -f "-v -parallel=2")
+* `--up-local` - Enable running operator locally with go run instead of as an image in the cluster
 * `-h, --help` - help for local
 
 ##### Use

doc/test-framework/writing-e2e-tests.md

@@ -121,10 +121,9 @@ in your namespaced manifest. The custom `Create` function use the controller-run
 creates a cleanup function that is called by `ctx.Cleanup` which deletes the resource and then waits for the resource to be
 fully deleted before returning. This is configurable with `CleanupOptions`. For info on how to use `CleanupOptions` see
 [this section](#how-to-use-cleanup).
- 
 
 If you want to make sure the operator's deployment is fully ready before moving onto the next part of the
-test, the `WaitForDeployment` function from [e2eutil][e2eutil-link] (in the sdk under `pkg/test/e2eutil`) can be used:
+test, the `WaitForOperatorDeployment` function from [e2eutil][e2eutil-link] (in the sdk under `pkg/test/e2eutil`) can be used:
 
 ```go
 // get namespace
@@ -135,7 +134,7 @@ if err != nil {
 // get global framework variables
 f := framework.Global
 // wait for memcached-operator to be ready
-err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "memcached-operator", 1, time.Second*5, time.Second*30)
+err = e2eutil.WaitForOperatorDeployment(t, f.KubeClient, namespace, "memcached-operator", 1, time.Second*5, time.Second*30)
 if err != nil {
     t.Fatal(err)
 }
@@ -188,7 +187,10 @@ if err != nil {
 ```
 
 Now we can check if the operator successfully worked. In the case of the memcached operator, it should have
-created a deployment called "example-memcached" with 3 replicas:
+created a deployment called "example-memcached" with 3 replicas. To check, we use the `WaitForDeployment` function, which
+is the same as `WaitForOperatorDeployment` with the exception that `WaitForOperatorDeployment` will skip waiting
+for the deployment if the test is run locally and the `--up-local` flag is set; the `WaitForDeployment` function always
+waits for the deployment:
 
 ```go
 // wait for example-memcached to reach 3 replicas
@@ -243,6 +245,16 @@ $ kubectl create namespace operator-test
 $ operator-sdk test local ./test/e2e --namespace operator-test
 ```
 
+To run the operator itself locally during the tests instead of starting a deployment in the cluster, you can use the
+`--up-local` flag. This mode will still create global resources, but by default will not create any in-cluster namespaced
+resources unless the user specifies one through the `--namespaced-manifest` flag. (Note: the `--up-local` flag requires
+the `--namespace` flag):
+
+```shell
+$ kubectl create namespace operator-test
+$ operator-sdk test local ./test/e2e --namespace operator-test --up-local
+```
+
 For more documentation on the `operator-sdk test local` command, see the [SDK CLI Reference][sdk-cli-ref] doc.
 
 For advanced use cases, it is possible to run the tests via `go test` directly. As long as all flags defined

hack/tests/test-subcommand.sh

@@ -10,3 +10,7 @@ operator-sdk test local . --global-manifest deploy/crds/cache_v1alpha1_memcached
 kubectl create namespace test-memcached
 operator-sdk test local . --namespace=test-memcached
 kubectl delete namespace test-memcached
+# test operator in up local mode
+kubectl create namespace test-memcached
+operator-sdk test local . --up-local --namespace=test-memcached
+kubectl delete namespace test-memcached

pkg/test/e2eutil/wait_util.go

@@ -18,6 +18,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/operator-framework/operator-sdk/pkg/test"
+
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -29,6 +31,20 @@ import (
 // This can be used in multiple ways, like verifying that a required resource is ready before trying to use it, or to test
 // failure handling, like simulated in SimulatePodFail.
 func WaitForDeployment(t *testing.T, kubeclient kubernetes.Interface, namespace, name string, replicas int, retryInterval, timeout time.Duration) error {
+	return waitForDeployment(t, kubeclient, namespace, name, replicas, retryInterval, timeout, false)
+}
+
+// WaitForOperatorDeployment has the same functionality as WaitForDeployment but will no wait for the deployment if the
+// test was run with a locally run operator (--up-local flag)
+func WaitForOperatorDeployment(t *testing.T, kubeclient kubernetes.Interface, namespace, name string, replicas int, retryInterval, timeout time.Duration) error {
+	return waitForDeployment(t, kubeclient, namespace, name, replicas, retryInterval, timeout, true)
+}
+
+func waitForDeployment(t *testing.T, kubeclient kubernetes.Interface, namespace, name string, replicas int, retryInterval, timeout time.Duration, isOperator bool) error {
+	if isOperator && test.Global.LocalOperator {
+		t.Logf("Operator is running locally; skip waitForDeployment\n")
+		return nil
+	}
 	err := wait.Poll(retryInterval, timeout, func() (done bool, err error) {
 		deployment, err := kubeclient.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{IncludeUninitialized: true})
 		if err != nil {

pkg/test/framework.go

@@ -54,9 +54,10 @@ type Framework struct {
 	Scheme            *runtime.Scheme
 	NamespacedManPath *string
 	Namespace         string
+	LocalOperator     bool
 }
 
-func setup(kubeconfigPath, namespacedManPath *string) error {
+func setup(kubeconfigPath, namespacedManPath *string, localOperator bool) error {
 	var err error
 	var kubeconfig *rest.Config
 	if *kubeconfigPath == "incluster" {
@@ -105,6 +106,7 @@ func setup(kubeconfigPath, namespacedManPath *string) error {
 		Scheme:            scheme,
 		NamespacedManPath: namespacedManPath,
 		Namespace:         namespace,
+		LocalOperator:     localOperator,
 	}
 	return nil
 }

pkg/test/main_entry.go

@@ -15,11 +15,21 @@
 package test
 
 import (
+	"bytes"
 	"flag"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"syscall"
 	"testing"
 
+	"k8s.io/client-go/tools/clientcmd"
+
+	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
+	"github.com/operator-framework/operator-sdk/pkg/scaffold"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -30,6 +40,7 @@ const (
 	GlobalManPathFlag     = "globalMan"
 	SingleNamespaceFlag   = "singleNamespace"
 	TestNamespaceEnv      = "TEST_NAMESPACE"
+	LocalOperatorFlag     = "localOperator"
 )
 
 func MainEntry(m *testing.M) {
@@ -38,21 +49,63 @@ func MainEntry(m *testing.M) {
 	globalManPath := flag.String(GlobalManPathFlag, "", "path to operator manifest")
 	namespacedManPath := flag.String(NamespacedManPathFlag, "", "path to rbac manifest")
 	singleNamespace = flag.Bool(SingleNamespaceFlag, false, "enable single namespace mode")
+	localOperator := flag.Bool(LocalOperatorFlag, false, "enable if operator is running locally (not in cluster)")
 	flag.Parse()
 	// go test always runs from the test directory; change to project root
 	err := os.Chdir(*projRoot)
 	if err != nil {
 		log.Fatalf("failed to change directory to project root: %v", err)
 	}
-	if err := setup(kubeconfigPath, namespacedManPath); err != nil {
+	if err := setup(kubeconfigPath, namespacedManPath, *localOperator); err != nil {
 		log.Fatalf("failed to set up framework: %v", err)
 	}
+	// setup local operator command, but don't start it yet
+	var localCmd *exec.Cmd
+	var localCmdOutBuf, localCmdErrBuf bytes.Buffer
+	if *localOperator {
+		// taken from commands/operator-sdk/cmd/up/local.go
+		runArgs := append([]string{"run"}, []string{filepath.Join(scaffold.ManagerDir, scaffold.CmdFile)}...)
+		localCmd = exec.Command("go", runArgs...)
+		localCmd.Stdout = &localCmdOutBuf
+		localCmd.Stderr = &localCmdErrBuf
+		c := make(chan os.Signal)
+		signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+		go func() {
+			<-c
+			err := localCmd.Process.Signal(os.Interrupt)
+			if err != nil {
+				log.Fatalf("failed to terminate the operator: (%v)", err)
+			}
+			os.Exit(0)
+		}()
+		if *kubeconfigPath != "" {
+			localCmd.Env = append(os.Environ(), fmt.Sprintf("%v=%v", k8sutil.KubeConfigEnvVar, kubeconfigPath))
+		} else {
+			// we can hardcode index 0 as that is the highest priority kubeconfig to be loaded and will always
+			// be populated by NewDefaultClientConfigLoadingRules()
+			localCmd.Env = append(os.Environ(), fmt.Sprintf("%v=%v", k8sutil.KubeConfigEnvVar, clientcmd.NewDefaultClientConfigLoadingRules().Precedence[0]))
+		}
+		localCmd.Env = append(localCmd.Env, fmt.Sprintf("%v=%v", k8sutil.WatchNamespaceEnvVar, Global.Namespace))
+	}
 	// setup context to use when setting up crd
 	ctx := NewTestCtx(nil)
 	// os.Exit stops the program before the deferred functions run
 	// to fix this, we put the exit in the defer as well
 	defer func() {
+		// start local operator before running tests
+		if *localOperator {
+			err := localCmd.Start()
+			if err != nil {
+				log.Fatalf("failed to run operator locally: (%v)", err)
+			}
+			log.Info("started local operator")
+		}
 		exitCode := m.Run()
+		if *localOperator {
+			localCmd.Process.Kill()
+			log.Infof("local operator stdout: %s", string(localCmdOutBuf.Bytes()))
+			log.Infof("local operator stderr: %s", string(localCmdErrBuf.Bytes()))
+		}
 		ctx.CleanupNoT()
 		os.Exit(exitCode)
 	}()

pkg/test/resource_creator.go

@@ -66,6 +66,10 @@ func (ctx *TestCtx) createFromYAML(yamlFile []byte, skipIfExists bool, cleanupOp
 	}
 	yamlSplit := bytes.Split(yamlFile, []byte("\n---\n"))
 	for _, yamlSpec := range yamlSplit {
+		// some autogenerated files may include an extra `---` at the end of the file or be empty
+		if string(yamlSpec) == "" {
+			continue
+		}
 		yamlSpec, err = setNamespaceYAML(yamlSpec, namespace)
 		if err != nil {
 			return err

test/e2e/incluster-test-code/memcached_test.go.tmpl

@@ -114,7 +114,7 @@ func MemcachedCluster(t *testing.T) {
 	// get global framework variables
 	f := framework.Global
 	// wait for memcached-operator to be ready
-	err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "memcached-operator", 1, retryInterval, timeout)
+	err = e2eutil.WaitForOperatorDeployment(t, f.KubeClient, namespace, "memcached-operator", 1, retryInterval, timeout)
 	if err != nil {
 		t.Fatal(err)
 	}

test/test-framework/memcached_test.go

@@ -114,7 +114,7 @@ func MemcachedCluster(t *testing.T) {
 	// get global framework variables
 	f := framework.Global
 	// wait for memcached-operator to be ready
-	err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "memcached-operator", 1, retryInterval, timeout)
+	err = e2eutil.WaitForOperatorDeployment(t, f.KubeClient, namespace, "memcached-operator", 1, retryInterval, timeout)
 	if err != nil {
 		t.Fatal(err)
 	}