commands/.../test,pkg/test: add --up-local flag to test local (#781)

* commands/.../test,pkg/test: add --up-local flag to test local

Author: AlexNPavel

CHANGELOG.md

@@ -8,6 +8,7 @@
 ### Added
 
 - A new command [`operator-sdk print-deps`](https://github.com/operator-framework/operator-sdk/blob/master/doc/sdk-cli-reference.md#print-deps) which prints Golang packages and versions expected by the current Operator SDK version. Supplying `--as-file` prints packages and versions in Gopkg.toml format. ([#772](https://github.com/operator-framework/operator-sdk/pull/772))
+- Add [`up-local`](https://github.com/operator-framework/operator-sdk/blob/master/doc/sdk-cli-reference.md#flags-9) flag to `test local` subcommand ([#781](https://github.com/operator-framework/operator-sdk/pull/781))
 - Add [`no-setup`](https://github.com/operator-framework/operator-sdk/blob/master/doc/sdk-cli-reference.md#flags-9) flag to `test local` subcommand ([#770](https://github.com/operator-framework/operator-sdk/pull/770))
 - Add [`image`](https://github.com/operator-framework/operator-sdk/blob/master/doc/sdk-cli-reference.md#flags-9) flag to `test local` subcommand ([#768](https://github.com/operator-framework/operator-sdk/pull/768))
 - Ansible Operator log output includes much more information for troubleshooting ansible errors. ([#713](https://github.com/operator-framework/operator-sdk/pull/713))

Gopkg.lock

@@ -45,6 +45,7 @@ type testLocalConfig struct {
 	namespacedManPath string
 	goTestFlags       string
 	namespace         string
+	upLocal           bool
 	noSetup           bool
 	image             string
 }
@@ -62,6 +63,7 @@ func NewTestLocalCmd() *cobra.Command {
 	testCmd.Flags().StringVar(&tlConfig.namespacedManPath, "namespaced-manifest", "", "Path to manifest for per-test, namespaced resources (e.g. RBAC and Operator manifest)")
 	testCmd.Flags().StringVar(&tlConfig.goTestFlags, "go-test-flags", "", "Additional flags to pass to go test")
 	testCmd.Flags().StringVar(&tlConfig.namespace, "namespace", "", "If non-empty, single namespace to run tests in")
+	testCmd.Flags().BoolVar(&tlConfig.upLocal, "up-local", false, "Enable running operator locally with go run instead of as an image in the cluster")
 	testCmd.Flags().BoolVar(&tlConfig.noSetup, "no-setup", false, "Disable test resource creation")
 	testCmd.Flags().StringVar(&tlConfig.image, "image", "", "Use a different operator image from the one specified in the namespaced manifest")
 
@@ -76,6 +78,10 @@ func testLocalFunc(cmd *cobra.Command, args []string) {
 		log.Fatal("the global-manifest and namespaced-manifest flags cannot be enabled at the same time as the no-setup flag")
 	}
 
+	if tlConfig.upLocal && tlConfig.namespace == "" {
+		log.Fatal("must specify a namespace to run in when -up-local flag is set")
+	}
+
 	log.Info("Testing operator locally.")
 
 	// if no namespaced manifest path is given, combine deploy/service_account.yaml, deploy/role.yaml, deploy/role_binding.yaml and deploy/operator.yaml
@@ -85,28 +91,29 @@ func testLocalFunc(cmd *cobra.Command, args []string) {
 			log.Fatalf("could not create %s: (%v)", deployTestDir, err)
 		}
 		tlConfig.namespacedManPath = filepath.Join(deployTestDir, "namespace-manifests.yaml")
-
-		sa, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.ServiceAccountYamlFile))
-		if err != nil {
-			log.Warnf("could not find the serviceaccount manifest: (%v)", err)
-		}
-		role, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.RoleYamlFile))
-		if err != nil {
-			log.Warnf("could not find role manifest: (%v)", err)
-		}
-		roleBinding, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.RoleBindingYamlFile))
-		if err != nil {
-			log.Warnf("could not find role_binding manifest: (%v)", err)
-		}
-		operator, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.OperatorYamlFile))
-		if err != nil {
-			log.Fatalf("could not find operator manifest: (%v)", err)
-		}
 		combined := []byte{}
-		combined = combineManifests(combined, sa)
-		combined = combineManifests(combined, role)
-		combined = combineManifests(combined, roleBinding)
-		combined = append(combined, operator...)
+		if !tlConfig.upLocal {
+			sa, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.ServiceAccountYamlFile))
+			if err != nil {
+				log.Warnf("could not find the serviceaccount manifest: (%v)", err)
+			}
+			role, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.RoleYamlFile))
+			if err != nil {
+				log.Warnf("could not find role manifest: (%v)", err)
+			}
+			roleBinding, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.RoleBindingYamlFile))
+			if err != nil {
+				log.Warnf("could not find role_binding manifest: (%v)", err)
+			}
+			operator, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.OperatorYamlFile))
+			if err != nil {
+				log.Fatalf("could not find operator manifest: (%v)", err)
+			}
+			combined = combineManifests(combined, sa)
+			combined = combineManifests(combined, role)
+			combined = combineManifests(combined, roleBinding)
+			combined = append(combined, operator...)
+		}
 		err = ioutil.WriteFile(tlConfig.namespacedManPath, combined, os.FileMode(fileutil.DefaultFileMode))
 		if err != nil {
 			log.Fatalf("could not create temporary namespaced manifest file: (%v)", err)
@@ -191,6 +198,9 @@ func testLocalFunc(cmd *cobra.Command, args []string) {
 	if tlConfig.namespace != "" || tlConfig.noSetup {
 		testArgs = append(testArgs, "-"+test.SingleNamespaceFlag, "-parallel=1")
 	}
+	if tlConfig.upLocal {
+		testArgs = append(testArgs, "-"+test.LocalOperatorFlag)
+	}
 	dc := exec.Command("go", testArgs...)
 	dc.Env = append(os.Environ(), fmt.Sprintf("%v=%v", test.TestNamespaceEnv, tlConfig.namespace))
 	dc.Dir = projutil.MustGetwd()

commands/operator-sdk/cmd/test/local.go

@@ -276,7 +276,8 @@ Runs the tests locally
 * `--namespaced-manifest` string - path to manifest for per-test, namespaced resources (default: combines deploy/service_account.yaml, deploy/rbac.yaml, and deploy/operator.yaml)
 *  `--namespace` string - if non-empty, single namespace to run tests in (e.g. "operator-test") (default: "")
 * `--go-test-flags` string - extra arguments to pass to `go test` (e.g. -f "-v -parallel=2")
-* `--no-setup` bool - disable test resource creation
+* `--up-local` - enable running operator locally with go run instead of as an image in the cluster
+* `--no-setup` - disable test resource creation
 * `--image` string - use a different operator image from the one specified in the namespaced manifest
 * `-h, --help` - help for local
 

doc/sdk-cli-reference.md

@@ -121,10 +121,9 @@ in your namespaced manifest. The custom `Create` function use the controller-run
 creates a cleanup function that is called by `ctx.Cleanup` which deletes the resource and then waits for the resource to be
 fully deleted before returning. This is configurable with `CleanupOptions`. For info on how to use `CleanupOptions` see
 [this section](#how-to-use-cleanup).
- 
 
 If you want to make sure the operator's deployment is fully ready before moving onto the next part of the
-test, the `WaitForDeployment` function from [e2eutil][e2eutil-link] (in the sdk under `pkg/test/e2eutil`) can be used:
+test, the `WaitForOperatorDeployment` function from [e2eutil][e2eutil-link] (in the sdk under `pkg/test/e2eutil`) can be used:
 
 ```go
 // get namespace
@@ -135,7 +134,7 @@ if err != nil {
 // get global framework variables
 f := framework.Global
 // wait for memcached-operator to be ready
-err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "memcached-operator", 1, time.Second*5, time.Second*30)
+err = e2eutil.WaitForOperatorDeployment(t, f.KubeClient, namespace, "memcached-operator", 1, time.Second*5, time.Second*30)
 if err != nil {
     t.Fatal(err)
 }
@@ -188,7 +187,10 @@ if err != nil {
 ```
 
 Now we can check if the operator successfully worked. In the case of the memcached operator, it should have
-created a deployment called "example-memcached" with 3 replicas:
+created a deployment called "example-memcached" with 3 replicas. To check, we use the `WaitForDeployment` function, which
+is the same as `WaitForOperatorDeployment` with the exception that `WaitForOperatorDeployment` will skip waiting
+for the deployment if the test is run locally and the `--up-local` flag is set; the `WaitForDeployment` function always
+waits for the deployment:
 
 ```go
 // wait for example-memcached to reach 3 replicas
@@ -228,28 +230,48 @@ default test settings, such as locations of your global resource manifest file (
 `deploy/crd.yaml`) and your namespaced resource manifest file (by default `deploy/service_account.yaml` concatenated with
 `deploy/rbac.yaml` and `deploy/operator.yaml`), and allows the user to configure runtime options. There are 2 ways to use the
 subcommand: local and cluster.
+
 ### Local
+
 To run the tests locally, run the `operator-sdk test local` command in your project root and pass the location of the tests
 as an argument. You can use `--help` to view the other configuration options and use `--go-test-flags` to pass in arguments to `go test`. Here is an example command:
 
 ```shell
 $ operator-sdk test local ./test/e2e --go-test-flags "-v -parallel=2"
 ```
 
+#### Image Flag
+
 If you wish to specify a different operator image than specified in your `operator.yaml` file (or a user-specified
 namespaced manifest file), you can use the `--image` flag:
 
 ```shell
 $ operator-sdk test local ./test/e2e --image quay.io/example/my-operator:v0.0.2
 ```
 
+#### Namespace Flag
+
 If you wish to run all the tests in 1 namespace (which also forces `-parallel=1`), you can use the `--namespace` flag:
 
 ```shell
 $ kubectl create namespace operator-test
 $ operator-sdk test local ./test/e2e --namespace operator-test
 ```
 
+#### Up-Local Flag
+
+To run the operator itself locally during the tests instead of starting a deployment in the cluster, you can use the
+`--up-local` flag. This mode will still create global resources, but by default will not create any in-cluster namespaced
+resources unless the user specifies one through the `--namespaced-manifest` flag. (Note: the `--up-local` flag requires
+the `--namespace` flag):
+
+```shell
+$ kubectl create namespace operator-test
+$ operator-sdk test local ./test/e2e --namespace operator-test --up-local
+```
+
+#### No-Setup Flag
+
 If you would prefer to create the resources yourself and skip resource creation, you can use the `--no-setup` flag:
 ```shell
 $ kubectl create namespace operator-test
@@ -263,6 +285,8 @@ $ operator-sdk test local ./test/e2e --namespace operator-test --no-setup
 
 For more documentation on the `operator-sdk test local` command, see the [SDK CLI Reference][sdk-cli-ref] doc.
 
+#### Running Go Test Directly (Not Recommended)
+
 For advanced use cases, it is possible to run the tests via `go test` directly. As long as all flags defined
 in [MainEntry][main-entry-link] are declared, the tests will run correctly. Running the tests directly with missing flags
 will result in undefined behavior. This is an example `go test` equivalent to the `operator-sdk test local` example above:

doc/test-framework/writing-e2e-tests.md

@@ -5,15 +5,22 @@ set -ex
 
 pushd test/test-framework
 # test framework with defaults
-operator-sdk test local .
+operator-sdk test local ./test/e2e
+
 # test operator-sdk test flags
-operator-sdk test local . --global-manifest deploy/crds/cache_v1alpha1_memcached_crd.yaml --namespaced-manifest deploy/namespace-init.yaml --go-test-flags "-parallel 1" --kubeconfig $HOME/.kube/config --image=quay.io/coreos/operator-sdk-dev:test-framework-operator-runtime
-# test operator-sdk test local single namespace mode
-kubectl create namespace test-memcached
+operator-sdk test local ./test/e2e --global-manifest deploy/crds/cache_v1alpha1_memcached_crd.yaml --namespaced-manifest deploy/namespace-init.yaml --go-test-flags "-parallel 1" --kubeconfig $HOME/.kube/config --image=quay.io/coreos/operator-sdk-dev:test-framework-operator-runtime
+
 # we use the test-memcached namespace for all future tests, so we only need to set this trap once
+kubectl create namespace test-memcached
 trap_add 'kubectl delete namespace test-memcached || true' EXIT
-operator-sdk test local . --namespace=test-memcached
+operator-sdk test local ./test/e2e --namespace=test-memcached
 kubectl delete namespace test-memcached
+
+# test operator in up local mode
+kubectl create namespace test-memcached
+operator-sdk test local ./test/e2e --up-local --namespace=test-memcached
+kubectl delete namespace test-memcached
+
 # test operator in no-setup mode
 kubectl create namespace test-memcached
 kubectl create -f deploy/crds/cache_v1alpha1_memcached_crd.yaml
@@ -23,6 +30,6 @@ kubectl create -f deploy/service_account.yaml --namespace test-memcached
 kubectl create -f deploy/role.yaml --namespace test-memcached
 kubectl create -f deploy/role_binding.yaml --namespace test-memcached
 kubectl create -f deploy/operator.yaml --namespace test-memcached
-operator-sdk test local . --namespace=test-memcached --no-setup
+operator-sdk test local ./test/e2e --namespace=test-memcached --no-setup
 kubectl delete namespace test-memcached
 popd

hack/tests/test-subcommand.sh

@@ -18,15 +18,14 @@ import (
 	"testing"
 
 	"github.com/operator-framework/operator-sdk/pkg/scaffold/internal/testutil"
-	"github.com/operator-framework/operator-sdk/pkg/test"
 )
 
 func TestPodTest(t *testing.T) {
 	s, buf := setupScaffoldAndWriter()
 	err := s.Execute(appConfig,
 		&TestPod{
 			Image:            "quay.io/app/operator:v1.0.0",
-			TestNamespaceEnv: test.TestNamespaceEnv,
+			TestNamespaceEnv: "TEST_NAMESPACE",
 		})
 	if err != nil {
 		t.Fatalf("failed to execute the scaffold: (%v)", err)