From 41b25a42a5cf9b6b0458f8a2123360512e689654 Mon Sep 17 00:00:00 2001 From: Brett Tofel Date: Mon, 20 May 2024 14:15:32 -0400 Subject: [PATCH 1/9] Proposed way to unskip some tests TestClusterExtensionChannelVersionExists mostly restored here Signed-off-by: Brett Tofel --- go.mod | 1 + .../clusterextension_controller_test.go | 55 +++++++++++----- internal/controllers/suite_test.go | 63 ++++++++++++++++--- 3 files changed, 94 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index fd378c54b..c740e962d 100644 --- a/go.mod +++ b/go.mod @@ -165,6 +165,7 @@ require ( github.com/skeema/knownhosts v1.2.2 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/ulikunitz/xz v0.5.11 // indirect github.com/vbatts/tar-split v0.11.5 // indirect diff --git a/internal/controllers/clusterextension_controller_test.go b/internal/controllers/clusterextension_controller_test.go index b0b121188..e83e16a42 100644 --- a/internal/controllers/clusterextension_controller_test.go +++ b/internal/controllers/clusterextension_controller_test.go @@ -3,16 +3,19 @@ package controllers_test import ( "context" "encoding/json" + "errors" "fmt" "testing" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/rand" featuregatetesting "k8s.io/component-base/featuregate/testing" ctrl "sigs.k8s.io/controller-runtime" @@ -21,6 +24,7 @@ import ( "github.com/operator-framework/operator-registry/alpha/declcfg" "github.com/operator-framework/operator-registry/alpha/property" rukpakv1alpha2 "github.com/operator-framework/rukpak/api/v1alpha2" + "github.com/operator-framework/rukpak/pkg/source" ocv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1" "github.com/operator-framework/operator-controller/internal/catalogmetadata" @@ -186,8 +190,17 @@ func TestClusterExtensionBundleDeploymentDoesNotExist(t *testing.T) { } func TestClusterExtensionChannelVersionExists(t *testing.T) { - t.Skip("Skip and replace with e2e for BundleDeployment") cl, reconciler := newClientAndReconciler(t) + mockUnpacker := unp.(*MockUnpacker) + // Set up the Unpack method to return a result with StateUnpacked + mockUnpacker.On("Unpack", mock.Anything, mock.AnythingOfType("*v1alpha2.BundleDeployment")).Return(&source.Result{ + State: source.StateUnpacked, + }, nil) + mockStorage := sto.(*MockStorage) + loadError := errors.New("load error") + mockStorage.On("Store", mock.Anything, mock.Anything, mock.Anything).Return(nil) + // Setting up the mock for Load method to return an error to avoid further mocking + mockStorage.On("Load", mock.Anything, mock.Anything, mock.Anything).Return(nil, loadError) ctx := context.Background() extKey := types.NamespacedName{Name: fmt.Sprintf("cluster-extension-test-%s", rand.String(8))} @@ -214,7 +227,10 @@ func TestClusterExtensionChannelVersionExists(t *testing.T) { t.Log("By running reconcile") res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey}) require.Equal(t, ctrl.Result{}, res) - require.NoError(t, err) + // Asserting that only loadError occurred + if assert.Error(t, err) { + assert.Equal(t, utilerrors.NewAggregate([]error{loadError}), err) + } t.Log("By fetching updated cluster extension after reconcile") require.NoError(t, cl.Get(ctx, extKey, clusterExtension)) @@ -229,22 +245,27 @@ func TestClusterExtensionChannelVersionExists(t *testing.T) { require.Equal(t, metav1.ConditionTrue, cond.Status) require.Equal(t, ocv1alpha1.ReasonSuccess, cond.Reason) require.Equal(t, "resolved to \"quay.io/operatorhubio/prometheus@fake1.0.0\"", cond.Message) - cond = apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeInstalled) - require.NotNil(t, cond) - require.Equal(t, metav1.ConditionUnknown, cond.Status) - require.Equal(t, ocv1alpha1.ReasonInstallationStatusUnknown, cond.Reason) - require.Equal(t, "bundledeployment status is unknown", cond.Message) - - t.Log("By fetching the bundled deployment") - bd := &rukpakv1alpha2.BundleDeployment{} - require.NoError(t, cl.Get(ctx, types.NamespacedName{Name: extKey.Name}, bd)) - require.Equal(t, "core-rukpak-io-registry", bd.Spec.ProvisionerClassName) - require.Equal(t, installNamespace, bd.Spec.InstallNamespace) - require.Equal(t, rukpakv1alpha2.SourceTypeImage, bd.Spec.Source.Type) - require.NotNil(t, bd.Spec.Source.Image) - require.Equal(t, "quay.io/operatorhubio/prometheus@fake1.0.0", bd.Spec.Source.Image.Ref) - verifyInvariants(ctx, t, reconciler.Client, clusterExtension) + // we're skipping based on same loadError since Status.Condition setting path is skipped and bundle deployment is absent + // also invariants will be different + if err != nil && err.Error() != utilerrors.NewAggregate([]error{loadError}).Error() { + cond = apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeInstalled) + require.NotNil(t, cond) + require.Equal(t, metav1.ConditionUnknown, cond.Status) + require.Equal(t, ocv1alpha1.ReasonInstallationStatusUnknown, cond.Reason) + require.Equal(t, "bundledeployment status is unknown", cond.Message) + + t.Log("By fetching the bundled deployment") + bd := &rukpakv1alpha2.BundleDeployment{} + require.NoError(t, cl.Get(ctx, types.NamespacedName{Name: extKey.Name}, bd)) + require.Equal(t, "core-rukpak-io-registry", bd.Spec.ProvisionerClassName) + require.Equal(t, installNamespace, bd.Spec.InstallNamespace) + require.Equal(t, rukpakv1alpha2.SourceTypeImage, bd.Spec.Source.Type) + require.NotNil(t, bd.Spec.Source.Image) + require.Equal(t, "quay.io/operatorhubio/prometheus@fake1.0.0", bd.Spec.Source.Image.Ref) + + verifyInvariants(ctx, t, reconciler.Client, clusterExtension) + } require.NoError(t, cl.DeleteAllOf(ctx, &ocv1alpha1.ClusterExtension{})) } diff --git a/internal/controllers/suite_test.go b/internal/controllers/suite_test.go index 9f4bf8cde..56c4ff017 100644 --- a/internal/controllers/suite_test.go +++ b/internal/controllers/suite_test.go @@ -17,29 +17,76 @@ limitations under the License. package controllers_test import ( - "crypto/x509" + "context" + "io/fs" "log" + "net/http" "os" "path/filepath" "testing" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/api/meta" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/manager" helmclient "github.com/operator-framework/helm-operator-plugins/pkg/client" + "github.com/operator-framework/rukpak/api/v1alpha2" "github.com/operator-framework/rukpak/pkg/source" - "github.com/operator-framework/rukpak/pkg/util" + "github.com/operator-framework/rukpak/pkg/storage" "github.com/operator-framework/operator-controller/internal/controllers" "github.com/operator-framework/operator-controller/pkg/scheme" testutil "github.com/operator-framework/operator-controller/test/util" ) +// MockUnpacker is a mock of Unpacker interface +type MockUnpacker struct { + mock.Mock +} + +// Unpack mocks the Unpack method +func (m *MockUnpacker) Unpack(ctx context.Context, bd *v1alpha2.BundleDeployment) (*source.Result, error) { + args := m.Called(ctx, bd) + return args.Get(0).(*source.Result), args.Error(1) +} + +// MockStorage is a mock of Storage interface +type MockStorage struct { + mock.Mock +} + +func (m *MockStorage) Load(ctx context.Context, owner client.Object) (fs.FS, error) { + args := m.Called(ctx, owner) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(fs.FS), args.Error(1) +} + +func (m *MockStorage) Delete(ctx context.Context, owner client.Object) error { + //TODO implement me + panic("implement me") +} + +func (m *MockStorage) ServeHTTP(writer http.ResponseWriter, request *http.Request) { + //TODO implement me + panic("implement me") +} + +func (m *MockStorage) URLFor(ctx context.Context, owner client.Object) (string, error) { + //TODO implement me + panic("implement me") +} + +func (m *MockStorage) Store(ctx context.Context, owner client.Object, bundle fs.FS) error { + args := m.Called(ctx, owner, bundle) + return args.Error(0) +} + func newClient(t *testing.T) client.Client { cl, err := client.New(cfg, client.Options{Scheme: scheme.Scheme}) require.NoError(t, err) @@ -55,6 +102,7 @@ func newClientAndReconciler(t *testing.T) (client.Client, *controllers.ClusterEx BundleProvider: &fakeCatalogClient, ActionClientGetter: acg, Unpacker: unp, + Storage: sto, } return cl, reconciler } @@ -62,7 +110,8 @@ func newClientAndReconciler(t *testing.T) (client.Client, *controllers.ClusterEx var ( cfg *rest.Config acg helmclient.ActionClientGetter - unp source.Unpacker + unp source.Unpacker // Interface, will be initialized as a mock in TestMain + sto storage.Storage ) func TestMain(m *testing.M) { @@ -85,10 +134,8 @@ func TestMain(m *testing.M) { acg, err = helmclient.NewActionClientGetter(cfgGetter) utilruntime.Must(err) - mgr, err := manager.New(cfg, manager.Options{}) - utilruntime.Must(err) - unp, err = source.NewDefaultUnpacker(mgr, util.DefaultSystemNamespace, util.DefaultUnpackImage, (*x509.CertPool)(nil)) - utilruntime.Must(err) + unp = new(MockUnpacker) + sto = new(MockStorage) code := m.Run() utilruntime.Must(testEnv.Stop()) From 50ab93e8b70900573fb967ca207bdb591836ebed Mon Sep 17 00:00:00 2001 From: Brett Tofel Date: Tue, 21 May 2024 10:05:05 -0400 Subject: [PATCH 2/9] Refactors global variable names Signed-off-by: Brett Tofel --- .../clusterextension_controller_test.go | 4 +-- ...terextension_registryv1_validation_test.go | 2 +- internal/controllers/suite_test.go | 28 +++++++++---------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/internal/controllers/clusterextension_controller_test.go b/internal/controllers/clusterextension_controller_test.go index e83e16a42..156baae24 100644 --- a/internal/controllers/clusterextension_controller_test.go +++ b/internal/controllers/clusterextension_controller_test.go @@ -191,12 +191,12 @@ func TestClusterExtensionBundleDeploymentDoesNotExist(t *testing.T) { func TestClusterExtensionChannelVersionExists(t *testing.T) { cl, reconciler := newClientAndReconciler(t) - mockUnpacker := unp.(*MockUnpacker) + mockUnpacker := unpacker.(*MockUnpacker) // Set up the Unpack method to return a result with StateUnpacked mockUnpacker.On("Unpack", mock.Anything, mock.AnythingOfType("*v1alpha2.BundleDeployment")).Return(&source.Result{ State: source.StateUnpacked, }, nil) - mockStorage := sto.(*MockStorage) + mockStorage := store.(*MockStorage) loadError := errors.New("load error") mockStorage.On("Store", mock.Anything, mock.Anything, mock.Anything).Return(nil) // Setting up the mock for Load method to return an error to avoid further mocking diff --git a/internal/controllers/clusterextension_registryv1_validation_test.go b/internal/controllers/clusterextension_registryv1_validation_test.go index 3d14b3ac4..b6f1f72f3 100644 --- a/internal/controllers/clusterextension_registryv1_validation_test.go +++ b/internal/controllers/clusterextension_registryv1_validation_test.go @@ -112,7 +112,7 @@ func TestClusterExtensionRegistryV1DisallowDependencies(t *testing.T) { reconciler := &controllers.ClusterExtensionReconciler{ Client: cl, BundleProvider: &fakeCatalogClient, - ActionClientGetter: acg, + ActionClientGetter: helmClientGetter, } installNamespace := fmt.Sprintf("test-ns-%s", rand.String(8)) diff --git a/internal/controllers/suite_test.go b/internal/controllers/suite_test.go index 56c4ff017..a46e1bc6d 100644 --- a/internal/controllers/suite_test.go +++ b/internal/controllers/suite_test.go @@ -88,7 +88,7 @@ func (m *MockStorage) Store(ctx context.Context, owner client.Object, bundle fs. } func newClient(t *testing.T) client.Client { - cl, err := client.New(cfg, client.Options{Scheme: scheme.Scheme}) + cl, err := client.New(config, client.Options{Scheme: scheme.Scheme}) require.NoError(t, err) require.NotNil(t, cl) return cl @@ -100,18 +100,18 @@ func newClientAndReconciler(t *testing.T) (client.Client, *controllers.ClusterEx reconciler := &controllers.ClusterExtensionReconciler{ Client: cl, BundleProvider: &fakeCatalogClient, - ActionClientGetter: acg, - Unpacker: unp, - Storage: sto, + ActionClientGetter: helmClientGetter, + Unpacker: unpacker, + Storage: store, } return cl, reconciler } var ( - cfg *rest.Config - acg helmclient.ActionClientGetter - unp source.Unpacker // Interface, will be initialized as a mock in TestMain - sto storage.Storage + config *rest.Config + helmClientGetter helmclient.ActionClientGetter + unpacker source.Unpacker // Interface, will be initialized as a mock in TestMain + store storage.Storage ) func TestMain(m *testing.M) { @@ -122,20 +122,20 @@ func TestMain(m *testing.M) { } var err error - cfg, err = testEnv.Start() + config, err = testEnv.Start() utilruntime.Must(err) - if cfg == nil { + if config == nil { log.Panic("expected cfg to not be nil") } rm := meta.NewDefaultRESTMapper(nil) - cfgGetter, err := helmclient.NewActionConfigGetter(cfg, rm) + cfgGetter, err := helmclient.NewActionConfigGetter(config, rm) utilruntime.Must(err) - acg, err = helmclient.NewActionClientGetter(cfgGetter) + helmClientGetter, err = helmclient.NewActionClientGetter(cfgGetter) utilruntime.Must(err) - unp = new(MockUnpacker) - sto = new(MockStorage) + unpacker = new(MockUnpacker) + store = new(MockStorage) code := m.Run() utilruntime.Must(testEnv.Stop()) From 0949f718de14f21da634ee72c0a9f33ee171c7de Mon Sep 17 00:00:00 2001 From: Brett Tofel Date: Tue, 21 May 2024 10:11:56 -0400 Subject: [PATCH 3/9] Removes BundleDeployment related checking In just one test, for now. Signed-off-by: Brett Tofel --- .../controllers/clusterextension_controller_test.go | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/internal/controllers/clusterextension_controller_test.go b/internal/controllers/clusterextension_controller_test.go index 156baae24..36843d676 100644 --- a/internal/controllers/clusterextension_controller_test.go +++ b/internal/controllers/clusterextension_controller_test.go @@ -246,8 +246,8 @@ func TestClusterExtensionChannelVersionExists(t *testing.T) { require.Equal(t, ocv1alpha1.ReasonSuccess, cond.Reason) require.Equal(t, "resolved to \"quay.io/operatorhubio/prometheus@fake1.0.0\"", cond.Message) - // we're skipping based on same loadError since Status.Condition setting path is skipped and bundle deployment is absent - // also invariants will be different + // we're skipping based on same loadError since Status.Condition setting path is skipped + // and invariants will be different if err != nil && err.Error() != utilerrors.NewAggregate([]error{loadError}).Error() { cond = apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeInstalled) require.NotNil(t, cond) @@ -255,15 +255,6 @@ func TestClusterExtensionChannelVersionExists(t *testing.T) { require.Equal(t, ocv1alpha1.ReasonInstallationStatusUnknown, cond.Reason) require.Equal(t, "bundledeployment status is unknown", cond.Message) - t.Log("By fetching the bundled deployment") - bd := &rukpakv1alpha2.BundleDeployment{} - require.NoError(t, cl.Get(ctx, types.NamespacedName{Name: extKey.Name}, bd)) - require.Equal(t, "core-rukpak-io-registry", bd.Spec.ProvisionerClassName) - require.Equal(t, installNamespace, bd.Spec.InstallNamespace) - require.Equal(t, rukpakv1alpha2.SourceTypeImage, bd.Spec.Source.Type) - require.NotNil(t, bd.Spec.Source.Image) - require.Equal(t, "quay.io/operatorhubio/prometheus@fake1.0.0", bd.Spec.Source.Image.Ref) - verifyInvariants(ctx, t, reconciler.Client, clusterExtension) } require.NoError(t, cl.DeleteAllOf(ctx, &ocv1alpha1.ClusterExtension{})) From 6d008b9367b0acfb5356080a801cde08d314ebd5 Mon Sep 17 00:00:00 2001 From: Varsha Prasad Narsing Date: Tue, 21 May 2024 11:40:52 -0700 Subject: [PATCH 4/9] fix unit test - TestClusterExtensionChannelVersionExists --- .../clusterextension_controller_test.go | 42 +++++++------------ .../controllers/clusterextension_status.go | 3 -- 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/internal/controllers/clusterextension_controller_test.go b/internal/controllers/clusterextension_controller_test.go index 36843d676..93b4fe7c3 100644 --- a/internal/controllers/clusterextension_controller_test.go +++ b/internal/controllers/clusterextension_controller_test.go @@ -3,7 +3,6 @@ package controllers_test import ( "context" "encoding/json" - "errors" "fmt" "testing" @@ -15,7 +14,6 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/rand" featuregatetesting "k8s.io/component-base/featuregate/testing" ctrl "sigs.k8s.io/controller-runtime" @@ -194,13 +192,9 @@ func TestClusterExtensionChannelVersionExists(t *testing.T) { mockUnpacker := unpacker.(*MockUnpacker) // Set up the Unpack method to return a result with StateUnpacked mockUnpacker.On("Unpack", mock.Anything, mock.AnythingOfType("*v1alpha2.BundleDeployment")).Return(&source.Result{ - State: source.StateUnpacked, + State: source.StatePending, }, nil) - mockStorage := store.(*MockStorage) - loadError := errors.New("load error") - mockStorage.On("Store", mock.Anything, mock.Anything, mock.Anything).Return(nil) - // Setting up the mock for Load method to return an error to avoid further mocking - mockStorage.On("Load", mock.Anything, mock.Anything, mock.Anything).Return(nil, loadError) + ctx := context.Background() extKey := types.NamespacedName{Name: fmt.Sprintf("cluster-extension-test-%s", rand.String(8))} @@ -227,10 +221,7 @@ func TestClusterExtensionChannelVersionExists(t *testing.T) { t.Log("By running reconcile") res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey}) require.Equal(t, ctrl.Result{}, res) - // Asserting that only loadError occurred - if assert.Error(t, err) { - assert.Equal(t, utilerrors.NewAggregate([]error{loadError}), err) - } + require.NoError(t, err, nil) t.Log("By fetching updated cluster extension after reconcile") require.NoError(t, cl.Get(ctx, extKey, clusterExtension)) @@ -240,23 +231,18 @@ func TestClusterExtensionChannelVersionExists(t *testing.T) { require.Empty(t, clusterExtension.Status.InstalledBundle) t.Log("By checking the expected conditions") - cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeResolved) - require.NotNil(t, cond) - require.Equal(t, metav1.ConditionTrue, cond.Status) - require.Equal(t, ocv1alpha1.ReasonSuccess, cond.Reason) - require.Equal(t, "resolved to \"quay.io/operatorhubio/prometheus@fake1.0.0\"", cond.Message) - - // we're skipping based on same loadError since Status.Condition setting path is skipped - // and invariants will be different - if err != nil && err.Error() != utilerrors.NewAggregate([]error{loadError}).Error() { - cond = apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeInstalled) - require.NotNil(t, cond) - require.Equal(t, metav1.ConditionUnknown, cond.Status) - require.Equal(t, ocv1alpha1.ReasonInstallationStatusUnknown, cond.Reason) - require.Equal(t, "bundledeployment status is unknown", cond.Message) + resolvedCond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeResolved) + require.NotNil(t, resolvedCond) + require.Equal(t, metav1.ConditionTrue, resolvedCond.Status) + require.Equal(t, ocv1alpha1.ReasonSuccess, resolvedCond.Reason) + require.Equal(t, "resolved to \"quay.io/operatorhubio/prometheus@fake1.0.0\"", resolvedCond.Message) + + t.Log("By checking the expected unpacked conditions") + unpackedCond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, rukpakv1alpha2.TypeUnpacked) + require.NotNil(t, unpackedCond) + require.Equal(t, metav1.ConditionFalse, unpackedCond.Status) + require.Equal(t, rukpakv1alpha2.ReasonUnpackPending, unpackedCond.Reason) - verifyInvariants(ctx, t, reconciler.Client, clusterExtension) - } require.NoError(t, cl.DeleteAllOf(ctx, &ocv1alpha1.ClusterExtension{})) } diff --git a/internal/controllers/clusterextension_status.go b/internal/controllers/clusterextension_status.go index ba5de76cd..eff77feb0 100644 --- a/internal/controllers/clusterextension_status.go +++ b/internal/controllers/clusterextension_status.go @@ -27,7 +27,6 @@ import ( ) func updateStatusUnpackFailing(status *ocv1alpha1.ClusterExtensionStatus, err error) error { - status.ResolvedBundle = nil status.InstalledBundle = nil meta.SetStatusCondition(&status.Conditions, metav1.Condition{ Type: rukpakv1alpha2.TypeUnpacked, @@ -40,7 +39,6 @@ func updateStatusUnpackFailing(status *ocv1alpha1.ClusterExtensionStatus, err er // TODO: verify if we need to update the installBundle status or leave it as is. func updateStatusUnpackPending(status *ocv1alpha1.ClusterExtensionStatus, result *source.Result) { - status.ResolvedBundle = nil status.InstalledBundle = nil meta.SetStatusCondition(&status.Conditions, metav1.Condition{ Type: rukpakv1alpha2.TypeUnpacked, @@ -52,7 +50,6 @@ func updateStatusUnpackPending(status *ocv1alpha1.ClusterExtensionStatus, result // TODO: verify if we need to update the installBundle status or leave it as is. func updateStatusUnpacking(status *ocv1alpha1.ClusterExtensionStatus, result *source.Result) { - status.ResolvedBundle = nil status.InstalledBundle = nil meta.SetStatusCondition(&status.Conditions, metav1.Condition{ Type: rukpakv1alpha2.TypeUnpacked, From 00c2a7b835813cc18b3439a76238ecaf92a82725 Mon Sep 17 00:00:00 2001 From: Brett Tofel Date: Wed, 22 May 2024 15:12:09 -0400 Subject: [PATCH 5/9] Unskip all - failing tests (up|down)grade should err Unskips all in clusterextension_controller_test.go Signed-off-by: Brett Tofel --- .../clusterextension_controller_test.go | 116 +++++------------- 1 file changed, 30 insertions(+), 86 deletions(-) diff --git a/internal/controllers/clusterextension_controller_test.go b/internal/controllers/clusterextension_controller_test.go index 93b4fe7c3..94e972274 100644 --- a/internal/controllers/clusterextension_controller_test.go +++ b/internal/controllers/clusterextension_controller_test.go @@ -128,65 +128,6 @@ func TestClusterExtensionNonExistentVersion(t *testing.T) { require.NoError(t, cl.DeleteAllOf(ctx, &ocv1alpha1.ClusterExtension{})) } -func TestClusterExtensionBundleDeploymentDoesNotExist(t *testing.T) { - t.Skip("Skip and replace with e2e for BundleDeployment") - cl, reconciler := newClientAndReconciler(t) - ctx := context.Background() - extKey := types.NamespacedName{Name: fmt.Sprintf("cluster-extension-test-%s", rand.String(8))} - const pkgName = "prometheus" - installNamespace := fmt.Sprintf("test-ns-%s", rand.String(8)) - - t.Log("When the cluster extension specifies a valid available package") - t.Log("By initializing cluster state") - clusterExtension := &ocv1alpha1.ClusterExtension{ - ObjectMeta: metav1.ObjectMeta{Name: extKey.Name}, - Spec: ocv1alpha1.ClusterExtensionSpec{ - PackageName: pkgName, - InstallNamespace: installNamespace, - }, - } - require.NoError(t, cl.Create(ctx, clusterExtension)) - - t.Log("When the BundleDeployment does not exist") - t.Log("By running reconcile") - res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey}) - require.Equal(t, ctrl.Result{}, res) - require.NoError(t, err) - - t.Log("By fetching updated cluster extension after reconcile") - require.NoError(t, cl.Get(ctx, extKey, clusterExtension)) - - t.Log("It results in the expected BundleDeployment") - bd := &rukpakv1alpha2.BundleDeployment{} - require.NoError(t, cl.Get(ctx, types.NamespacedName{Name: extKey.Name}, bd)) - require.Equal(t, "core-rukpak-io-registry", bd.Spec.ProvisionerClassName) - require.Equal(t, installNamespace, bd.Spec.InstallNamespace) - require.NotNil(t, bd.Spec.Source.Image) - require.Equal(t, "quay.io/operatorhubio/prometheus@fake2.0.0", bd.Spec.Source.Image.Ref) - - t.Log("It sets the ResolvedBundle status field") - require.Equal(t, &ocv1alpha1.BundleMetadata{Name: "operatorhub/prometheus/beta/2.0.0", Version: "2.0.0"}, clusterExtension.Status.ResolvedBundle) - - t.Log("It sets the InstalledBundle status field") - require.Empty(t, clusterExtension.Status.InstalledBundle) - - t.Log("It sets the status on the cluster extension") - cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeResolved) - require.NotNil(t, cond) - require.Equal(t, metav1.ConditionTrue, cond.Status) - require.Equal(t, ocv1alpha1.ReasonSuccess, cond.Reason) - require.Equal(t, "resolved to \"quay.io/operatorhubio/prometheus@fake2.0.0\"", cond.Message) - - cond = apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeInstalled) - require.NotNil(t, cond) - require.Equal(t, metav1.ConditionUnknown, cond.Status) - require.Equal(t, ocv1alpha1.ReasonInstallationStatusUnknown, cond.Reason) - require.Equal(t, "bundledeployment status is unknown", cond.Message) - - verifyInvariants(ctx, t, reconciler.Client, clusterExtension) - require.NoError(t, cl.DeleteAllOf(ctx, &ocv1alpha1.ClusterExtension{})) -} - func TestClusterExtensionChannelVersionExists(t *testing.T) { cl, reconciler := newClientAndReconciler(t) mockUnpacker := unpacker.(*MockUnpacker) @@ -221,7 +162,7 @@ func TestClusterExtensionChannelVersionExists(t *testing.T) { t.Log("By running reconcile") res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey}) require.Equal(t, ctrl.Result{}, res) - require.NoError(t, err, nil) + require.NoError(t, err) t.Log("By fetching updated cluster extension after reconcile") require.NoError(t, cl.Get(ctx, extKey, clusterExtension)) @@ -247,8 +188,13 @@ func TestClusterExtensionChannelVersionExists(t *testing.T) { } func TestClusterExtensionChannelExistsNoVersion(t *testing.T) { - t.Skip("Skip and replace with e2e for BundleDeployment") cl, reconciler := newClientAndReconciler(t) + mockUnpacker := unpacker.(*MockUnpacker) + // Set up the Unpack method to return a result with StateUnpacked + mockUnpacker.On("Unpack", mock.Anything, mock.AnythingOfType("*v1alpha2.BundleDeployment")).Return(&source.Result{ + State: source.StatePending, + }, nil) + ctx := context.Background() extKey := types.NamespacedName{Name: fmt.Sprintf("cluster-extension-test-%s", rand.String(8))} @@ -267,13 +213,15 @@ func TestClusterExtensionChannelExistsNoVersion(t *testing.T) { InstallNamespace: installNamespace, }, } - require.NoError(t, cl.Create(ctx, clusterExtension)) + err := cl.Create(ctx, clusterExtension) + require.NoError(t, err) t.Log("It sets resolution success status") t.Log("By running reconcile") res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey}) require.Equal(t, ctrl.Result{}, res) require.NoError(t, err) + t.Log("By fetching updated cluster extension after reconcile") require.NoError(t, cl.Get(ctx, extKey, clusterExtension)) @@ -282,27 +230,18 @@ func TestClusterExtensionChannelExistsNoVersion(t *testing.T) { require.Empty(t, clusterExtension.Status.InstalledBundle) t.Log("By checking the expected conditions") - cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeResolved) - require.NotNil(t, cond) - require.Equal(t, metav1.ConditionTrue, cond.Status) - require.Equal(t, ocv1alpha1.ReasonSuccess, cond.Reason) - require.Equal(t, "resolved to \"quay.io/operatorhubio/prometheus@fake2.0.0\"", cond.Message) - cond = apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeInstalled) - require.NotNil(t, cond) - require.Equal(t, metav1.ConditionUnknown, cond.Status) - require.Equal(t, ocv1alpha1.ReasonInstallationStatusUnknown, cond.Reason) - require.Equal(t, "bundledeployment status is unknown", cond.Message) + resolvedCond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeResolved) + require.NotNil(t, resolvedCond) + require.Equal(t, metav1.ConditionTrue, resolvedCond.Status) + require.Equal(t, ocv1alpha1.ReasonSuccess, resolvedCond.Reason) + require.Equal(t, "resolved to \"quay.io/operatorhubio/prometheus@fake2.0.0\"", resolvedCond.Message) - t.Log("By fetching the bundledeployment") - bd := &rukpakv1alpha2.BundleDeployment{} - require.NoError(t, cl.Get(ctx, types.NamespacedName{Name: extKey.Name}, bd)) - require.Equal(t, "core-rukpak-io-registry", bd.Spec.ProvisionerClassName) - require.Equal(t, installNamespace, bd.Spec.InstallNamespace) - require.Equal(t, rukpakv1alpha2.SourceTypeImage, bd.Spec.Source.Type) - require.NotNil(t, bd.Spec.Source.Image) - require.Equal(t, "quay.io/operatorhubio/prometheus@fake2.0.0", bd.Spec.Source.Image.Ref) + t.Log("By checking the expected unpacked conditions") + unpackedCond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, rukpakv1alpha2.TypeUnpacked) + require.NotNil(t, unpackedCond) + require.Equal(t, metav1.ConditionFalse, unpackedCond.Status) + require.Equal(t, rukpakv1alpha2.ReasonUnpackPending, unpackedCond.Reason) - verifyInvariants(ctx, t, reconciler.Client, clusterExtension) require.NoError(t, cl.DeleteAllOf(ctx, &ocv1alpha1.ClusterExtension{})) } @@ -476,10 +415,14 @@ func verifyConditionsInvariants(t *testing.T, ext *ocv1alpha1.ClusterExtension) func TestClusterExtensionUpgrade(t *testing.T) { cl, reconciler := newClientAndReconciler(t) + mockUnpacker := unpacker.(*MockUnpacker) + // Set up the Unpack method to return a result with StateUnpacked + mockUnpacker.On("Unpack", mock.Anything, mock.AnythingOfType("*v1alpha2.BundleDeployment")).Return(&source.Result{ + State: source.StatePending, + }, nil) ctx := context.Background() t.Run("semver upgrade constraints enforcement of upgrades within major version", func(t *testing.T) { - t.Skip("Skip and replace with e2e for BundleDeployment") defer featuregatetesting.SetFeatureGateDuringTest(t, features.OperatorControllerFeatureGate, features.ForceSemverUpgradeConstraints, true)() defer func() { require.NoError(t, cl.DeleteAllOf(ctx, &ocv1alpha1.ClusterExtension{})) @@ -573,7 +516,6 @@ func TestClusterExtensionUpgrade(t *testing.T) { }) t.Run("legacy semantics upgrade constraints enforcement", func(t *testing.T) { - t.Skip("Skip and replace with e2e for BundleDeployment") defer featuregatetesting.SetFeatureGateDuringTest(t, features.OperatorControllerFeatureGate, features.ForceSemverUpgradeConstraints, false)() defer func() { require.NoError(t, cl.DeleteAllOf(ctx, &ocv1alpha1.ClusterExtension{})) @@ -667,7 +609,6 @@ func TestClusterExtensionUpgrade(t *testing.T) { }) t.Run("ignore upgrade constraints", func(t *testing.T) { - t.Skip("Skip and replace with e2e for BundleDeployment") for _, tt := range []struct { name string flagState bool @@ -754,10 +695,14 @@ func TestClusterExtensionUpgrade(t *testing.T) { func TestClusterExtensionDowngrade(t *testing.T) { cl, reconciler := newClientAndReconciler(t) + mockUnpacker := unpacker.(*MockUnpacker) + // Set up the Unpack method to return a result with StateUnpacked + mockUnpacker.On("Unpack", mock.Anything, mock.AnythingOfType("*v1alpha2.BundleDeployment")).Return(&source.Result{ + State: source.StatePending, + }, nil) ctx := context.Background() t.Run("enforce upgrade constraints", func(t *testing.T) { - t.Skip("Skip and replace with e2e for BundleDeployment") for _, tt := range []struct { name string flagState bool @@ -840,7 +785,6 @@ func TestClusterExtensionDowngrade(t *testing.T) { }) t.Run("ignore upgrade constraints", func(t *testing.T) { - t.Skip("Skip and replace with e2e for BundleDeployment") for _, tt := range []struct { name string flagState bool From 39e42df0d915c43d265f3f33feced3fb31485375 Mon Sep 17 00:00:00 2001 From: Brett Tofel Date: Thu, 23 May 2024 10:42:19 -0400 Subject: [PATCH 6/9] Unskip all clusterextension_registryv1_validation_test.go Still failing tests (up|down)grade should err b/c we don't have an installedBundle to check against Signed-off-by: Brett Tofel --- ...terextension_registryv1_validation_test.go | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/internal/controllers/clusterextension_registryv1_validation_test.go b/internal/controllers/clusterextension_registryv1_validation_test.go index b6f1f72f3..a289d65ba 100644 --- a/internal/controllers/clusterextension_registryv1_validation_test.go +++ b/internal/controllers/clusterextension_registryv1_validation_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -16,6 +17,7 @@ import ( "github.com/operator-framework/operator-registry/alpha/declcfg" "github.com/operator-framework/operator-registry/alpha/property" + "github.com/operator-framework/rukpak/pkg/source" ocv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1" "github.com/operator-framework/operator-controller/internal/catalogmetadata" @@ -31,7 +33,6 @@ func TestClusterExtensionRegistryV1DisallowDependencies(t *testing.T) { name string bundle *catalogmetadata.Bundle wantErr string - skip bool }{ { name: "package with no dependencies", @@ -46,9 +47,6 @@ func TestClusterExtensionRegistryV1DisallowDependencies(t *testing.T) { }, CatalogName: "fake-catalog", }, - // Skipping the happy path, since it requires us to mock unpacker, store and the - // entire installation. This should be handled in an e2e instead. - skip: true, }, { name: "package with olm.package.required property", @@ -103,16 +101,20 @@ func TestClusterExtensionRegistryV1DisallowDependencies(t *testing.T) { defer func() { require.NoError(t, cl.DeleteAllOf(ctx, &ocv1alpha1.ClusterExtension{})) }() - - if tt.skip { - return - } - fakeCatalogClient := testutil.NewFakeCatalogClient([]*catalogmetadata.Bundle{tt.bundle}) + mockUnpacker := unpacker.(*MockUnpacker) + // Verify if mockUnpacker is correctly set + t.Log("MockUnpacker set to:", mockUnpacker) + // Set up the Unpack method to return a result with StatePending + mockUnpacker.On("Unpack", mock.Anything, mock.AnythingOfType("*v1alpha2.BundleDeployment")).Return(&source.Result{ + State: source.StatePending, + }, nil) + reconciler := &controllers.ClusterExtensionReconciler{ Client: cl, BundleProvider: &fakeCatalogClient, ActionClientGetter: helmClientGetter, + Unpacker: unpacker, } installNamespace := fmt.Sprintf("test-ns-%s", rand.String(8)) From 6caaf33a2a8af6a47b85f0842547808111f95782 Mon Sep 17 00:00:00 2001 From: Varsha Prasad Narsing Date: Thu, 23 May 2024 14:26:56 -0700 Subject: [PATCH 7/9] debugging Signed-off-by: Varsha Prasad Narsing --- internal/.DS_Store | Bin 0 -> 6148 bytes .../clusterextension_controller.go | 75 ++++++------- .../clusterextension_controller_test.go | 99 +++++++++++++++++- .../controllers/clusterextension_status.go | 69 ------------ internal/controllers/common_controller.go | 45 ++++++++ 5 files changed, 179 insertions(+), 109 deletions(-) create mode 100644 internal/.DS_Store delete mode 100644 internal/controllers/clusterextension_status.go diff --git a/internal/.DS_Store b/internal/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Tue, 28 May 2024 14:07:45 -0400 Subject: [PATCH 8/9] Fixes and additions pending & unpack path Signed-off-by: Brett Tofel --- api/v1alpha1/clusterextension_types.go | 12 +++++-- .../clusterextension_controller.go | 9 ++++- .../clusterextension_controller_test.go | 1 + internal/controllers/common_controller.go | 33 ++++++++++++++++--- 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/api/v1alpha1/clusterextension_types.go b/api/v1alpha1/clusterextension_types.go index 97578bbf8..c69803614 100644 --- a/api/v1alpha1/clusterextension_types.go +++ b/api/v1alpha1/clusterextension_types.go @@ -92,6 +92,7 @@ const ( TypePackageDeprecated = "PackageDeprecated" TypeChannelDeprecated = "ChannelDeprecated" TypeBundleDeprecated = "BundleDeprecated" + TypeUnpacked = "Unpacked" ReasonErrorGettingClient = "ErrorGettingClient" ReasonBundleLoadFailed = "BundleLoadFailed" @@ -101,9 +102,11 @@ const ( ReasonInstallationSucceeded = "InstallationSucceeded" ReasonResolutionFailed = "ResolutionFailed" - ReasonSuccess = "Success" - ReasonDeprecated = "Deprecated" - ReasonUpgradeFailed = "UpgradeFailed" + ReasonSuccess = "Success" + ReasonDeprecated = "Deprecated" + ReasonUpgradeFailed = "UpgradeFailed" + ReasonHasValidBundleUnknown = "HasValidBundleUnknown" + ReasonUnpackPending = "UnpackPending" ) func init() { @@ -116,6 +119,7 @@ func init() { TypePackageDeprecated, TypeChannelDeprecated, TypeBundleDeprecated, + TypeUnpacked, ) // TODO(user): add Reasons from above conditionsets.ConditionReasons = append(conditionsets.ConditionReasons, @@ -128,6 +132,8 @@ func init() { ReasonBundleLoadFailed, ReasonErrorGettingClient, ReasonInstallationStatusUnknown, + ReasonHasValidBundleUnknown, + ReasonUnpackPending, ) } diff --git a/internal/controllers/clusterextension_controller.go b/internal/controllers/clusterextension_controller.go index 9a81a5d39..e49c25ef7 100644 --- a/internal/controllers/clusterextension_controller.go +++ b/internal/controllers/clusterextension_controller.go @@ -235,6 +235,8 @@ func (r *ClusterExtensionReconciler) reconcile(ctx context.Context, ext *ocv1alp setDeprecationStatusesUnknown(&ext.Status.Conditions, "deprecation checks have not been attempted as installation has failed", ext.GetGeneration()) return ctrl.Result{}, err } + // set deprecation status after _successful_ resolution + SetDeprecationStatus(ext, bundle) bundleVersion, err := bundle.Version() if err != nil { @@ -259,12 +261,17 @@ func (r *ClusterExtensionReconciler) reconcile(ctx context.Context, ext *ocv1alp switch unpackResult.State { case rukpaksource.StatePending: - updateStatusUnpackPending(&ext.Status, unpackResult) + updateStatusUnpackPending(&ext.Status, unpackResult, ext.GetGeneration()) // There must be a limit to number of entries if status is stuck at // unpack pending. + setHasValidBundleUnknown(&ext.Status.Conditions, "unpack pending", ext.GetGeneration()) + setInstalledStatusConditionUnknown(&ext.Status.Conditions, "installation has not been attempted as unpack is pending", ext.GetGeneration()) + return ctrl.Result{}, nil case rukpaksource.StateUnpacking: updateStatusUnpacking(&ext.Status, unpackResult) + setHasValidBundleUnknown(&ext.Status.Conditions, "unpack pending", ext.GetGeneration()) + setInstalledStatusConditionUnknown(&ext.Status.Conditions, "installation has not been attempted as unpack is pending", ext.GetGeneration()) return ctrl.Result{}, nil case rukpaksource.StateUnpacked: // TODO: Add finalizer to clean the stored bundles, after https://github.com/operator-framework/rukpak/pull/897 diff --git a/internal/controllers/clusterextension_controller_test.go b/internal/controllers/clusterextension_controller_test.go index 509fcdc22..5dcd2ea6a 100644 --- a/internal/controllers/clusterextension_controller_test.go +++ b/internal/controllers/clusterextension_controller_test.go @@ -243,6 +243,7 @@ func TestClusterExtensionChannelExistsNoVersion(t *testing.T) { require.Equal(t, metav1.ConditionFalse, unpackedCond.Status) require.Equal(t, rukpakv1alpha2.ReasonUnpackPending, unpackedCond.Reason) + verifyInvariants(ctx, t, reconciler.Client, clusterExtension) require.NoError(t, cl.DeleteAllOf(ctx, &ocv1alpha1.ClusterExtension{})) } diff --git a/internal/controllers/common_controller.go b/internal/controllers/common_controller.go index 2e4c19aeb..29a465207 100644 --- a/internal/controllers/common_controller.go +++ b/internal/controllers/common_controller.go @@ -46,6 +46,28 @@ func setResolvedStatusConditionSuccess(conditions *[]metav1.Condition, message s }) } +// setInstalledStatusConditionUnknown sets the installed status condition to unknown. +func setInstalledStatusConditionUnknown(conditions *[]metav1.Condition, message string, generation int64) { + apimeta.SetStatusCondition(conditions, metav1.Condition{ + Type: ocv1alpha1.TypeInstalled, + Status: metav1.ConditionUnknown, + Reason: ocv1alpha1.ReasonInstallationStatusUnknown, + Message: message, + ObservedGeneration: generation, + }) +} + +// setHasValidBundleUnknown sets the installed status condition to unknown. +func setHasValidBundleUnknown(conditions *[]metav1.Condition, message string, generation int64) { + apimeta.SetStatusCondition(conditions, metav1.Condition{ + Type: ocv1alpha1.TypeHasValidBundle, + Status: metav1.ConditionUnknown, + Reason: ocv1alpha1.ReasonHasValidBundleUnknown, + Message: message, + ObservedGeneration: generation, + }) +} + // setResolvedStatusConditionFailed sets the resolved status condition to failed. func setResolvedStatusConditionFailed(conditions *[]metav1.Condition, message string, generation int64) { apimeta.SetStatusCondition(conditions, metav1.Condition{ @@ -111,13 +133,14 @@ func updateStatusUnpackFailing(status *ocv1alpha1.ClusterExtensionStatus, err er } // TODO: verify if we need to update the installBundle status or leave it as is. -func updateStatusUnpackPending(status *ocv1alpha1.ClusterExtensionStatus, result *source.Result) { +func updateStatusUnpackPending(status *ocv1alpha1.ClusterExtensionStatus, result *source.Result, generation int64) { status.InstalledBundle = nil meta.SetStatusCondition(&status.Conditions, metav1.Condition{ - Type: rukpakv1alpha2.TypeUnpacked, - Status: metav1.ConditionFalse, - Reason: rukpakv1alpha2.ReasonUnpackPending, - Message: result.Message, + Type: rukpakv1alpha2.TypeUnpacked, + Status: metav1.ConditionFalse, + Reason: rukpakv1alpha2.ReasonUnpackPending, + Message: result.Message, + ObservedGeneration: generation, }) } From 507c86e1dad60da746e5ad66d43556aaf1a5f829 Mon Sep 17 00:00:00 2001 From: dtfranz Date: Tue, 28 May 2024 13:03:07 -0700 Subject: [PATCH 9/9] Fix linter, remove debug logs and extraneous file Signed-off-by: dtfranz --- internal/.DS_Store | Bin 6148 -> 0 bytes .../controllers/clusterextension_controller.go | 3 --- internal/controllers/common_controller.go | 14 +++++++------- 3 files changed, 7 insertions(+), 10 deletions(-) delete mode 100644 internal/.DS_Store diff --git a/internal/.DS_Store b/internal/.DS_Store deleted file mode 100644 index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0