helm: use chunking release driver in systemNamespace

Signed-off-by: Joe Lanford <[email protected]>

Author: joelanford

cmd/manager/main.go

@@ -29,7 +29,6 @@ import (
 	apiextensionsv1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
 	k8slabels "k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/selection"
-	"k8s.io/apimachinery/pkg/types"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"k8s.io/client-go/rest"
@@ -159,38 +158,23 @@ func main() {
 		os.Exit(1)
 	}
 
-	installNamespaceMapper := helmclient.ObjectToStringMapper(func(obj client.Object) (string, error) {
-		ext := obj.(*ocv1alpha1.ClusterExtension)
-		return ext.Spec.InstallNamespace, nil
-	})
 	coreClient, err := corev1client.NewForConfig(mgr.GetConfig())
 	if err != nil {
 		setupLog.Error(err, "unable to create core client")
 		os.Exit(1)
 	}
 	tokenGetter := authentication.NewTokenGetter(coreClient, authentication.WithExpirationDuration(1*time.Hour))
 
-	restConfigMapper := func(ctx context.Context, o client.Object, c *rest.Config) (*rest.Config, error) {
-		cExt, ok := o.(*ocv1alpha1.ClusterExtension)
-		if !ok {
-			return c, nil
-		}
-		namespacedName := types.NamespacedName{
-			Name:      cExt.Spec.ServiceAccount.Name,
-			Namespace: cExt.Spec.InstallNamespace,
-		}
-		token, err := tokenGetter.Get(ctx, namespacedName)
-		if err != nil {
-			return nil, fmt.Errorf("failed to extract SA token, %w", err)
-		}
-		tempConfig := rest.AnonymousClientConfig(c)
-		tempConfig.BearerToken = token
-		return tempConfig, nil
-	}
 	cfgGetter, err := helmclient.NewActionConfigGetter(mgr.GetConfig(), mgr.GetRESTMapper(),
-		helmclient.StorageNamespaceMapper(installNamespaceMapper),
-		helmclient.ClientNamespaceMapper(installNamespaceMapper),
-		helmclient.RestConfigMapper(restConfigMapper),
+		helmclient.StorageRestConfigMapper(func(ctx context.Context, obj client.Object, config *rest.Config) (*rest.Config, error) {
+			return rest.CopyConfig(config), nil
+		}),
+		helmclient.StorageDriverMapper(action.ChunkedStorageDriverMapper(coreClient, mgr.GetAPIReader(), systemNamespace)),
+		helmclient.ClientNamespaceMapper(func(obj client.Object) (string, error) {
+			ext := obj.(*ocv1alpha1.ClusterExtension)
+			return ext.Spec.InstallNamespace, nil
+		}),
+		helmclient.ClientRestConfigMapper(action.ServiceAccountRestConfigMapper(tokenGetter)),
 	)
 	if err != nil {
 		setupLog.Error(err, "unable to config for creating helm client")

config/base/rbac/role.yaml

@@ -30,18 +30,6 @@ rules:
   verbs:
   - list
   - watch
-- apiGroups:
-  - ""
-  resources:
-  - secrets
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
 - apiGroups:
   - ""
   resources:
@@ -71,3 +59,23 @@ rules:
   verbs:
   - patch
   - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: manager-role
+  namespace: system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch

config/base/rbac/role_binding.yaml

@@ -10,3 +10,17 @@ subjects:
 - kind: ServiceAccount
   name: controller-manager
   namespace: system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: manager-rolebinding
+  namespace: system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: manager-role
+subjects:
+  - kind: ServiceAccount
+    name: controller-manager
+    namespace: system

config/samples/olm_v1alpha1_clusterextension.yaml

@@ -1,10 +1,104 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: argocd
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: argocd-installer
+  namespace: argocd
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: argocd-installer-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: argocd-installer-clusterrole
+subjects:
+- kind: ServiceAccount
+  name: argocd-installer
+  namespace: argocd
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: argocd-installer-clusterrole
+rules:
+- apiGroups: ["*"]
+  resources: ["*"]
+  verbs: ["*"]
+- apiGroups: [apiextensions.k8s.io]
+  resources: [customresourcedefinitions]
+  verbs: [get, list, watch, create, update, patch, delete]
+  resourceNames:
+  - appprojects.argoproj.io
+  - argocds.argoproj.io
+  - applications.argoproj.io
+  - argocdexports.argoproj.io
+  - applicationsets.argoproj.io
+- apiGroups: [rbac.authorization.k8s.io]
+  resources: [clusterroles]
+  verbs: [get, list, watch, create, update, patch, delete]
+  resourceNames:
+  - argocd-operator.v0-1dhiybrldl1gyksid1dk2dqjsc72psdybc7iyvse5gpx
+  - argocd-operator-metrics-reader
+  - argocd-operator.v0-22gmilmgp91wu25is5i2ec598hni8owq3l71bbkl7iz3
+- apiGroups: [rbac.authorization.k8s.io]
+  resources: [clusterrolebindings]
+  verbs: [get, list, watch, create, update, patch, delete]
+  resourceNames:
+  - argocd-operator.v0-1dhiybrldl1gyksid1dk2dqjsc72psdybc7iyvse5gpx
+  - argocd-operator.v0-22gmilmgp91wu25is5i2ec598hni8owq3l71bbkl7iz3
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: argocd-installer-role
+  namespace: argocd
+rules:
+- apiGroups: [""]
+  resources: [serviceaccounts]
+  verbs: [get, list, watch, create, update, patch, delete]
+  resourceNames: [argocd-operator-controller-manager]
+- apiGroups: [""]
+  resources: [configmaps]
+  verbs: [get, list, watch, create, update, patch, delete]
+  resourceNames: [argocd-operator-manager-config]
+- apiGroups: [""]
+  resources: [services]
+  verbs: [get, list, watch, create, update, patch, delete]
+  resourceNames: [argocd-operator-controller-manager-metrics-service]
+- apiGroups: [apps]
+  resources: [deployments]
+  verbs: [get, list, watch, create, update, patch, delete]
+  resourceNames: [argocd-operator-controller-manager]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: argocd-installer-binding
+  namespace: argocd
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: argocd-installer-role
+subjects:
+- kind: ServiceAccount
+  name: argocd-installer
+  namespace: argocd
+---
 apiVersion: olm.operatorframework.io/v1alpha1
 kind: ClusterExtension
 metadata:
-  name: clusterextension-sample
+  name: argocd
 spec:
-  installNamespace: default
+  installNamespace: argocd
   packageName: argocd-operator
   version: 0.6.0
   serviceAccount:
-    name: default
+    name: argocd-installer
+---

go.mod

@@ -35,6 +35,8 @@ require (
 	sigs.k8s.io/yaml v1.4.0
 )
 
+replace github.com/operator-framework/helm-operator-plugins => github.com/joelanford/helm-operator v0.0.8-0.20240719203048-14906f8dfcf8
+
 require (
 	carvel.dev/vendir v0.40.0 // indirect
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect

go.sum

@@ -443,6 +443,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
 github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
+github.com/joelanford/helm-operator v0.0.8-0.20240719203048-14906f8dfcf8 h1:+hG2OiAiMz4/zUYLi6+FzjrCRNVJedafECfjaYgTkwc=
+github.com/joelanford/helm-operator v0.0.8-0.20240719203048-14906f8dfcf8/go.mod h1:5Kx1PyLnRVPyQmLq+frv+HJgSZzXG+W6LavSCxzm8sI=
 github.com/joelanford/ignore v0.1.0 h1:VawbTDeg5EL+PN7W8gxVzGerfGpVo3gFdR5ZAqnkYRk=
 github.com/joelanford/ignore v0.1.0/go.mod h1:Vb0PQMAQXK29fmiPjDukpO8I2NTcp1y8LbhFijD1/0o=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -590,8 +592,6 @@ github.com/operator-framework/api v0.26.0 h1:YVntU2NkVl5zSLLwK5kFcH6P3oSvN9QDgTs
 github.com/operator-framework/api v0.26.0/go.mod h1:3IxOwzVUeGxYlzfwKCcfCyS+q3EEhWA/4kv7UehbeyM=
 github.com/operator-framework/catalogd v0.18.0 h1:3eFoURYkn+vspoqYL5ijzUjAyVMSSR60BMOiKFvdVmM=
 github.com/operator-framework/catalogd v0.18.0/go.mod h1:3i2dDt0yg6L/xHbNt93/Teao6xBC8rhL6UgewjQpvI8=
-github.com/operator-framework/helm-operator-plugins v0.3.0 h1:LNhcb5nPT/TAxZSsKH2LTYh79RgiN2twGFptQR96sRM=
-github.com/operator-framework/helm-operator-plugins v0.3.0/go.mod h1:ly6Bd9rSzmt37Wy6WtZHmA+IY9zG958MryJFLcVpCXw=
 github.com/operator-framework/operator-lib v0.14.0 h1:er+BgZymZD1im2wytLJiPLZpGALAX6N0gXaHx3PKbO4=
 github.com/operator-framework/operator-lib v0.14.0/go.mod h1:wUu4Xb9xzXnIpglvaZ3yucTMSlqGXHIoUEH9+5gWiu0=
 github.com/operator-framework/operator-registry v1.45.0 h1:9c5NshWjPncdZtWEY0cfMnAjx3pShVnjw5UmZXp/xNE=

internal/action/restconfig.go

@@ -0,0 +1,32 @@
+package action
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	helmclient "github.com/operator-framework/helm-operator-plugins/pkg/client"
+
+	ocv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1"
+	"github.com/operator-framework/operator-controller/internal/authentication"
+)
+
+func ServiceAccountRestConfigMapper(tokenGetter *authentication.TokenGetter) helmclient.ObjectToRestConfigMapper {
+	return func(ctx context.Context, o client.Object, c *rest.Config) (*rest.Config, error) {
+		cExt := o.(*ocv1alpha1.ClusterExtension)
+		saKey := types.NamespacedName{
+			Name:      cExt.Spec.ServiceAccount.Name,
+			Namespace: cExt.Spec.InstallNamespace,
+		}
+		token, err := tokenGetter.Get(ctx, saKey)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get token for serviceaccount %q: %w", saKey, err)
+		}
+		saConfig := rest.AnonymousClientConfig(c)
+		saConfig.BearerToken = token
+		return saConfig, nil
+	}
+}

internal/action/storagedriver.go

@@ -0,0 +1,110 @@
+package action
+
+import (
+	"context"
+	"fmt"
+
+	"helm.sh/helm/v3/pkg/storage/driver"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	k8slabels "k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/watch"
+	clientcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	helmclient "github.com/operator-framework/helm-operator-plugins/pkg/client"
+	"github.com/operator-framework/helm-operator-plugins/pkg/storage"
+)
+
+func ChunkedStorageDriverMapper(secretsGetter clientcorev1.SecretsGetter, reader client.Reader, namespace string) helmclient.ObjectToStorageDriverMapper {
+	secretsClient := newSecretsDelegatingClient(secretsGetter, reader, namespace)
+	return func(ctx context.Context, object client.Object, config *rest.Config) (driver.Driver, error) {
+		log := logf.FromContext(ctx).V(2)
+		ownerRefs := []metav1.OwnerReference{*metav1.NewControllerRef(object, object.GetObjectKind().GroupVersionKind())}
+		ownerRefSecretClient := helmclient.NewOwnerRefSecretClient(secretsClient, ownerRefs, func(secret *corev1.Secret) bool {
+			return secret.Type == storage.SecretTypeChunkedIndex
+		})
+		return storage.NewChunkedSecrets(ownerRefSecretClient, "operator-controller", storage.ChunkedSecretsConfig{
+			ChunkSize:      1024 * 1024,
+			MaxReadChunks:  10,
+			MaxWriteChunks: 10,
+			Log:            func(format string, args ...interface{}) { log.Info(fmt.Sprintf(format, args...)) },
+		}), nil
+	}
+}
+
+var _ clientcorev1.SecretInterface = &secretsDelegatingClient{}
+
+type secretsDelegatingClient struct {
+	clientcorev1.SecretInterface
+	reader    client.Reader
+	namespace string
+}
+
+func newSecretsDelegatingClient(secretsGetter clientcorev1.SecretsGetter, reader client.Reader, namespace string) clientcorev1.SecretInterface {
+	return &secretsDelegatingClient{
+		SecretInterface: secretsGetter.Secrets(namespace),
+		namespace:       namespace,
+		reader:          reader,
+	}
+}
+
+func (s secretsDelegatingClient) Get(ctx context.Context, name string, opts metav1.GetOptions) (*corev1.Secret, error) {
+	var secret corev1.Secret
+	if err := s.reader.Get(ctx, client.ObjectKey{Namespace: s.namespace, Name: name}, &secret, &client.GetOptions{Raw: &opts}); err != nil {
+		return nil, err
+	}
+	return &secret, nil
+}
+
+func (s secretsDelegatingClient) List(ctx context.Context, opts metav1.ListOptions) (*corev1.SecretList, error) {
+	listOpts, err := metaOptionsToClientOptions(s.namespace, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var secrets corev1.SecretList
+	if err := s.reader.List(ctx, &secrets, listOpts); err != nil {
+		return nil, fmt.Errorf("error listing from cache: %w", err)
+	}
+	return &secrets, nil
+}
+
+func (s secretsDelegatingClient) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
+	panic("intentionally not implemented: watch is not intended to be called")
+}
+
+func metaOptionsToClientOptions(namespace string, opts metav1.ListOptions) (*client.ListOptions, error) {
+	clientListOptions := &client.ListOptions{
+		Namespace: namespace,
+		Limit:     opts.Limit,
+		Continue:  opts.Continue,
+	}
+
+	if opts.LabelSelector != "" {
+		labelSelector, err := k8slabels.Parse(opts.LabelSelector)
+		if err != nil {
+			return nil, err
+		}
+		clientListOptions.LabelSelector = labelSelector
+	}
+
+	if opts.FieldSelector != "" {
+		fieldSelector, err := fields.ParseSelector(opts.FieldSelector)
+		if err != nil {
+			return nil, err
+		}
+		clientListOptions.FieldSelector = fieldSelector
+	}
+
+	opts.LabelSelector = ""
+	opts.FieldSelector = ""
+	opts.Limit = 0
+	opts.Continue = ""
+	clientListOptions.Raw = &opts
+
+	return clientListOptions, nil
+}