Skip to content

Commit 011246e

Browse files
committed
helm: use chunking release driver in systemNamespace
Signed-off-by: Joe Lanford <[email protected]>
1 parent 0796ecf commit 011246e

File tree

4 files changed

+132
-8
lines changed

4 files changed

+132
-8
lines changed

cmd/manager/main.go

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -154,13 +154,22 @@ func main() {
154154
os.Exit(1)
155155
}
156156

157-
installNamespaceMapper := helmclient.ObjectToStringMapper(func(obj client.Object) (string, error) {
158-
ext := obj.(*ocv1alpha1.ClusterExtension)
159-
return ext.Spec.InstallNamespace, nil
160-
})
157+
// Setup a custom storage driver to use. We do this for two reasons:
158+
// 1. We need to chunk secrets to avoid hitting etcd limits on the size of a single object.
159+
// 2. We need to avoid using the same storage driver as the Helm CLI to avoid conflicts and
160+
// accidental inheritance of behavior (e.g. adoption of a Helm CLI-created release by
161+
// operator-controller)
162+
storageDriverMapper, err := action.ChunkedStorageDriverMapper(mgr, systemNamespace)
163+
if err != nil {
164+
setupLog.Error(err, "unable to create storage driver mapper")
165+
os.Exit(1)
166+
}
161167
cfgGetter, err := helmclient.NewActionConfigGetter(mgr.GetConfig(), mgr.GetRESTMapper(),
162-
helmclient.StorageNamespaceMapper(installNamespaceMapper),
163-
helmclient.ClientNamespaceMapper(installNamespaceMapper),
168+
helmclient.StorageDriverMapper(storageDriverMapper),
169+
helmclient.ClientNamespaceMapper(func(obj client.Object) (string, error) {
170+
ext := obj.(*ocv1alpha1.ClusterExtension)
171+
return ext.Spec.InstallNamespace, nil
172+
}),
164173
)
165174
if err != nil {
166175
setupLog.Error(err, "unable to config for creating helm client")

go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ require (
3737
sigs.k8s.io/yaml v1.4.0
3838
)
3939

40+
replace github.com/operator-framework/helm-operator-plugins => github.com/joelanford/helm-operator v0.0.8-0.20240718195442-f187d057d430
41+
4042
require (
4143
cloud.google.com/go/compute/metadata v0.3.0 // indirect
4244
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -441,6 +441,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
441441
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
442442
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
443443
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
444+
github.com/joelanford/helm-operator v0.0.8-0.20240718195442-f187d057d430 h1:xFJglVJFjVvHF63LHQZpKr81L5Lu74JGFMiXsngTMo0=
445+
github.com/joelanford/helm-operator v0.0.8-0.20240718195442-f187d057d430/go.mod h1:5Kx1PyLnRVPyQmLq+frv+HJgSZzXG+W6LavSCxzm8sI=
444446
github.com/joelanford/ignore v0.1.0 h1:VawbTDeg5EL+PN7W8gxVzGerfGpVo3gFdR5ZAqnkYRk=
445447
github.com/joelanford/ignore v0.1.0/go.mod h1:Vb0PQMAQXK29fmiPjDukpO8I2NTcp1y8LbhFijD1/0o=
446448
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -588,8 +590,6 @@ github.com/operator-framework/api v0.26.0 h1:YVntU2NkVl5zSLLwK5kFcH6P3oSvN9QDgTs
588590
github.com/operator-framework/api v0.26.0/go.mod h1:3IxOwzVUeGxYlzfwKCcfCyS+q3EEhWA/4kv7UehbeyM=
589591
github.com/operator-framework/catalogd v0.18.0 h1:3eFoURYkn+vspoqYL5ijzUjAyVMSSR60BMOiKFvdVmM=
590592
github.com/operator-framework/catalogd v0.18.0/go.mod h1:3i2dDt0yg6L/xHbNt93/Teao6xBC8rhL6UgewjQpvI8=
591-
github.com/operator-framework/helm-operator-plugins v0.3.0 h1:LNhcb5nPT/TAxZSsKH2LTYh79RgiN2twGFptQR96sRM=
592-
github.com/operator-framework/helm-operator-plugins v0.3.0/go.mod h1:ly6Bd9rSzmt37Wy6WtZHmA+IY9zG958MryJFLcVpCXw=
593593
github.com/operator-framework/operator-lib v0.14.0 h1:er+BgZymZD1im2wytLJiPLZpGALAX6N0gXaHx3PKbO4=
594594
github.com/operator-framework/operator-lib v0.14.0/go.mod h1:wUu4Xb9xzXnIpglvaZ3yucTMSlqGXHIoUEH9+5gWiu0=
595595
github.com/operator-framework/operator-registry v1.44.0 h1:NW5/xHYR77J2EUYm+6iBER1WNGLNS8gM+G5GBQWqTTs=

internal/action/storagedriver.go

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package action
2+
3+
import (
4+
"context"
5+
"fmt"
6+
7+
"helm.sh/helm/v3/pkg/storage/driver"
8+
corev1 "k8s.io/api/core/v1"
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
"k8s.io/apimachinery/pkg/fields"
11+
k8slabels "k8s.io/apimachinery/pkg/labels"
12+
"k8s.io/apimachinery/pkg/watch"
13+
clientcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
14+
"k8s.io/client-go/rest"
15+
crcache "sigs.k8s.io/controller-runtime/pkg/cache"
16+
"sigs.k8s.io/controller-runtime/pkg/client"
17+
logf "sigs.k8s.io/controller-runtime/pkg/log"
18+
"sigs.k8s.io/controller-runtime/pkg/manager"
19+
20+
helmclient "github.com/operator-framework/helm-operator-plugins/pkg/client"
21+
"github.com/operator-framework/helm-operator-plugins/pkg/storage"
22+
)
23+
24+
func ChunkedStorageDriverMapper(mgr manager.Manager, releaseStorageNamespace string) (helmclient.ObjectToStorageDriverMapper, error) {
25+
corev1Client, err := clientcorev1.NewForConfig(mgr.GetConfig())
26+
if err != nil {
27+
return nil, fmt.Errorf("unable to create corev1 client: %w", err)
28+
}
29+
secretsClient := newSecretsDelegatingClient(corev1Client.Secrets(releaseStorageNamespace), releaseStorageNamespace, mgr.GetCache())
30+
31+
return func(ctx context.Context, object client.Object, config *rest.Config) (driver.Driver, error) {
32+
log := logf.FromContext(ctx).V(2)
33+
return storage.NewChunkedSecrets(secretsClient, "operator-controller", storage.ChunkedSecretsConfig{
34+
ChunkSize: 1024 * 1024,
35+
MaxReadChunks: 10,
36+
MaxWriteChunks: 10,
37+
Log: func(format string, args ...interface{}) { log.Info(fmt.Sprintf(format, args...)) },
38+
}), nil
39+
}, nil
40+
}
41+
42+
var _ clientcorev1.SecretInterface = &secretsDelegatingClient{}
43+
44+
type secretsDelegatingClient struct {
45+
clientcorev1.SecretInterface
46+
namespace string
47+
cache crcache.Cache
48+
}
49+
50+
func newSecretsDelegatingClient(client clientcorev1.SecretInterface, namespace string, cache crcache.Cache) clientcorev1.SecretInterface {
51+
return &secretsDelegatingClient{
52+
SecretInterface: client,
53+
namespace: namespace,
54+
cache: cache,
55+
}
56+
}
57+
58+
func (s secretsDelegatingClient) Get(ctx context.Context, name string, opts metav1.GetOptions) (*corev1.Secret, error) {
59+
var secret corev1.Secret
60+
if err := s.cache.Get(ctx, client.ObjectKey{Namespace: s.namespace, Name: name}, &secret, &client.GetOptions{Raw: &opts}); err != nil {
61+
return nil, err
62+
}
63+
return &secret, nil
64+
}
65+
66+
func (s secretsDelegatingClient) List(ctx context.Context, opts metav1.ListOptions) (*corev1.SecretList, error) {
67+
listOpts, err := metaOptionsToClientOptions(s.namespace, opts)
68+
if err != nil {
69+
return nil, err
70+
}
71+
72+
var secrets corev1.SecretList
73+
if err := s.cache.List(ctx, &secrets, listOpts); err != nil {
74+
return nil, fmt.Errorf("error listing from cache: %w", err)
75+
}
76+
return &secrets, nil
77+
}
78+
79+
func (s secretsDelegatingClient) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
80+
panic("intentionally not implemented: watch is not intended to be called")
81+
}
82+
83+
func metaOptionsToClientOptions(namespace string, opts metav1.ListOptions) (*client.ListOptions, error) {
84+
clientListOptions := &client.ListOptions{
85+
Namespace: namespace,
86+
Limit: opts.Limit,
87+
Continue: opts.Continue,
88+
}
89+
90+
if opts.LabelSelector != "" {
91+
labelSelector, err := k8slabels.Parse(opts.LabelSelector)
92+
if err != nil {
93+
return nil, err
94+
}
95+
clientListOptions.LabelSelector = labelSelector
96+
}
97+
98+
if opts.FieldSelector != "" {
99+
fieldSelector, err := fields.ParseSelector(opts.FieldSelector)
100+
if err != nil {
101+
return nil, err
102+
}
103+
clientListOptions.FieldSelector = fieldSelector
104+
}
105+
106+
opts.LabelSelector = ""
107+
opts.FieldSelector = ""
108+
opts.Limit = 0
109+
opts.Continue = ""
110+
clientListOptions.Raw = &opts
111+
112+
return clientListOptions, nil
113+
}

0 commit comments

Comments
 (0)