diff --git a/Makefile b/Makefile
index dd45b7f61..af69dc0aa 100644
--- a/Makefile
+++ b/Makefile
@@ -14,8 +14,8 @@ all: manager
# Run tests
test: generate fmt vet manifests api-docs
- go test ./... -coverprofile cover.out
- cd api; go test ./... -coverprofile cover.out
+ go test ./... -race -coverprofile cover.out
+ cd api; go test ./... -race -coverprofile cover.out
# Build manager binary
manager: generate fmt vet
diff --git a/api/go.mod b/api/go.mod
index f93f020ed..adcf9ddf1 100644
--- a/api/go.mod
+++ b/api/go.mod
@@ -4,6 +4,6 @@ go 1.16
require (
github.com/fluxcd/pkg/apis/meta v0.10.0
- k8s.io/apimachinery v0.21.1
+ k8s.io/apimachinery v0.21.2
sigs.k8s.io/controller-runtime v0.9.0
)
diff --git a/api/go.sum b/api/go.sum
index bef3f46c1..d6b9fa204 100644
--- a/api/go.sum
+++ b/api/go.sum
@@ -512,6 +512,7 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@@ -674,8 +675,9 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
k8s.io/api v0.21.1 h1:94bbZ5NTjdINJEdzOkpS4vdPhkb1VFpTYC9zh43f75c=
k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s=
k8s.io/apiextensions-apiserver v0.21.1/go.mod h1:KESQFCGjqVcVsZ9g0xX5bacMjyX5emuWcS2arzdEouA=
-k8s.io/apimachinery v0.21.1 h1:Q6XuHGlj2xc+hlMCvqyYfbv3H7SRGn2c8NycxJquDVs=
k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY=
+k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc=
+k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM=
k8s.io/apiserver v0.21.1/go.mod h1:nLLYZvMWn35glJ4/FZRhzLG/3MPxAaZTgV4FJZdr+tY=
k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs=
k8s.io/code-generator v0.21.1/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q=
diff --git a/api/v1beta1/bucket_types.go b/api/v1beta1/bucket_types.go
index 492002b82..5650f6fe8 100644
--- a/api/v1beta1/bucket_types.go
+++ b/api/v1beta1/bucket_types.go
@@ -18,7 +18,6 @@ package v1beta1
import (
"github.com/fluxcd/pkg/apis/meta"
- apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -112,51 +111,20 @@ const (
BucketOperationFailedReason string = "BucketOperationFailed"
)
-// BucketProgressing resets the conditions of the Bucket to metav1.Condition of
-// type meta.ReadyCondition with status 'Unknown' and meta.ProgressingReason
-// reason and message. It returns the modified Bucket.
-func BucketProgressing(bucket Bucket) Bucket {
- bucket.Status.ObservedGeneration = bucket.Generation
- bucket.Status.URL = ""
- bucket.Status.Conditions = []metav1.Condition{}
- meta.SetResourceCondition(&bucket, meta.ReadyCondition, metav1.ConditionUnknown, meta.ProgressingReason, "reconciliation in progress")
- return bucket
-}
-
-// BucketReady sets the given Artifact and URL on the Bucket and sets the
-// meta.ReadyCondition to 'True', with the given reason and message. It returns
-// the modified Bucket.
-func BucketReady(bucket Bucket, artifact Artifact, url, reason, message string) Bucket {
- bucket.Status.Artifact = &artifact
- bucket.Status.URL = url
- meta.SetResourceCondition(&bucket, meta.ReadyCondition, metav1.ConditionTrue, reason, message)
- return bucket
-}
-
-// BucketNotReady sets the meta.ReadyCondition on the Bucket to 'False', with
-// the given reason and message. It returns the modified Bucket.
-func BucketNotReady(bucket Bucket, reason, message string) Bucket {
- meta.SetResourceCondition(&bucket, meta.ReadyCondition, metav1.ConditionFalse, reason, message)
- return bucket
-}
-
-// BucketReadyMessage returns the message of the metav1.Condition of type
-// meta.ReadyCondition with status 'True' if present, or an empty string.
-func BucketReadyMessage(bucket Bucket) string {
- if c := apimeta.FindStatusCondition(bucket.Status.Conditions, meta.ReadyCondition); c != nil {
- if c.Status == metav1.ConditionTrue {
- return c.Message
- }
- }
- return ""
-}
-
// GetArtifact returns the latest artifact from the source if present in the
// status sub-resource.
func (in *Bucket) GetArtifact() *Artifact {
return in.Status.Artifact
}
+func (in Bucket) GetConditions() []metav1.Condition {
+ return in.Status.Conditions
+}
+
+func (in *Bucket) SetConditions(conditions []metav1.Condition) {
+ in.Status.Conditions = conditions
+}
+
// GetStatusConditions returns a pointer to the Status.Conditions slice
func (in *Bucket) GetStatusConditions() *[]metav1.Condition {
return &in.Status.Conditions
diff --git a/api/v1beta1/condition_types.go b/api/v1beta1/condition_types.go
index 4077a2ab6..8c1c5c6f4 100644
--- a/api/v1beta1/condition_types.go
+++ b/api/v1beta1/condition_types.go
@@ -18,6 +18,16 @@ package v1beta1
const SourceFinalizer = "finalizers.fluxcd.io"
+const (
+ SourceAvailableCondition string = "SourceAvailable"
+
+ SourceVerifiedCondition string = "SourceVerified"
+
+ ArtifactAvailableCondition string = "ArtifactAvailable"
+
+ SourceRefReadyCondition string = "SourceRefReady"
+)
+
const (
// URLInvalidReason represents the fact that a given source has an invalid URL.
URLInvalidReason string = "URLInvalid"
diff --git a/api/v1beta1/gitrepository_types.go b/api/v1beta1/gitrepository_types.go
index 6c178d02c..21bbeaecc 100644
--- a/api/v1beta1/gitrepository_types.go
+++ b/api/v1beta1/gitrepository_types.go
@@ -18,7 +18,6 @@ package v1beta1
import (
"github.com/fluxcd/pkg/apis/meta"
- apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -183,55 +182,22 @@ const (
GitOperationFailedReason string = "GitOperationFailed"
)
-// GitRepositoryProgressing resets the conditions of the GitRepository to
-// metav1.Condition of type meta.ReadyCondition with status 'Unknown' and
-// meta.ProgressingReason reason and message. It returns the modified
-// GitRepository.
-func GitRepositoryProgressing(repository GitRepository) GitRepository {
- repository.Status.ObservedGeneration = repository.Generation
- repository.Status.URL = ""
- repository.Status.Conditions = []metav1.Condition{}
- meta.SetResourceCondition(&repository, meta.ReadyCondition, metav1.ConditionUnknown, meta.ProgressingReason, "reconciliation in progress")
- return repository
-}
-
-// GitRepositoryReady sets the given Artifact and URL on the GitRepository and
-// sets the meta.ReadyCondition to 'True', with the given reason and message. It
-// returns the modified GitRepository.
-func GitRepositoryReady(repository GitRepository, artifact Artifact, includedArtifacts []*Artifact, url, reason, message string) GitRepository {
- repository.Status.Artifact = &artifact
- repository.Status.IncludedArtifacts = includedArtifacts
- repository.Status.URL = url
- meta.SetResourceCondition(&repository, meta.ReadyCondition, metav1.ConditionTrue, reason, message)
- return repository
-}
-
-// GitRepositoryNotReady sets the meta.ReadyCondition on the given GitRepository
-// to 'False', with the given reason and message. It returns the modified
-// GitRepository.
-func GitRepositoryNotReady(repository GitRepository, reason, message string) GitRepository {
- meta.SetResourceCondition(&repository, meta.ReadyCondition, metav1.ConditionFalse, reason, message)
- return repository
+// GetArtifact returns the latest artifact from the source if present in the
+// status sub-resource.
+func (in GitRepository) GetArtifact() *Artifact {
+ return in.Status.Artifact
}
-// GitRepositoryReadyMessage returns the message of the metav1.Condition of type
-// meta.ReadyCondition with status 'True' if present, or an empty string.
-func GitRepositoryReadyMessage(repository GitRepository) string {
- if c := apimeta.FindStatusCondition(repository.Status.Conditions, meta.ReadyCondition); c != nil {
- if c.Status == metav1.ConditionTrue {
- return c.Message
- }
- }
- return ""
+func (in GitRepository) GetConditions() []metav1.Condition {
+ return in.Status.Conditions
}
-// GetArtifact returns the latest artifact from the source if present in the
-// status sub-resource.
-func (in *GitRepository) GetArtifact() *Artifact {
- return in.Status.Artifact
+func (in *GitRepository) SetConditions(conditions []metav1.Condition) {
+ in.Status.Conditions = conditions
}
-// GetStatusConditions returns a pointer to the Status.Conditions slice
+// GetStatusConditions returns a pointer to the Status.Conditions slice.
+// Deprecated.
func (in *GitRepository) GetStatusConditions() *[]metav1.Condition {
return &in.Status.Conditions
}
diff --git a/api/v1beta1/helmchart_types.go b/api/v1beta1/helmchart_types.go
index 96f027800..c0ffd8af9 100644
--- a/api/v1beta1/helmchart_types.go
+++ b/api/v1beta1/helmchart_types.go
@@ -18,13 +18,19 @@ package v1beta1
import (
"github.com/fluxcd/pkg/apis/meta"
- apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// HelmChartKind is the string representation of a HelmChart.
const HelmChartKind = "HelmChart"
+const (
+ DependenciesBuildCondition string = "DependenciesBuild"
+ ValuesFilesMergedCondition string = "ValuesFilesMerged"
+ ChartPackagedCondition string = "ChartPackaged"
+ ChartReconciled string = "ChartReconciled"
+)
+
// HelmChartSpec defines the desired state of a Helm chart.
type HelmChartSpec struct {
// The name or path the Helm chart is available at in the SourceRef.
@@ -122,53 +128,24 @@ const (
ChartPackageSucceededReason string = "ChartPackageSucceeded"
)
-// HelmChartProgressing resets the conditions of the HelmChart to meta.Condition
-// of type meta.ReadyCondition with status 'Unknown' and meta.ProgressingReason
-// reason and message. It returns the modified HelmChart.
-func HelmChartProgressing(chart HelmChart) HelmChart {
- chart.Status.ObservedGeneration = chart.Generation
- chart.Status.URL = ""
- chart.Status.Conditions = []metav1.Condition{}
- meta.SetResourceCondition(&chart, meta.ReadyCondition, metav1.ConditionUnknown, meta.ProgressingReason, "reconciliation in progress")
- return chart
-}
-
-// HelmChartReady sets the given Artifact and URL on the HelmChart and sets the
-// meta.ReadyCondition to 'True', with the given reason and message. It returns
-// the modified HelmChart.
-func HelmChartReady(chart HelmChart, artifact Artifact, url, reason, message string) HelmChart {
- chart.Status.Artifact = &artifact
- chart.Status.URL = url
- meta.SetResourceCondition(&chart, meta.ReadyCondition, metav1.ConditionTrue, reason, message)
- return chart
-}
-
-// HelmChartNotReady sets the meta.ReadyCondition on the given HelmChart to
-// 'False', with the given reason and message. It returns the modified
-// HelmChart.
-func HelmChartNotReady(chart HelmChart, reason, message string) HelmChart {
- meta.SetResourceCondition(&chart, meta.ReadyCondition, metav1.ConditionFalse, reason, message)
- return chart
-}
-
-// HelmChartReadyMessage returns the message of the meta.ReadyCondition with
-// status 'True', or an empty string.
-func HelmChartReadyMessage(chart HelmChart) string {
- if c := apimeta.FindStatusCondition(chart.Status.Conditions, meta.ReadyCondition); c != nil {
- if c.Status == metav1.ConditionTrue {
- return c.Message
- }
- }
- return ""
-}
-
// GetArtifact returns the latest artifact from the source if present in the
// status sub-resource.
func (in *HelmChart) GetArtifact() *Artifact {
return in.Status.Artifact
}
-// GetStatusConditions returns a pointer to the Status.Conditions slice
+// GetConditions returns the status conditions.
+func (in HelmChart) GetConditions() []metav1.Condition {
+ return in.Status.Conditions
+}
+
+// SetConditions sets the status conditions.
+func (in *HelmChart) SetConditions(conditions []metav1.Condition) {
+ in.Status.Conditions = conditions
+}
+
+// GetStatusConditions returns a pointer to the Status.Conditions slice.
+// Deprecated: use GetConditions instead.
func (in *HelmChart) GetStatusConditions() *[]metav1.Condition {
return &in.Status.Conditions
}
diff --git a/api/v1beta1/helmrepository_types.go b/api/v1beta1/helmrepository_types.go
index 40f918d2d..f023266b0 100644
--- a/api/v1beta1/helmrepository_types.go
+++ b/api/v1beta1/helmrepository_types.go
@@ -18,7 +18,6 @@ package v1beta1
import (
"github.com/fluxcd/pkg/apis/meta"
- apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -99,53 +98,20 @@ const (
IndexationSucceededReason string = "IndexationSucceed"
)
-// HelmRepositoryProgressing resets the conditions of the HelmRepository to
-// metav1.Condition of type meta.ReadyCondition with status 'Unknown' and
-// meta.ProgressingReason reason and message. It returns the modified
-// HelmRepository.
-func HelmRepositoryProgressing(repository HelmRepository) HelmRepository {
- repository.Status.ObservedGeneration = repository.Generation
- repository.Status.URL = ""
- repository.Status.Conditions = []metav1.Condition{}
- meta.SetResourceCondition(&repository, meta.ReadyCondition, metav1.ConditionUnknown, meta.ProgressingReason, "reconciliation in progress")
- return repository
-}
-
-// HelmRepositoryReady sets the given Artifact and URL on the HelmRepository and
-// sets the meta.ReadyCondition to 'True', with the given reason and message. It
-// returns the modified HelmRepository.
-func HelmRepositoryReady(repository HelmRepository, artifact Artifact, url, reason, message string) HelmRepository {
- repository.Status.Artifact = &artifact
- repository.Status.URL = url
- meta.SetResourceCondition(&repository, meta.ReadyCondition, metav1.ConditionTrue, reason, message)
- return repository
-}
-
-// HelmRepositoryNotReady sets the meta.ReadyCondition on the given
-// HelmRepository to 'False', with the given reason and message. It returns the
-// modified HelmRepository.
-func HelmRepositoryNotReady(repository HelmRepository, reason, message string) HelmRepository {
- meta.SetResourceCondition(&repository, meta.ReadyCondition, metav1.ConditionFalse, reason, message)
- return repository
-}
-
-// HelmRepositoryReadyMessage returns the message of the metav1.Condition of type
-// meta.ReadyCondition with status 'True' if present, or an empty string.
-func HelmRepositoryReadyMessage(repository HelmRepository) string {
- if c := apimeta.FindStatusCondition(repository.Status.Conditions, meta.ReadyCondition); c != nil {
- if c.Status == metav1.ConditionTrue {
- return c.Message
- }
- }
- return ""
-}
-
// GetArtifact returns the latest artifact from the source if present in the
// status sub-resource.
func (in *HelmRepository) GetArtifact() *Artifact {
return in.Status.Artifact
}
+func (in HelmRepository) GetConditions() []metav1.Condition {
+ return in.Status.Conditions
+}
+
+func (in *HelmRepository) SetConditions(conditions []metav1.Condition) {
+ in.Status.Conditions = conditions
+}
+
// GetStatusConditions returns a pointer to the Status.Conditions slice
func (in *HelmRepository) GetStatusConditions() *[]metav1.Condition {
return &in.Status.Conditions
diff --git a/api/v1beta1/source.go b/api/v1beta1/source.go
index 5c10d00fc..14402db13 100644
--- a/api/v1beta1/source.go
+++ b/api/v1beta1/source.go
@@ -18,6 +18,7 @@ package v1beta1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
)
const (
@@ -29,9 +30,15 @@ const (
// Source interface must be supported by all API types.
// +k8s:deepcopy-gen=false
type Source interface {
+ metav1.Object
+ runtime.Object
+
// GetArtifact returns the latest artifact from the source if present in the
// status sub-resource.
GetArtifact() *Artifact
+ // GetConditions returns the conditions of the source if present in the
+ // status sub-resource.
+ GetConditions() []metav1.Condition
// GetInterval returns the interval at which the source is updated.
GetInterval() metav1.Duration
}
diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
index fd20920de..e0ba49e32 100644
--- a/api/v1beta1/zz_generated.deepcopy.go
+++ b/api/v1beta1/zz_generated.deepcopy.go
@@ -23,7 +23,7 @@ package v1beta1
import (
"github.com/fluxcd/pkg/apis/meta"
"k8s.io/apimachinery/pkg/apis/meta/v1"
- runtime "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/runtime"
)
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
diff --git a/controllers/artifact.go b/controllers/artifact.go
index 0e16fd03c..3cc61d0ec 100644
--- a/controllers/artifact.go
+++ b/controllers/artifact.go
@@ -2,6 +2,25 @@ package controllers
import sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+type artifactSet []*sourcev1.Artifact
+
+func (s artifactSet) Diff(set artifactSet) bool {
+ if len(s) != len(set) {
+ return false
+ }
+
+outer:
+ for _, j := range s {
+ for _, k := range set {
+ if k.HasRevision(j.Revision) {
+ continue outer
+ }
+ }
+ return true
+ }
+ return false
+}
+
// hasArtifactUpdated returns true if any of the revisions in the current artifacts
// does not match any of the artifacts in the updated artifacts
func hasArtifactUpdated(current []*sourcev1.Artifact, updated []*sourcev1.Artifact) bool {
diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go
index 9bd62c2d3..1e7b4de6f 100644
--- a/controllers/bucket_controller.go
+++ b/controllers/bucket_controller.go
@@ -20,7 +20,6 @@ import (
"context"
"crypto/sha1"
"fmt"
- "io/ioutil"
"os"
"path/filepath"
"strings"
@@ -31,12 +30,9 @@ import (
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/s3utils"
corev1 "k8s.io/api/core/v1"
- apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
- kuberecorder "k8s.io/client-go/tools/record"
- "k8s.io/client-go/tools/reference"
+ kerrors "k8s.io/apimachinery/pkg/util/errors"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -44,8 +40,10 @@ import (
"sigs.k8s.io/controller-runtime/pkg/predicate"
"github.com/fluxcd/pkg/apis/meta"
+ "github.com/fluxcd/pkg/runtime/conditions"
+ helper "github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/events"
- "github.com/fluxcd/pkg/runtime/metrics"
+ "github.com/fluxcd/pkg/runtime/patch"
"github.com/fluxcd/pkg/runtime/predicates"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
@@ -60,11 +58,10 @@ import (
// BucketReconciler reconciles a Bucket object
type BucketReconciler struct {
client.Client
- Scheme *runtime.Scheme
- Storage *Storage
- EventRecorder kuberecorder.EventRecorder
- ExternalEventRecorder *events.Recorder
- MetricsRecorder *metrics.Recorder
+ helper.Events
+ helper.Metrics
+
+ Storage *Storage
}
type BucketReconcilerOptions struct {
@@ -83,153 +80,247 @@ func (r *BucketReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, opts Buc
Complete(r)
}
-func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, retErr error) {
start := time.Now()
log := ctrl.LoggerFrom(ctx)
- var bucket sourcev1.Bucket
- if err := r.Get(ctx, req.NamespacedName, &bucket); err != nil {
+ // Fetch the Bucket
+ obj := &sourcev1.Bucket{}
+ if err := r.Get(ctx, req.NamespacedName, obj); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
// Record suspended status metric
- defer r.recordSuspension(ctx, bucket)
+ r.RecordSuspend(ctx, obj, obj.Spec.Suspend)
- // Add our finalizer if it does not exist
- if !controllerutil.ContainsFinalizer(&bucket, sourcev1.SourceFinalizer) {
- controllerutil.AddFinalizer(&bucket, sourcev1.SourceFinalizer)
- if err := r.Update(ctx, &bucket); err != nil {
- log.Error(err, "unable to register finalizer")
- return ctrl.Result{}, err
+ // Return early if the object is suspended
+ if obj.Spec.Suspend {
+ log.Info("Reconciliation is suspended for this object")
+ return ctrl.Result{}, nil
+ }
+
+ // Initialize the patch helper
+ patchHelper, err := patch.NewHelper(obj, r.Client)
+ if err != nil {
+ return ctrl.Result{}, err
+ }
+
+ // Always attempt to patch the object and status after each
+ // reconciliation
+ defer func() {
+ // Record the value of the reconciliation request, if any
+ if v, ok := meta.ReconcileAnnotationValue(obj.GetAnnotations()); ok {
+ obj.Status.SetLastHandledReconcileRequest(v)
}
+
+ // Summarize Ready condition
+ conditions.SetSummary(obj,
+ meta.ReadyCondition,
+ conditions.WithConditions(
+ sourcev1.ArtifactAvailableCondition,
+ sourcev1.SourceAvailableCondition,
+ ),
+ )
+
+ // Patch the object, ignoring conflicts on the conditions owned by
+ // this controller
+ patchOpts := []patch.Option{
+ patch.WithOwnedConditions{
+ Conditions: []string{
+ sourcev1.ArtifactAvailableCondition,
+ sourcev1.SourceAvailableCondition,
+ meta.ReadyCondition,
+ meta.ReconcilingCondition,
+ meta.StalledCondition,
+ },
+ },
+ }
+
+ // Determine if the resource is still being reconciled, or if
+ // it has stalled, and record this observation
+ if retErr == nil && (result.IsZero() || !result.Requeue) {
+ // We are no longer reconciling
+ conditions.Delete(obj, meta.ReconcilingCondition)
+
+ // We have now observed this generation
+ patchOpts = append(patchOpts, patch.WithStatusObservedGeneration{})
+
+ readyCondition := conditions.Get(obj, meta.ReadyCondition)
+ switch readyCondition.Status {
+ case metav1.ConditionFalse:
+ // As we are no longer reconciling and the end-state
+ // is not ready, the reconciliation has stalled
+ conditions.MarkStalled(obj, readyCondition.Reason, readyCondition.Message)
+ case metav1.ConditionTrue:
+ // As we are no longer reconciling and the end-state
+ // is ready, the reconciliation is no longer stalled
+ conditions.Delete(obj, meta.StalledCondition)
+ }
+ }
+
+ // Finally, patch the resource
+ if err := patchHelper.Patch(ctx, obj, patchOpts...); err != nil {
+ retErr = kerrors.NewAggregate([]error{retErr, err})
+ }
+
+ // Always record readiness and duration metrics
+ r.Metrics.RecordReadiness(ctx, obj)
+ r.Metrics.RecordDuration(ctx, obj, start)
+ }()
+
+ // Add finalizer first if not exist to avoid the race condition
+ // between init and delete
+ if !controllerutil.ContainsFinalizer(obj, sourcev1.SourceFinalizer) {
+ controllerutil.AddFinalizer(obj, sourcev1.SourceFinalizer)
+ return ctrl.Result{Requeue: true}, nil
}
// Examine if the object is under deletion
- if !bucket.ObjectMeta.DeletionTimestamp.IsZero() {
- return r.reconcileDelete(ctx, bucket)
+ if !obj.ObjectMeta.DeletionTimestamp.IsZero() {
+ return r.reconcileDelete(ctx, obj)
}
- // Return early if the object is suspended.
- if bucket.Spec.Suspend {
- log.Info("Reconciliation is suspended for this object")
- return ctrl.Result{}, nil
- }
+ // Reconcile actual object
+ return r.reconcile(ctx, obj)
+}
- // record reconciliation duration
- if r.MetricsRecorder != nil {
- objRef, err := reference.GetReference(r.Scheme, &bucket)
- if err != nil {
- return ctrl.Result{}, err
- }
- defer r.MetricsRecorder.RecordDuration(*objRef, start)
+func (r *BucketReconciler) reconcile(ctx context.Context, obj *sourcev1.Bucket) (ctrl.Result, error) {
+ // Mark the resource as under reconciliation
+ conditions.MarkReconciling(obj, "Reconciling", "")
+ logr.FromContext(ctx).Info("Starting reconciliation")
+
+ // Reconcile the storage data
+ if result, err := r.reconcileStorage(ctx, obj); err != nil {
+ return result, err
}
- // set initial status
- if resetBucket, ok := r.resetStatus(bucket); ok {
- bucket = resetBucket
- if err := r.updateStatus(ctx, req, bucket.Status); err != nil {
- log.Error(err, "unable to update status")
- return ctrl.Result{Requeue: true}, err
- }
- r.recordReadiness(ctx, bucket)
+ // Create temp dir for the bucket objects
+ tmpDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s-%s-", obj.Kind, obj.Namespace, obj.Name))
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to create temporary directory: %s", err)
+ return ctrl.Result{}, err
}
+ defer os.RemoveAll(tmpDir)
- // record the value of the reconciliation request, if any
- // TODO(hidde): would be better to defer this in combination with
- // always patching the status sub-resource after a reconciliation.
- if v, ok := meta.ReconcileAnnotationValue(bucket.GetAnnotations()); ok {
- bucket.Status.SetLastHandledReconcileRequest(v)
+ // Reconcile the source from upstream
+ var artifact sourcev1.Artifact
+ if result, err := r.reconcileSource(ctx, obj, &artifact, tmpDir); err != nil || conditions.IsFalse(obj, sourcev1.SourceAvailableCondition) {
+ return result, err
}
- // purge old artifacts from storage
- if err := r.gc(bucket); err != nil {
- log.Error(err, "unable to purge old artifacts")
+ // Reconcile the artifact to storage
+ if result, err := r.reconcileArtifact(ctx, obj, artifact, tmpDir); err != nil {
+ return result, err
}
- // reconcile bucket by downloading its content
- reconciledBucket, reconcileErr := r.reconcile(ctx, *bucket.DeepCopy())
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+}
- // update status with the reconciliation result
- if err := r.updateStatus(ctx, req, reconciledBucket.Status); err != nil {
- log.Error(err, "unable to update status")
- return ctrl.Result{Requeue: true}, err
+// reconcileStorage reconciles the storage data for the given object
+// by garbage collecting previous advertised artifact(s) from storage,
+// observing if the artifact in the status still exists, and
+// ensuring the URLs are up-to-date with the current hostname
+// configuration.
+func (r *BucketReconciler) reconcileStorage(ctx context.Context, obj *sourcev1.Bucket) (ctrl.Result, error) {
+ // Garbage collect previous advertised artifact(s) from storage
+ if err := r.garbageCollect(obj); err != nil {
+ r.Events.Eventf(ctx, obj, events.EventSeverityError, "GarbageCollectionFailed", "Garbage collection failed: %s", err)
}
- // if reconciliation failed, record the failure and requeue immediately
- if reconcileErr != nil {
- r.event(ctx, reconciledBucket, events.EventSeverityError, reconcileErr.Error())
- r.recordReadiness(ctx, reconciledBucket)
- return ctrl.Result{Requeue: true}, reconcileErr
+ // Determine if the advertised artifact is still in storage
+ if artifact := obj.GetArtifact(); artifact != nil && !r.Storage.ArtifactExist(*artifact) {
+ obj.Status.Artifact = nil
+ obj.Status.URL = ""
}
- // emit revision change event
- if bucket.Status.Artifact == nil || reconciledBucket.Status.Artifact.Revision != bucket.Status.Artifact.Revision {
- r.event(ctx, reconciledBucket, events.EventSeverityInfo, sourcev1.BucketReadyMessage(reconciledBucket))
+ // Record that we do not have an artifact
+ if obj.GetArtifact() == nil {
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, "NoArtifactFound", "No artifact for resource in storage")
+ return ctrl.Result{Requeue: true}, nil
}
- r.recordReadiness(ctx, reconciledBucket)
- log.Info(fmt.Sprintf("Reconciliation finished in %s, next run in %s",
- time.Now().Sub(start).String(),
- bucket.GetInterval().Duration.String(),
- ))
+ // Always update URLs to ensure hostname is up-to-date
+ r.Storage.SetArtifactURL(obj.GetArtifact())
+ obj.Status.URL = r.Storage.SetHostname(obj.Status.URL)
- return ctrl.Result{RequeueAfter: bucket.GetInterval().Duration}, nil
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
}
-func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, error) {
- s3Client, err := r.auth(ctx, bucket)
- if err != nil {
- err = fmt.Errorf("auth error: %w", err)
- return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err
+// reconcileSource reconciles the bucket from upstream to the given
+// directory path while using the information on the object to determine
+// authentication and exclude strategies.
+// On a successful download of all bucket objects, the given pointer is
+// set to a new artifact.
+func (r *BucketReconciler) reconcileSource(ctx context.Context, obj *sourcev1.Bucket, artifact *sourcev1.Artifact, dir string) (ctrl.Result, error) {
+ // Attempt to retrieve secret if one is configured
+ var secret *corev1.Secret
+ if obj.Spec.SecretRef != nil {
+ secret = &corev1.Secret{}
+ name := types.NamespacedName{
+ Namespace: obj.GetNamespace(),
+ Name: obj.Spec.SecretRef.Name,
+ }
+ if err := r.Client.Get(ctx, name, secret); err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.AuthenticationFailedReason, "Failed to get secret '%s': %s", name.String(), err.Error())
+ r.Events.Event(ctx, obj, events.EventSeverityError, sourcev1.AuthenticationFailedReason, conditions.Get(obj, sourcev1.SourceAvailableCondition).Message)
+ // Return transient errors but wait for next interval on not found
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, client.IgnoreNotFound(err)
+ }
}
- // create tmp dir
- tempDir, err := ioutil.TempDir("", bucket.Name)
+ // Build the client with the configuration from the object and secret
+ s3Client, err := r.buildClient(obj, secret)
if err != nil {
- err = fmt.Errorf("tmp dir error: %w", err)
- return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.BucketOperationFailedReason, "Failed to construct S3 client: %s", err.Error())
+ // Recovering from this without a change to the secret or object
+ // is impossible
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
}
- defer os.RemoveAll(tempDir)
-
- ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration)
+ ctxTimeout, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration)
defer cancel()
- exists, err := s3Client.BucketExists(ctxTimeout, bucket.Spec.BucketName)
+ // Confirm bucket exists
+ exists, err := s3Client.BucketExists(ctxTimeout, obj.Spec.BucketName)
if err != nil {
- return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
+ // Error may be transient
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.BucketOperationFailedReason, "Failed to verify existence of bucket %q: %s", obj.Spec.BucketName, err.Error())
+ return ctrl.Result{}, err
}
if !exists {
- err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName)
- return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.BucketOperationFailedReason, "Bucket %q does not exist", obj.Spec.BucketName)
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
}
// Look for file with ignore rules first
// NB: S3 has flat filepath keys making it impossible to look
// for files in "subdirectories" without building up a tree first.
- path := filepath.Join(tempDir, sourceignore.IgnoreFile)
- if err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path, minio.GetObjectOptions{}); err != nil {
+ path := filepath.Join(dir, sourceignore.IgnoreFile)
+ if err := s3Client.FGetObject(ctxTimeout, obj.Spec.BucketName, sourceignore.IgnoreFile, path, minio.GetObjectOptions{}); err != nil {
if resp, ok := err.(minio.ErrorResponse); ok && resp.Code != "NoSuchKey" {
- return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.BucketOperationFailedReason, "Failed to download '%s' file: %s", sourceignore.IgnoreFile, err.Error())
+ return ctrl.Result{}, err
}
}
ps, err := sourceignore.ReadIgnoreFile(path, nil)
if err != nil {
- return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.BucketOperationFailedReason, "Failed to read '%s' file: %s", sourceignore.IgnoreFile, err.Error())
+ return ctrl.Result{}, err
}
// In-spec patterns take precedence
- if bucket.Spec.Ignore != nil {
- ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*bucket.Spec.Ignore), nil)...)
+ if obj.Spec.Ignore != nil {
+ ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*obj.Spec.Ignore), nil)...)
}
matcher := sourceignore.NewMatcher(ps)
- // download bucket content
- for object := range s3Client.ListObjects(ctxTimeout, bucket.Spec.BucketName, minio.ListObjectsOptions{
+ // Download bucket objects while taking the ignore rules into account
+ var objCount int64
+ for object := range s3Client.ListObjects(ctxTimeout, obj.Spec.BucketName, minio.ListObjectsOptions{
Recursive: true,
UseV1: s3utils.IsGoogleEndpoint(*s3Client.EndpointURL()),
}) {
- if object.Err != nil {
- err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, object.Err)
- return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
+ if err = object.Err; err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.BucketOperationFailedReason, "Failed to list objects from bucket %q: %s", obj.Spec.BucketName, err.Error())
+ return ctrl.Result{}, err
}
if strings.HasSuffix(object.Key, "/") || object.Key == sourceignore.IgnoreFile {
@@ -240,120 +331,134 @@ func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket
continue
}
- localPath := filepath.Join(tempDir, object.Key)
- err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Key, localPath, minio.GetObjectOptions{})
- if err != nil {
- err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err)
- return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
+ localPath := filepath.Join(dir, object.Key)
+ if err = s3Client.FGetObject(ctx, obj.Spec.BucketName, object.Key, localPath, minio.GetObjectOptions{}); err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.BucketOperationFailedReason, "Failed to download object %q from bucket %q: %s", object.Key, obj.Spec.BucketName, err.Error())
+ return ctrl.Result{}, err
}
+
+ objCount++
}
- revision, err := r.checksum(tempDir)
+ // Compute the checksum of the downloaded file contents, which is used as the revision
+ checksum, err := r.checksum(dir)
if err != nil {
- return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.BucketOperationFailedReason, "Failed to compute checksum for downloaded objects: %s", err.Error())
+ return ctrl.Result{}, err
}
- // return early on unchanged revision
- artifact := r.Storage.NewArtifactFor(bucket.Kind, bucket.GetObjectMeta(), revision, fmt.Sprintf("%s.tar.gz", revision))
- if apimeta.IsStatusConditionTrue(bucket.Status.Conditions, meta.ReadyCondition) && bucket.GetArtifact().HasRevision(artifact.Revision) {
- if artifact.URL != bucket.GetArtifact().URL {
- r.Storage.SetArtifactURL(bucket.GetArtifact())
- bucket.Status.URL = r.Storage.SetHostname(bucket.Status.URL)
- }
- return bucket, nil
- }
+ // Create potential new artifact
+ *artifact = r.Storage.NewArtifactFor(obj.Kind, obj, checksum, fmt.Sprintf("%s.tar.gz", checksum))
+ conditions.MarkTrue(obj, sourcev1.SourceAvailableCondition, sourcev1.BucketOperationSucceedReason, "Downloaded %d objects from bucket %q", objCount, obj.Spec.BucketName)
- // create artifact dir
- err = r.Storage.MkdirAll(artifact)
- if err != nil {
- err = fmt.Errorf("mkdir dir error: %w", err)
- return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+}
+
+// reconcileArtifact reconciles the downloaded files to the artifact
+// storage by archiving the directory.
+// On a successful archive, the artifact and includes in the status of
+// the given object are set, and the symlink in the storage is updated
+// to its path.
+func (r *BucketReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.Bucket, artifact sourcev1.Artifact, dir string) (ctrl.Result, error) {
+ // The artifact is up-to-date
+ if obj.GetArtifact().HasRevision(artifact.Revision) {
+ logr.FromContext(ctx).Info("Artifact is up-to-date")
+ conditions.MarkTrue(obj, sourcev1.ArtifactAvailableCondition, meta.SucceededReason, "Compressed source to artifact with revision '%s'", artifact.Revision)
+ return ctrl.Result{RequeueAfter: obj.GetInterval().Duration}, nil
+ }
+
+ // Ensure target path exists and is a directory
+ if f, err := os.Stat(dir); err != nil {
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to stat source path: %s", err.Error())
+ return ctrl.Result{}, err
+ } else if !f.IsDir() {
+ err = fmt.Errorf("source path %q is not a directory", dir)
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to reconcile artifact: %s", err.Error())
+ return ctrl.Result{}, err
}
- // acquire lock
+ // Ensure artifact directory exists and acquire lock
+ if err := r.Storage.MkdirAll(artifact); err != nil {
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to create directory: %s", err)
+ return ctrl.Result{}, err
+ }
unlock, err := r.Storage.Lock(artifact)
if err != nil {
- err = fmt.Errorf("unable to acquire lock: %w", err)
- return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to acquire lock: %s", err)
+ return ctrl.Result{}, err
}
defer unlock()
- // archive artifact and check integrity
- if err := r.Storage.Archive(&artifact, tempDir, nil); err != nil {
- err = fmt.Errorf("storage archive error: %w", err)
- return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err
+ // Archive directory to storage
+ if err = r.Storage.Archive(&artifact, dir, nil); err != nil {
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Unable to archive artifact to storage: %s", err)
+ return ctrl.Result{}, err
}
- // update latest symlink
+ // Record it on the object
+ obj.Status.Artifact = artifact.DeepCopy()
+ conditions.MarkTrue(obj, sourcev1.ArtifactAvailableCondition, meta.SucceededReason, "Compressed source to artifact with revision '%s'", artifact.Revision)
+ r.Events.EventWithMetaf(ctx, obj, map[string]string{
+ "revision": obj.GetArtifact().Revision,
+ }, events.EventSeverityInfo, meta.SucceededReason, conditions.Get(obj, sourcev1.ArtifactAvailableCondition).Message)
+
+ // Update symlink on a "best effort" basis
url, err := r.Storage.Symlink(artifact, "latest.tar.gz")
if err != nil {
- err = fmt.Errorf("storage symlink error: %w", err)
- return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err
+ r.Events.Eventf(ctx, obj, events.EventSeverityError, sourcev1.StorageOperationFailedReason, "Failed to update status URL symlink: %s", err)
+ }
+ if url != "" {
+ obj.Status.URL = url
}
- message := fmt.Sprintf("Fetched revision: %s", artifact.Revision)
- return sourcev1.BucketReady(bucket, artifact, url, sourcev1.BucketOperationSucceedReason, message), nil
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
}
-func (r *BucketReconciler) reconcileDelete(ctx context.Context, bucket sourcev1.Bucket) (ctrl.Result, error) {
- if err := r.gc(bucket); err != nil {
- r.event(ctx, bucket, events.EventSeverityError,
- fmt.Sprintf("garbage collection for deleted resource failed: %s", err.Error()))
+// reconcileDelete reconciles the delete of an object by garbage
+// collecting all artifacts for the object in the artifact storage,
+// if successful, the finalizer is removed from the object.
+func (r *BucketReconciler) reconcileDelete(ctx context.Context, obj *sourcev1.Bucket) (ctrl.Result, error) {
+ // Garbage collect the resource's artifacts
+ if err := r.garbageCollect(obj); err != nil {
+ r.Events.Eventf(ctx, obj, events.EventSeverityError, "GarbageCollectionFailed", "Garbage collection for deleted resource failed: %s", err)
// Return the error so we retry the failed garbage collection
return ctrl.Result{}, err
}
- // Record deleted status
- r.recordReadiness(ctx, bucket)
-
- // Remove our finalizer from the list and update it
- controllerutil.RemoveFinalizer(&bucket, sourcev1.SourceFinalizer)
- if err := r.Update(ctx, &bucket); err != nil {
- return ctrl.Result{}, err
- }
+ // Remove our finalizer from the list
+ controllerutil.RemoveFinalizer(obj, sourcev1.SourceFinalizer)
// Stop reconciliation as the object is being deleted
return ctrl.Result{}, nil
}
-func (r *BucketReconciler) auth(ctx context.Context, bucket sourcev1.Bucket) (*minio.Client, error) {
- opt := minio.Options{
- Region: bucket.Spec.Region,
- Secure: !bucket.Spec.Insecure,
+// buildClient constructs a minio.Client with the data from the given
+// Bucket and Secret. It returns an error if the given Secret does not
+// have the required fields, or if there is no credential handler
+// configured.
+func (r *BucketReconciler) buildClient(obj *sourcev1.Bucket, secret *corev1.Secret) (*minio.Client, error) {
+ opts := minio.Options{
+ Region: obj.Spec.Region,
+ Secure: !obj.Spec.Insecure,
}
- if bucket.Spec.SecretRef != nil {
- secretName := types.NamespacedName{
- Namespace: bucket.GetNamespace(),
- Name: bucket.Spec.SecretRef.Name,
- }
-
- var secret corev1.Secret
- if err := r.Get(ctx, secretName, &secret); err != nil {
- return nil, fmt.Errorf("credentials secret error: %w", err)
- }
-
- accesskey := ""
- secretkey := ""
+ if secret != nil {
+ var accessKey, secretKey string
if k, ok := secret.Data["accesskey"]; ok {
- accesskey = string(k)
+ accessKey = string(k)
}
if k, ok := secret.Data["secretkey"]; ok {
- secretkey = string(k)
+ secretKey = string(k)
}
- if accesskey == "" || secretkey == "" {
- return nil, fmt.Errorf("invalid '%s' secret data: required fields 'accesskey' and 'secretkey'", secret.Name)
+ if accessKey == "" || secretKey == "" {
+ return nil, fmt.Errorf("invalid %q secret data: required fields 'accesskey' and 'secretkey'", secret.Name)
}
- opt.Creds = credentials.NewStaticV4(accesskey, secretkey, "")
- } else if bucket.Spec.Provider == sourcev1.AmazonBucketProvider {
- opt.Creds = credentials.NewIAM("")
+ opts.Creds = credentials.NewStaticV4(accessKey, secretKey, "")
+ } else if obj.Spec.Provider == sourcev1.AmazonBucketProvider {
+ opts.Creds = credentials.NewIAM("")
}
- if opt.Creds == nil {
- return nil, fmt.Errorf("no bucket credentials found")
- }
-
- return minio.New(bucket.Spec.Endpoint, &opt)
+ return minio.New(obj.Spec.Endpoint, &opts)
}
// checksum calculates the SHA1 checksum of the given root directory.
@@ -368,7 +473,7 @@ func (r *BucketReconciler) checksum(root string) (string, error) {
if !info.Mode().IsRegular() {
return nil
}
- data, err := ioutil.ReadFile(path)
+ data, err := os.ReadFile(path)
if err != nil {
return err
}
@@ -384,102 +489,20 @@ func (r *BucketReconciler) checksum(root string) (string, error) {
return fmt.Sprintf("%x", sum.Sum(nil)), nil
}
-// resetStatus returns a modified v1beta1.Bucket and a boolean indicating
-// if the status field has been reset.
-func (r *BucketReconciler) resetStatus(bucket sourcev1.Bucket) (sourcev1.Bucket, bool) {
- // We do not have an artifact, or it does no longer exist
- if bucket.GetArtifact() == nil || !r.Storage.ArtifactExist(*bucket.GetArtifact()) {
- bucket = sourcev1.BucketProgressing(bucket)
- bucket.Status.Artifact = nil
- return bucket, true
- }
- if bucket.Generation != bucket.Status.ObservedGeneration {
- return sourcev1.BucketProgressing(bucket), true
- }
- return bucket, false
-}
-
-// gc performs a garbage collection for the given v1beta1.Bucket.
-// It removes all but the current artifact except for when the
-// deletion timestamp is set, which will result in the removal of
-// all artifacts for the resource.
-func (r *BucketReconciler) gc(bucket sourcev1.Bucket) error {
- if !bucket.DeletionTimestamp.IsZero() {
- return r.Storage.RemoveAll(r.Storage.NewArtifactFor(bucket.Kind, bucket.GetObjectMeta(), "", "*"))
- }
- if bucket.GetArtifact() != nil {
- return r.Storage.RemoveAllButCurrent(*bucket.GetArtifact())
- }
- return nil
-}
-
-// event emits a Kubernetes event and forwards the event to notification controller if configured
-func (r *BucketReconciler) event(ctx context.Context, bucket sourcev1.Bucket, severity, msg string) {
- log := logr.FromContext(ctx)
- if r.EventRecorder != nil {
- r.EventRecorder.Eventf(&bucket, "Normal", severity, msg)
- }
- if r.ExternalEventRecorder != nil {
- objRef, err := reference.GetReference(r.Scheme, &bucket)
- if err != nil {
- log.Error(err, "unable to send event")
- return
- }
-
- if err := r.ExternalEventRecorder.Eventf(*objRef, nil, severity, severity, msg); err != nil {
- log.Error(err, "unable to send event")
- return
+// garbageCollect performs a garbage collection for the given
+// v1beta1.Bucket. It removes all but the current artifact
+// except for when the deletion timestamp is set, which will result
+// in the removal of all artifacts for the resource.
+func (r *BucketReconciler) garbageCollect(obj *sourcev1.Bucket) error {
+ if !obj.DeletionTimestamp.IsZero() {
+ if err := r.Storage.RemoveAll(r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), "", "*")); err != nil {
+ return err
}
+ obj.Status.Artifact = nil
+ return nil
}
-}
-
-func (r *BucketReconciler) recordReadiness(ctx context.Context, bucket sourcev1.Bucket) {
- log := logr.FromContext(ctx)
- if r.MetricsRecorder == nil {
- return
- }
- objRef, err := reference.GetReference(r.Scheme, &bucket)
- if err != nil {
- log.Error(err, "unable to record readiness metric")
- return
- }
- if rc := apimeta.FindStatusCondition(bucket.Status.Conditions, meta.ReadyCondition); rc != nil {
- r.MetricsRecorder.RecordCondition(*objRef, *rc, !bucket.DeletionTimestamp.IsZero())
- } else {
- r.MetricsRecorder.RecordCondition(*objRef, metav1.Condition{
- Type: meta.ReadyCondition,
- Status: metav1.ConditionUnknown,
- }, !bucket.DeletionTimestamp.IsZero())
- }
-}
-
-func (r *BucketReconciler) recordSuspension(ctx context.Context, bucket sourcev1.Bucket) {
- if r.MetricsRecorder == nil {
- return
- }
- log := logr.FromContext(ctx)
-
- objRef, err := reference.GetReference(r.Scheme, &bucket)
- if err != nil {
- log.Error(err, "unable to record suspended metric")
- return
- }
-
- if !bucket.DeletionTimestamp.IsZero() {
- r.MetricsRecorder.RecordSuspend(*objRef, false)
- } else {
- r.MetricsRecorder.RecordSuspend(*objRef, bucket.Spec.Suspend)
- }
-}
-
-func (r *BucketReconciler) updateStatus(ctx context.Context, req ctrl.Request, newStatus sourcev1.BucketStatus) error {
- var bucket sourcev1.Bucket
- if err := r.Get(ctx, req.NamespacedName, &bucket); err != nil {
- return err
+ if obj.GetArtifact() != nil {
+ return r.Storage.RemoveAllButCurrent(*obj.GetArtifact())
}
-
- patch := client.MergeFrom(bucket.DeepCopy())
- bucket.Status = newStatus
-
- return r.Status().Patch(ctx, &bucket, patch)
+ return nil
}
diff --git a/controllers/bucket_controller_test.go b/controllers/bucket_controller_test.go
index 963d1f729..7a84d0782 100644
--- a/controllers/bucket_controller_test.go
+++ b/controllers/bucket_controller_test.go
@@ -17,12 +17,521 @@ limitations under the License.
package controllers
import (
- "io/ioutil"
+ "context"
+ "crypto/md5"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
"os"
+ "path"
"path/filepath"
+ "strings"
"testing"
+ "time"
+
+ "github.com/go-logr/logr"
+ . "github.com/onsi/gomega"
+ corev1 "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
+ "sigs.k8s.io/controller-runtime/pkg/log"
+
+ "github.com/fluxcd/pkg/apis/meta"
+ "github.com/fluxcd/pkg/runtime/conditions"
+
+ sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
)
+func TestBucketReconciler_Reconcile(t *testing.T) {
+ g := NewWithT(t)
+
+ s3Server := newS3Server("test-bucket")
+ s3Server.Objects = []*s3MockObject{
+ {
+ Key: "test.txt",
+ Content: []byte("test"),
+ ContentType: "text/plain",
+ LastModified: time.Now(),
+ },
+ }
+ s3Server.Start()
+ defer s3Server.Stop()
+
+ g.Expect(s3Server.HTTPAddress()).ToNot(BeEmpty())
+ u, err := url.Parse(s3Server.HTTPAddress())
+ g.Expect(err).NotTo(HaveOccurred())
+
+ secret := &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ GenerateName: "bucket-reconcile-",
+ Namespace: "default",
+ },
+ Data: map[string][]byte{
+ "accesskey": []byte("key"),
+ "secretkey": []byte("secret"),
+ },
+ }
+ g.Expect(env.Create(ctx, secret)).To(Succeed())
+ defer env.Delete(ctx, secret)
+
+ obj := &sourcev1.Bucket{
+ ObjectMeta: metav1.ObjectMeta{
+ GenerateName: "bucket-reconcile-",
+ Namespace: "default",
+ },
+ Spec: sourcev1.BucketSpec{
+ Provider: "generic",
+ BucketName: s3Server.BucketName,
+ Endpoint: u.Host,
+ Insecure: true,
+ Interval: metav1.Duration{Duration: interval},
+ Timeout: &metav1.Duration{Duration: timeout},
+ SecretRef: &meta.LocalObjectReference{
+ Name: secret.Name,
+ },
+ },
+ }
+ g.Expect(env.Create(ctx, obj)).To(Succeed())
+
+ key := client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace}
+
+ // Wait for finalizer to be set
+ g.Eventually(func() bool {
+ if err := env.Get(ctx, key, obj); err != nil {
+ return false
+ }
+ return len(obj.Finalizers) > 0
+ }, timeout).Should(BeTrue())
+
+ // Wait for Bucket to be Ready
+ g.Eventually(func() bool {
+ if err := env.Get(ctx, key, obj); err != nil {
+ return false
+ }
+
+ if !conditions.Has(obj, sourcev1.ArtifactAvailableCondition) ||
+ !conditions.Has(obj, sourcev1.SourceAvailableCondition) ||
+ !conditions.Has(obj, meta.ReadyCondition) ||
+ obj.Status.Artifact == nil {
+ return false
+ }
+
+ readyCondition := conditions.Get(obj, meta.ReadyCondition)
+
+ return readyCondition.Status == metav1.ConditionTrue &&
+ obj.Generation == readyCondition.ObservedGeneration
+ }, timeout).Should(BeTrue())
+
+ g.Expect(env.Delete(ctx, obj)).To(Succeed())
+
+ // Wait for Bucket to be deleted
+ g.Eventually(func() bool {
+ if err := env.Get(ctx, key, obj); err != nil {
+ return apierrors.IsNotFound(err)
+ }
+ return false
+ }, timeout).Should(BeTrue())
+}
+
+func TestBucketReconciler_reconcileStorage(t *testing.T) {
+ tests := []struct {
+ name string
+ beforeFunc func(obj *sourcev1.Bucket, storage *Storage) error
+ want ctrl.Result
+ wantErr bool
+ assertArtifact *sourcev1.Artifact
+ assertConditions []metav1.Condition
+ assertPaths []string
+ }{
+ {
+ name: "garbage collects",
+ beforeFunc: func(obj *sourcev1.Bucket, storage *Storage) error {
+ revisions := []string{"a", "b", "c"}
+ for n := range revisions {
+ v := revisions[n]
+ obj.Status.Artifact = &sourcev1.Artifact{
+ Path: fmt.Sprintf("/reconcile-storage/%s.txt", v),
+ Revision: v,
+ }
+ if err := storage.MkdirAll(*obj.Status.Artifact); err != nil {
+ return err
+ }
+ if err := storage.AtomicWriteFile(obj.Status.Artifact, strings.NewReader(v), 0644); err != nil {
+ return err
+ }
+ }
+ storage.SetArtifactURL(obj.Status.Artifact)
+ return nil
+ },
+ assertArtifact: &sourcev1.Artifact{
+ Path: "/reconcile-storage/c.txt",
+ Revision: "c",
+ Checksum: "84a516841ba77a5b4648de2cd0dfcb30ea46dbb4",
+ URL: storage.Hostname + "/reconcile-storage/c.txt",
+ },
+ assertPaths: []string{
+ "/reconcile-storage/c.txt",
+ "!/reconcile-storage/b.txt",
+ "!/reconcile-storage/a.txt",
+ },
+ },
+ {
+ name: "notices missing artifact in storage",
+ beforeFunc: func(obj *sourcev1.Bucket, storage *Storage) error {
+ obj.Status.Artifact = &sourcev1.Artifact{
+ Path: fmt.Sprintf("/reconcile-storage/invalid.txt"),
+ Revision: "d",
+ }
+ storage.SetArtifactURL(obj.Status.Artifact)
+ return nil
+ },
+ want: ctrl.Result{Requeue: true},
+ assertPaths: []string{
+ "!/reconcile-storage/invalid.txt",
+ },
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.ArtifactAvailableCondition, "NoArtifactFound", "No artifact for resource in storage"),
+ },
+ },
+ {
+ name: "updates hostname on diff from current",
+ beforeFunc: func(obj *sourcev1.Bucket, storage *Storage) error {
+ obj.Status.Artifact = &sourcev1.Artifact{
+ Path: fmt.Sprintf("/reconcile-storage/hostname.txt"),
+ Revision: "f",
+ Checksum: "971c419dd609331343dee105fffd0f4608dc0bf2",
+ URL: "http://outdated.com/reconcile-storage/hostname.txt",
+ }
+ if err := storage.MkdirAll(*obj.Status.Artifact); err != nil {
+ return err
+ }
+ if err := storage.AtomicWriteFile(obj.Status.Artifact, strings.NewReader("file"), 0644); err != nil {
+ return err
+ }
+ return nil
+ },
+ assertPaths: []string{
+ "/reconcile-storage/hostname.txt",
+ },
+ assertArtifact: &sourcev1.Artifact{
+ Path: "/reconcile-storage/hostname.txt",
+ Revision: "f",
+ Checksum: "971c419dd609331343dee105fffd0f4608dc0bf2",
+ URL: storage.Hostname + "/reconcile-storage/hostname.txt",
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ r := &BucketReconciler{
+ Storage: storage,
+ }
+
+ obj := &sourcev1.Bucket{
+ ObjectMeta: metav1.ObjectMeta{
+ GenerateName: "test-",
+ },
+ }
+ if tt.beforeFunc != nil {
+ g.Expect(tt.beforeFunc(obj, storage)).To(Succeed())
+ }
+
+ got, err := r.reconcileStorage(context.TODO(), obj)
+ g.Expect(err != nil).To(Equal(tt.wantErr))
+ g.Expect(got).To(Equal(tt.want))
+
+ g.Expect(obj.Status.Artifact).To(MatchArtifact(tt.assertArtifact))
+ if tt.assertArtifact != nil && tt.assertArtifact.URL != "" {
+ g.Expect(obj.Status.Artifact.URL).To(Equal(tt.assertArtifact.URL))
+ }
+ g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions))
+
+ for _, p := range tt.assertPaths {
+ absoluteP := filepath.Join(storage.BasePath, p)
+ if !strings.HasPrefix(p, "!") {
+ g.Expect(absoluteP).To(BeAnExistingFile())
+ continue
+ }
+ g.Expect(absoluteP).NotTo(BeAnExistingFile())
+ }
+ })
+ }
+}
+
+func TestBucketReconciler_reconcileSource(t *testing.T) {
+ tests := []struct {
+ name string
+ bucketName string
+ bucketObjects []*s3MockObject
+ middleware http.Handler
+ secret *corev1.Secret
+ beforeFunc func(obj *sourcev1.Bucket)
+ want ctrl.Result
+ wantErr bool
+ assertArtifact sourcev1.Artifact
+ assertConditions []metav1.Condition
+ }{
+ {
+ name: "reconciles source",
+ bucketName: "dummy",
+ bucketObjects: []*s3MockObject{
+ {
+ Key: "test.txt",
+ Content: []byte("test"),
+ ContentType: "text/plain",
+ LastModified: time.Now(),
+ },
+ },
+ assertArtifact: sourcev1.Artifact{
+ Path: "bucket/test-bucket/8f6e217935490b6283d2f257576e2f8674d70963.tar.gz",
+ Revision: "8f6e217935490b6283d2f257576e2f8674d70963",
+ },
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.SourceAvailableCondition, sourcev1.BucketOperationSucceedReason, "Downloaded 1 objects from bucket"),
+ },
+ },
+ // TODO(hidde): middleware for mock server
+ //{
+ // name: "authenticates using secretRef",
+ // bucketName: "dummy",
+ //},
+ {
+ name: "observes non-existing secretRef",
+ bucketName: "dummy",
+ beforeFunc: func(obj *sourcev1.Bucket) {
+ obj.Spec.SecretRef = &meta.LocalObjectReference{
+ Name: "dummy",
+ }
+ },
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceAvailableCondition, sourcev1.AuthenticationFailedReason, "Failed to get secret '/dummy': secrets \"dummy\" not found"),
+ },
+ },
+ {
+ name: "observes invalid secretRef",
+ bucketName: "dummy",
+ secret: &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "dummy",
+ },
+ },
+ beforeFunc: func(obj *sourcev1.Bucket) {
+ obj.Spec.SecretRef = &meta.LocalObjectReference{
+ Name: "dummy",
+ }
+ },
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceAvailableCondition, sourcev1.BucketOperationFailedReason, "Failed to construct S3 client: invalid \"dummy\" secret data: required fields"),
+ },
+ },
+ {
+ name: "observes non-existing bucket name",
+ bucketName: "dummy",
+ beforeFunc: func(obj *sourcev1.Bucket) {
+ obj.Spec.BucketName = "invalid"
+ },
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceAvailableCondition, sourcev1.BucketOperationFailedReason, "Bucket \"invalid\" does not exist"),
+ },
+ },
+ {
+ name: "transient bucket name API failure",
+ beforeFunc: func(obj *sourcev1.Bucket) {
+ obj.Spec.Endpoint = "transient.example.com"
+ obj.Spec.BucketName = "unavailable"
+ },
+ wantErr: true,
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceAvailableCondition, sourcev1.BucketOperationFailedReason, "Failed to verify existence of bucket \"unavailable\""),
+ },
+ },
+ {
+ // TODO(hidde): test the lesser happy paths
+ name: ".sourceignore",
+ bucketName: "dummy",
+ bucketObjects: []*s3MockObject{
+ {
+ Key: ".sourceignore",
+ Content: []byte("ignored/file.txt"),
+ ContentType: "text/plain",
+ LastModified: time.Now(),
+ },
+ {
+ Key: "ignored/file.txt",
+ Content: []byte("ignored/file.txt"),
+ ContentType: "text/plain",
+ LastModified: time.Now(),
+ },
+ {
+ Key: "included/file.txt",
+ Content: []byte("included/file.txt"),
+ ContentType: "text/plain",
+ LastModified: time.Now(),
+ },
+ },
+ assertArtifact: sourcev1.Artifact{
+ Path: "bucket/test-bucket/36d3a0fb2a71d0026e66af1495ff0879ad5ff54e.tar.gz",
+ Revision: "36d3a0fb2a71d0026e66af1495ff0879ad5ff54e",
+ },
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.SourceAvailableCondition, sourcev1.BucketOperationSucceedReason, "Downloaded 1 objects from bucket"),
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ builder := fakeclient.NewClientBuilder().WithScheme(env.Scheme())
+ if tt.secret != nil {
+ builder.WithObjects(tt.secret)
+ }
+ r := &BucketReconciler{
+ Client: builder.Build(),
+ Storage: storage,
+ }
+ tmpDir, err := os.MkdirTemp("", "reconcile-bucket-source-")
+ g.Expect(err).ToNot(HaveOccurred())
+ defer os.RemoveAll(tmpDir)
+
+ obj := &sourcev1.Bucket{
+ TypeMeta: metav1.TypeMeta{
+ Kind: sourcev1.BucketKind,
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-bucket",
+ },
+ Spec: sourcev1.BucketSpec{
+ Timeout: &metav1.Duration{Duration: timeout},
+ },
+ }
+
+ var server *s3MockServer
+ if tt.bucketName != "" {
+ server = newS3Server(tt.bucketName)
+ server.Objects = tt.bucketObjects
+ server.Start()
+ defer server.Stop()
+
+ g.Expect(server.HTTPAddress()).ToNot(BeEmpty())
+ u, err := url.Parse(server.HTTPAddress())
+ g.Expect(err).NotTo(HaveOccurred())
+
+ obj.Spec.BucketName = tt.bucketName
+ obj.Spec.Endpoint = u.Host
+ // TODO(hidde): also test TLS
+ obj.Spec.Insecure = true
+ }
+ if tt.beforeFunc != nil {
+ tt.beforeFunc(obj)
+ }
+
+ artifact := &sourcev1.Artifact{}
+ got, err := r.reconcileSource(context.TODO(), obj, artifact, tmpDir)
+ g.Expect(err != nil).To(Equal(tt.wantErr))
+ g.Expect(got).To(Equal(tt.want))
+
+ g.Expect(artifact).To(MatchArtifact(tt.assertArtifact.DeepCopy()))
+ g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions))
+ })
+ }
+}
+
+func TestBucketReconciler_reconcileArtifact(t *testing.T) {
+ tests := []struct {
+ name string
+ artifact sourcev1.Artifact
+ beforeFunc func(obj *sourcev1.Bucket, artifact sourcev1.Artifact, dir string)
+ want ctrl.Result
+ wantErr bool
+ assertConditions []metav1.Condition
+ }{
+ {
+ name: "artifact revision up-to-date",
+ artifact: sourcev1.Artifact{
+ Revision: "existing",
+ },
+ beforeFunc: func(obj *sourcev1.Bucket, artifact sourcev1.Artifact, dir string) {
+ obj.Status.Artifact = &artifact
+ },
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.ArtifactAvailableCondition, meta.SucceededReason, "Compressed source to artifact with revision 'existing'"),
+ },
+ },
+ {
+ name: "dir path deleted",
+ beforeFunc: func(obj *sourcev1.Bucket, artifact sourcev1.Artifact, dir string) {
+ _ = os.RemoveAll(dir)
+ },
+ wantErr: true,
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to stat source path"),
+ },
+ },
+ //{
+ // name: "dir path empty",
+ //},
+ //{
+ // name: "success",
+ // artifact: sourcev1.Artifact{
+ // Revision: "existing",
+ // },
+ // beforeFunc: func(obj *sourcev1.Bucket, artifact sourcev1.Artifact, dir string) {
+ // obj.Status.Artifact = &artifact
+ // },
+ // assertConditions: []metav1.Condition{
+ // *conditions.TrueCondition(sourcev1.ArtifactAvailableCondition, meta.SucceededReason, "Compressed source to artifact with revision 'existing'"),
+ // },
+ //},
+ //{
+ // name: "symlink",
+ //},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ tmpDir, err := os.MkdirTemp("", "reconcile-bucket-artifact-")
+ g.Expect(err).ToNot(HaveOccurred())
+ defer os.RemoveAll(tmpDir)
+
+ obj := &sourcev1.Bucket{
+ TypeMeta: metav1.TypeMeta{
+ Kind: sourcev1.BucketKind,
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-bucket",
+ },
+ Spec: sourcev1.BucketSpec{
+ Timeout: &metav1.Duration{Duration: timeout},
+ },
+ }
+
+ if tt.beforeFunc != nil {
+ tt.beforeFunc(obj, tt.artifact, tmpDir)
+ }
+
+ r := &BucketReconciler{
+ Storage: storage,
+ }
+
+ got, err := r.reconcileArtifact(logr.NewContext(ctx, log.NullLogger{}), obj, tt.artifact, tmpDir)
+ g.Expect(err != nil).To(Equal(tt.wantErr))
+ g.Expect(got).To(Equal(tt.want))
+
+ //g.Expect(artifact).To(MatchArtifact(tt.assertArtifact.DeepCopy()))
+ g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions))
+ })
+ }
+}
+
func TestBucketReconciler_checksum(t *testing.T) {
tests := []struct {
name string
@@ -51,7 +560,7 @@ func TestBucketReconciler_checksum(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- root, err := ioutil.TempDir("", "bucket-checksum-")
+ root, err := os.MkdirTemp("", "bucket-checksum-")
if err != nil {
t.Fatal(err)
}
@@ -71,13 +580,132 @@ func TestBucketReconciler_checksum(t *testing.T) {
}
}
+// helpers
+
func mockFile(root, path, content string) error {
filePath := filepath.Join(root, path)
if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
panic(err)
}
- if err := ioutil.WriteFile(filePath, []byte(content), 0644); err != nil {
+ if err := os.WriteFile(filePath, []byte(content), 0644); err != nil {
panic(err)
}
return nil
}
+
+type s3MockObject struct {
+ Key string
+ LastModified time.Time
+ ContentType string
+ Content []byte
+}
+
+type s3MockServer struct {
+ srv *httptest.Server
+ mux *http.ServeMux
+
+ BucketName string
+ Objects []*s3MockObject
+}
+
+func newS3Server(bucketName string) *s3MockServer {
+ s := &s3MockServer{BucketName: bucketName}
+ s.mux = http.NewServeMux()
+ s.mux.Handle(fmt.Sprintf("/%s/", s.BucketName), http.HandlerFunc(s.handler))
+
+ s.srv = httptest.NewUnstartedServer(s.mux)
+
+ return s
+}
+
+func (s *s3MockServer) Start() {
+ s.srv.Start()
+}
+
+func (s *s3MockServer) Stop() {
+ s.srv.Close()
+}
+
+func (s *s3MockServer) HTTPAddress() string {
+ return s.srv.URL
+}
+
+func (s *s3MockServer) handler(w http.ResponseWriter, r *http.Request) {
+ key := path.Base(r.URL.Path)
+
+ switch key {
+ case s.BucketName:
+ w.Header().Add("Content-Type", "application/xml")
+
+ if r.Method == http.MethodHead {
+ return
+ }
+
+ q := r.URL.Query()
+
+ if q["location"] != nil {
+ fmt.Fprint(w, `
+
+Europe
+ `)
+ return
+ }
+
+ contents := ""
+ for _, o := range s.Objects {
+ etag := md5.Sum(o.Content)
+ contents += fmt.Sprintf(`
+
+ %s
+ %s
+ %d
+ "%b"
+ STANDARD
+ `, o.Key, o.LastModified.UTC().Format(time.RFC3339), len(o.Content), etag)
+ }
+
+ fmt.Fprintf(w, `
+
+
+ %s
+
+
+ %d
+ 1000
+ false
+ %s
+
+ `, s.BucketName, len(s.Objects), contents)
+ default:
+ key, err := filepath.Rel("/"+s.BucketName, r.URL.Path)
+ if err != nil {
+ w.WriteHeader(500)
+ return
+ }
+
+ var found *s3MockObject
+ for _, o := range s.Objects {
+ if key == o.Key {
+ found = o
+ }
+ }
+ if found == nil {
+ w.WriteHeader(404)
+ return
+ }
+
+ etag := md5.Sum(found.Content)
+ lastModified := strings.Replace(found.LastModified.UTC().Format(time.RFC1123), "UTC", "GMT", 1)
+
+ w.Header().Add("Content-Type", found.ContentType)
+ w.Header().Add("Last-Modified", lastModified)
+ w.Header().Add("ETag", fmt.Sprintf("\"%b\"", etag))
+ w.Header().Add("Content-Length", fmt.Sprintf("%d", len(found.Content)))
+
+ if r.Method == http.MethodHead {
+ return
+ }
+
+ w.Write(found.Content)
+ }
+}
diff --git a/controllers/gitrepository_controller.go b/controllers/gitrepository_controller.go
index 6e4f6e704..a768ccc45 100644
--- a/controllers/gitrepository_controller.go
+++ b/controllers/gitrepository_controller.go
@@ -19,21 +19,20 @@ package controllers
import (
"context"
"fmt"
- "io/ioutil"
"os"
- "path/filepath"
"strings"
"time"
securejoin "github.com/cyphar/filepath-securejoin"
+ "github.com/fluxcd/pkg/runtime/conditions"
+ "github.com/fluxcd/pkg/runtime/patch"
+ "github.com/fluxcd/source-controller/pkg/sourceignore"
"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
- apimeta "k8s.io/apimachinery/pkg/api/meta"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
- kuberecorder "k8s.io/client-go/tools/record"
- "k8s.io/client-go/tools/reference"
+ kerrors "k8s.io/apimachinery/pkg/util/errors"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -42,14 +41,13 @@ import (
"sigs.k8s.io/controller-runtime/pkg/predicate"
"github.com/fluxcd/pkg/apis/meta"
+ helper "github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/events"
- "github.com/fluxcd/pkg/runtime/metrics"
"github.com/fluxcd/pkg/runtime/predicates"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
"github.com/fluxcd/source-controller/pkg/git"
"github.com/fluxcd/source-controller/pkg/git/strategy"
- "github.com/fluxcd/source-controller/pkg/sourceignore"
)
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=gitrepositories,verbs=get;list;watch;create;update;patch;delete
@@ -60,12 +58,12 @@ import (
// GitRepositoryReconciler reconciles a GitRepository object
type GitRepositoryReconciler struct {
client.Client
- requeueDependency time.Duration
- Scheme *runtime.Scheme
- Storage *Storage
- EventRecorder kuberecorder.EventRecorder
- ExternalEventRecorder *events.Recorder
- MetricsRecorder *metrics.Recorder
+ helper.Events
+ helper.Metrics
+
+ Storage *Storage
+
+ requeueDependency time.Duration
}
type GitRepositoryReconcilerOptions struct {
@@ -88,410 +86,457 @@ func (r *GitRepositoryReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, o
Complete(r)
}
-func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, retErr error) {
start := time.Now()
log := logr.FromContext(ctx)
- var repository sourcev1.GitRepository
- if err := r.Get(ctx, req.NamespacedName, &repository); err != nil {
+ // Fetch the GitRepository
+ obj := &sourcev1.GitRepository{}
+ if err := r.Get(ctx, req.NamespacedName, obj); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
// Record suspended status metric
- defer r.recordSuspension(ctx, repository)
+ r.RecordSuspend(ctx, obj, obj.Spec.Suspend)
- // Add our finalizer if it does not exist
- if !controllerutil.ContainsFinalizer(&repository, sourcev1.SourceFinalizer) {
- controllerutil.AddFinalizer(&repository, sourcev1.SourceFinalizer)
- if err := r.Update(ctx, &repository); err != nil {
- log.Error(err, "unable to register finalizer")
- return ctrl.Result{}, err
- }
+ // Return early if the object is suspended
+ if obj.Spec.Suspend {
+ log.Info("Reconciliation is suspended for this object")
+ return ctrl.Result{}, nil
}
- // Examine if the object is under deletion
- if !repository.ObjectMeta.DeletionTimestamp.IsZero() {
- return r.reconcileDelete(ctx, repository)
+ // Initialize the patch helper
+ patchHelper, err := patch.NewHelper(obj, r.Client)
+ if err != nil {
+ return ctrl.Result{}, err
}
- // Return early if the object is suspended.
- if repository.Spec.Suspend {
- log.Info("Reconciliation is suspended for this object")
- return ctrl.Result{}, nil
- }
+ // Always attempt to patch the object and status after each
+ // reconciliation
+ defer func() {
+ // Record the value of the reconciliation request, if any
+ if v, ok := meta.ReconcileAnnotationValue(obj.GetAnnotations()); ok {
+ obj.Status.SetLastHandledReconcileRequest(v)
+ }
- // check dependencies
- if len(repository.Spec.Include) > 0 {
- if err := r.checkDependencies(repository); err != nil {
- repository = sourcev1.GitRepositoryNotReady(repository, meta.DependencyNotReadyReason, err.Error())
- if err := r.updateStatus(ctx, req, repository.Status); err != nil {
- log.Error(err, "unable to update status for dependency not ready")
- return ctrl.Result{Requeue: true}, err
- }
- // we can't rely on exponential backoff because it will prolong the execution too much,
- // instead we requeue on a fix interval.
- msg := fmt.Sprintf("Dependencies do not meet ready condition, retrying in %s", r.requeueDependency.String())
- log.Info(msg)
- r.event(ctx, repository, events.EventSeverityInfo, msg)
- r.recordReadiness(ctx, repository)
- return ctrl.Result{RequeueAfter: r.requeueDependency}, nil
+ // We are no longer reconciling
+ conditions.Delete(obj, meta.ReconcilingCondition)
+
+ // Summarize Ready condition
+ conditions.SetSummary(obj,
+ meta.ReadyCondition,
+ conditions.WithConditions(
+ sourcev1.ArtifactAvailableCondition,
+ sourcev1.SourceVerifiedCondition,
+ sourcev1.SourceAvailableCondition,
+ ),
+ )
+
+ // Patch the object, ignoring conflicts on the conditions owned by
+ // this controller
+ patchOpts := []patch.Option{
+ patch.WithOwnedConditions{
+ Conditions: []string{
+ sourcev1.ArtifactAvailableCondition,
+ sourcev1.SourceVerifiedCondition,
+ sourcev1.SourceAvailableCondition,
+ meta.ReadyCondition,
+ meta.ReconcilingCondition,
+ meta.StalledCondition,
+ },
+ },
}
- log.Info("All dependencies area ready, proceeding with reconciliation")
- }
- // record reconciliation duration
- if r.MetricsRecorder != nil {
- objRef, err := reference.GetReference(r.Scheme, &repository)
- if err != nil {
- return ctrl.Result{}, err
+ // Determine if the resource is still being reconciled, or if
+ // it has stalled, and record this observation
+ if retErr == nil && (result.IsZero() || !result.Requeue) {
+ // We are no longer reconciling
+ conditions.Delete(obj, meta.ReconcilingCondition)
+
+ // We have now observed this generation
+ patchOpts = append(patchOpts, patch.WithStatusObservedGeneration{})
+
+ readyCondition := conditions.Get(obj, meta.ReadyCondition)
+ switch readyCondition.Status {
+ case metav1.ConditionFalse:
+ // As we are no longer reconciling and the end-state
+ // is not ready, the reconciliation has stalled
+ conditions.MarkStalled(obj, readyCondition.Reason, readyCondition.Message)
+ case metav1.ConditionTrue:
+ // As we are no longer reconciling and the end-state
+ // is ready, the reconciliation is no longer stalled
+ conditions.Delete(obj, meta.StalledCondition)
+ }
}
- defer r.MetricsRecorder.RecordDuration(*objRef, start)
- }
- // set initial status
- if resetRepository, ok := r.resetStatus(repository); ok {
- repository = resetRepository
- if err := r.updateStatus(ctx, req, repository.Status); err != nil {
- log.Error(err, "unable to update status")
- return ctrl.Result{Requeue: true}, err
+ // Finally, patch the resource
+ if err := patchHelper.Patch(ctx, obj, patchOpts...); err != nil {
+ retErr = kerrors.NewAggregate([]error{retErr, err})
}
- r.recordReadiness(ctx, repository)
- }
- // record the value of the reconciliation request, if any
- // TODO(hidde): would be better to defer this in combination with
- // always patching the status sub-resource after a reconciliation.
- if v, ok := meta.ReconcileAnnotationValue(repository.GetAnnotations()); ok {
- repository.Status.SetLastHandledReconcileRequest(v)
+ // Always record readiness and duration metrics
+ r.Metrics.RecordReadiness(ctx, obj)
+ r.Metrics.RecordDuration(ctx, obj, start)
+ }()
+
+ // Add finalizer first if not exist to avoid the race condition
+ // between init and delete
+ if !controllerutil.ContainsFinalizer(obj, sourcev1.SourceFinalizer) {
+ controllerutil.AddFinalizer(obj, sourcev1.SourceFinalizer)
+ return ctrl.Result{Requeue: true}, nil
}
- // purge old artifacts from storage
- if err := r.gc(repository); err != nil {
- log.Error(err, "unable to purge old artifacts")
+ // Examine if the object is under deletion
+ if !obj.ObjectMeta.DeletionTimestamp.IsZero() {
+ return r.reconcileDelete(ctx, obj)
}
- // reconcile repository by pulling the latest Git commit
- reconciledRepository, reconcileErr := r.reconcile(ctx, *repository.DeepCopy())
+ // Reconcile actual object
+ return r.reconcile(ctx, obj)
+}
+
+func (r *GitRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.GitRepository) (ctrl.Result, error) {
+ // Mark the resource as under reconciliation
+ conditions.MarkReconciling(obj, "Reconciling", "")
+
+ // Reconcile the storage data
+ if result, err := r.reconcileStorage(ctx, obj); err != nil {
+ return result, err
+ }
- // update status with the reconciliation result
- if err := r.updateStatus(ctx, req, reconciledRepository.Status); err != nil {
- log.Error(err, "unable to update status")
- return ctrl.Result{Requeue: true}, err
+ // Create temp dir for Git clone
+ tmpDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s-%s-", obj.Kind, obj.Namespace, obj.Name))
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to create temporary directory: %s", err)
+ return ctrl.Result{}, err
}
+ defer os.RemoveAll(tmpDir)
- // if reconciliation failed, record the failure and requeue immediately
- if reconcileErr != nil {
- r.event(ctx, reconciledRepository, events.EventSeverityError, reconcileErr.Error())
- r.recordReadiness(ctx, reconciledRepository)
- return ctrl.Result{Requeue: true}, reconcileErr
+ // Reconcile the source from upstream
+ var artifact sourcev1.Artifact
+ if result, err := r.reconcileSource(ctx, obj, &artifact, tmpDir); err != nil || conditions.IsFalse(obj, sourcev1.SourceAvailableCondition) {
+ return result, err
}
- // emit revision change event
- if repository.Status.Artifact == nil || reconciledRepository.Status.Artifact.Revision != repository.Status.Artifact.Revision {
- r.event(ctx, reconciledRepository, events.EventSeverityInfo, sourcev1.GitRepositoryReadyMessage(reconciledRepository))
+ // Reconcile includes from the storage
+ var includes artifactSet
+ if result, err := r.reconcileInclude(ctx, obj, includes, tmpDir); err != nil || len(includes) != len(obj.Spec.Include) {
+ return result, err
}
- r.recordReadiness(ctx, reconciledRepository)
- log.Info(fmt.Sprintf("Reconciliation finished in %s, next run in %s",
- time.Now().Sub(start).String(),
- repository.GetInterval().Duration.String(),
- ))
+ // Reconcile the artifact to storage
+ if result, err := r.reconcileArtifact(ctx, obj, artifact, includes, tmpDir); err != nil {
+ return result, err
+ }
- return ctrl.Result{RequeueAfter: repository.GetInterval().Duration}, nil
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
}
-func (r *GitRepositoryReconciler) checkDependencies(repository sourcev1.GitRepository) error {
- for _, d := range repository.Spec.Include {
- dName := types.NamespacedName{Name: d.GitRepositoryRef.Name, Namespace: repository.Namespace}
- var gr sourcev1.GitRepository
- err := r.Get(context.Background(), dName, &gr)
- if err != nil {
- return fmt.Errorf("unable to get '%s' dependency: %w", dName, err)
- }
+// reconcileStorage reconciles the storage data for the given object
+// by garbage collecting previous advertised artifact(s) from storage,
+// observing if the artifact in the status still exists, and
+// ensuring the URLs are up-to-date with the current hostname
+// configuration.
+func (r *GitRepositoryReconciler) reconcileStorage(ctx context.Context, obj *sourcev1.GitRepository) (ctrl.Result, error) {
+ // Garbage collect previous advertised artifact(s) from storage
+ if err := r.garbageCollect(obj); err != nil {
+ r.Events.Eventf(ctx, obj, events.EventSeverityError, "GarbageCollectionFailed", "Garbage collection failed: %s", err)
+ }
- if len(gr.Status.Conditions) == 0 || gr.Generation != gr.Status.ObservedGeneration {
- return fmt.Errorf("dependency '%s' is not ready", dName)
- }
+ // Determine if the advertised artifact is still in storage
+ if artifact := obj.GetArtifact(); artifact != nil && !r.Storage.ArtifactExist(*artifact) {
+ obj.Status.Artifact = nil
+ obj.Status.URL = ""
+ }
- if !apimeta.IsStatusConditionTrue(gr.Status.Conditions, meta.ReadyCondition) {
- return fmt.Errorf("dependency '%s' is not ready", dName)
- }
+ // Record that we do not have an artifact
+ if obj.GetArtifact() == nil {
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, "NoArtifactFound", "No artifact for resource in storage")
+ return ctrl.Result{Requeue: true}, nil
}
- return nil
+ // Always update URLs to ensure hostname is up-to-date
+ r.Storage.SetArtifactURL(obj.GetArtifact())
+ obj.Status.URL = r.Storage.SetHostname(obj.Status.URL)
+
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
}
-func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sourcev1.GitRepository) (sourcev1.GitRepository, error) {
- // create tmp dir for the Git clone
- tmpGit, err := ioutil.TempDir("", repository.Name)
- if err != nil {
- err = fmt.Errorf("tmp dir error: %w", err)
- return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- defer os.RemoveAll(tmpGit)
+// reconcileSource reconciles the Git repository from upstream to the
+// given directory path while using the information on the object to
+// determine authentication and checkout strategies.
+// On a successful checkout of HEAD the artifact metadata the given
+// pointer is set to a new artifact.
+func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, obj *sourcev1.GitRepository, artifact *sourcev1.Artifact, dir string) (ctrl.Result, error) {
+ log := logr.FromContext(ctx)
- // determine auth method
+ // Configure authentication strategy to access the source
auth := &git.Auth{}
- if repository.Spec.SecretRef != nil {
- authStrategy, err := strategy.AuthSecretStrategyForURL(
- repository.Spec.URL,
- git.CheckoutOptions{
- GitImplementation: repository.Spec.GitImplementation,
- RecurseSubmodules: repository.Spec.RecurseSubmodules,
- })
+ if obj.Spec.SecretRef != nil {
+ // Determine the auth strategy
+ authStrategy, err := strategy.AuthSecretStrategyForURL(obj.Spec.URL, git.CheckoutOptions{
+ GitImplementation: obj.Spec.GitImplementation,
+ RecurseSubmodules: obj.Spec.RecurseSubmodules,
+ })
if err != nil {
- return sourcev1.GitRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
+ log.Error(err, "failed to get auth strategy")
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.AuthenticationFailedReason, "Failed to get auth strategy: %s", err)
+ // Do not return err as recovery without changes is impossible
+ return ctrl.Result{}, nil
}
+ // Attempt to retrieve secret
name := types.NamespacedName{
- Namespace: repository.GetNamespace(),
- Name: repository.Spec.SecretRef.Name,
+ Namespace: obj.GetNamespace(),
+ Name: obj.Spec.SecretRef.Name,
}
-
var secret corev1.Secret
- err = r.Client.Get(ctx, name, &secret)
- if err != nil {
- err = fmt.Errorf("auth secret error: %w", err)
- return sourcev1.GitRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
+ if err = r.Client.Get(ctx, name, &secret); err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.AuthenticationFailedReason, "Failed to get secret %s: %s", name.String(), err.Error())
+ r.Events.Event(ctx, obj, events.EventSeverityError, sourcev1.AuthenticationFailedReason, conditions.Get(obj, sourcev1.SourceAvailableCondition).Message)
+ // Return transient errors but wait for next interval on not found
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, client.IgnoreNotFound(err)
}
+ // Configure strategy with secret
auth, err = authStrategy.Method(secret)
if err != nil {
- err = fmt.Errorf("auth error: %w", err)
- return sourcev1.GitRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.AuthenticationFailedReason, "Failed to configure auth strategy: %s", err)
+ r.Events.Event(ctx, obj, events.EventSeverityError, sourcev1.AuthenticationFailedReason, conditions.Get(obj, sourcev1.SourceAvailableCondition).Message)
+ // Return err as the content of the secret may change
+ return ctrl.Result{}, err
}
}
- checkoutStrategy, err := strategy.CheckoutStrategyForRef(
- repository.Spec.Reference,
- git.CheckoutOptions{
- GitImplementation: repository.Spec.GitImplementation,
- RecurseSubmodules: repository.Spec.RecurseSubmodules,
- },
- )
+ // Configure checkout strategy
+ checkoutStrategy, err := strategy.CheckoutStrategyForRef(obj.Spec.Reference, git.CheckoutOptions{
+ GitImplementation: obj.Spec.GitImplementation,
+ RecurseSubmodules: obj.Spec.RecurseSubmodules,
+ })
if err != nil {
- return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.GitOperationFailedReason, "Failed to configure checkout strategy: %s", err)
+ r.Events.Event(ctx, obj, events.EventSeverityError, sourcev1.AuthenticationFailedReason, conditions.Get(obj, sourcev1.SourceAvailableCondition).Message)
+ // Do not return err as recovery without changes is impossible
+ return ctrl.Result{}, nil
}
- gitCtx, cancel := context.WithTimeout(ctx, repository.Spec.Timeout.Duration)
+ // Checkout HEAD of commit referenced in object
+ gitCtx, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration)
defer cancel()
-
- commit, revision, err := checkoutStrategy.Checkout(gitCtx, tmpGit, repository.Spec.URL, auth)
+ commit, revision, err := checkoutStrategy.Checkout(gitCtx, dir, obj.Spec.URL, auth)
if err != nil {
- return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.GitOperationFailedReason, "Failed to checkout and determine HEAD revision: %s", err)
+ r.Events.Eventf(ctx, obj, events.EventSeverityError, "GitCheckoutFailed", conditions.Get(obj, sourcev1.SourceAvailableCondition).Message)
+ // Coin flip on transient or persistent error, requeue
+ // TODO(hidde): likely better to detect the err type
+ return ctrl.Result{}, err
}
- artifact := r.Storage.NewArtifactFor(repository.Kind, repository.GetObjectMeta(), revision, fmt.Sprintf("%s.tar.gz", commit.Hash()))
-
- // copy all included repository into the artifact
- includedArtifacts := []*sourcev1.Artifact{}
- for _, incl := range repository.Spec.Include {
- dName := types.NamespacedName{Name: incl.GitRepositoryRef.Name, Namespace: repository.Namespace}
- var gr sourcev1.GitRepository
- err := r.Get(context.Background(), dName, &gr)
- if err != nil {
- return sourcev1.GitRepositoryNotReady(repository, meta.DependencyNotReadyReason, err.Error()), err
- }
- includedArtifacts = append(includedArtifacts, gr.GetArtifact())
+ // Verify commit signature
+ if result, err := r.verifyCommitSignature(ctx, obj, commit); err != nil || conditions.IsFalse(obj, sourcev1.SourceVerifiedCondition) {
+ return result, err
}
- // return early on unchanged revision and unchanged included repositories
- if apimeta.IsStatusConditionTrue(repository.Status.Conditions, meta.ReadyCondition) && repository.GetArtifact().HasRevision(artifact.Revision) && !hasArtifactUpdated(repository.Status.IncludedArtifacts, includedArtifacts) {
- if artifact.URL != repository.GetArtifact().URL {
- r.Storage.SetArtifactURL(repository.GetArtifact())
- repository.Status.URL = r.Storage.SetHostname(repository.Status.URL)
- }
- return repository, nil
- }
+ // Create potential new artifact
+ *artifact = r.Storage.NewArtifactFor(obj.Kind, obj, revision, fmt.Sprintf("%s.tar.gz", commit.Hash()))
+ conditions.MarkTrue(obj, sourcev1.SourceAvailableCondition, sourcev1.GitOperationSucceedReason, "Checked out revision %s", revision)
- // verify PGP signature
- if repository.Spec.Verification != nil {
- publicKeySecret := types.NamespacedName{
- Namespace: repository.Namespace,
- Name: repository.Spec.Verification.SecretRef.Name,
- }
- var secret corev1.Secret
- if err := r.Client.Get(ctx, publicKeySecret, &secret); err != nil {
- err = fmt.Errorf("PGP public keys secret error: %w", err)
- return sourcev1.GitRepositoryNotReady(repository, sourcev1.VerificationFailedReason, err.Error()), err
- }
-
- err := commit.Verify(secret)
- if err != nil {
- return sourcev1.GitRepositoryNotReady(repository, sourcev1.VerificationFailedReason, err.Error()), err
- }
- }
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+}
- // create artifact dir
- err = r.Storage.MkdirAll(artifact)
- if err != nil {
- err = fmt.Errorf("mkdir dir error: %w", err)
- return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
+// reconcileArtifact reconciles the Git checkout and includes in the
+// given directory path to the artifact storage by archiving the
+// directory while taking into account the ignore patterns in the
+// directory and object.
+// On a successful archive, the artifact and includes in the status of
+// the given object are set, and the symlink in the storage is updated
+// to its path.
+func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.GitRepository, artifact sourcev1.Artifact, includes artifactSet, dir string) (ctrl.Result, error) {
+ // The artifact is up-to-date
+ if obj.GetArtifact().HasRevision(artifact.Revision) && !includes.Diff(obj.Status.IncludedArtifacts) {
+ logr.FromContext(ctx).Info("Artifact is up-to-date")
+ conditions.MarkTrue(obj, sourcev1.ArtifactAvailableCondition, "ArchivedArtifact", "Artifact revision %s", artifact.Revision)
+ return ctrl.Result{RequeueAfter: obj.GetInterval().Duration}, nil
+ }
+
+ // Ensure target path exists and is a directory
+ if f, err := os.Stat(dir); err != nil {
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to stat source path: %s", err.Error())
+ return ctrl.Result{}, err
+ } else if !f.IsDir() {
+ err = fmt.Errorf("source path %q is not a directory", dir)
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to reconcile artifact: %s", err.Error())
+ return ctrl.Result{}, err
}
- for i, incl := range repository.Spec.Include {
- toPath, err := securejoin.SecureJoin(tmpGit, incl.GetToPath())
- if err != nil {
- return sourcev1.GitRepositoryNotReady(repository, meta.DependencyNotReadyReason, err.Error()), err
- }
- err = r.Storage.CopyToPath(includedArtifacts[i], incl.GetFromPath(), toPath)
- if err != nil {
- return sourcev1.GitRepositoryNotReady(repository, meta.DependencyNotReadyReason, err.Error()), err
- }
+ // Ensure artifact directory exists and acquire lock
+ if err := r.Storage.MkdirAll(artifact); err != nil {
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to create directory: %s", err)
+ return ctrl.Result{}, err
}
-
- // acquire lock
unlock, err := r.Storage.Lock(artifact)
if err != nil {
- err = fmt.Errorf("unable to acquire lock: %w", err)
- return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to acquire lock: %s", err)
+ return ctrl.Result{}, err
}
defer unlock()
- // archive artifact and check integrity
- ignoreDomain := strings.Split(tmpGit, string(filepath.Separator))
- ps, err := sourceignore.LoadIgnorePatterns(tmpGit, ignoreDomain)
+ // Load ignore rules for archiving
+ ps, err := sourceignore.LoadIgnorePatterns(dir, nil)
if err != nil {
- err = fmt.Errorf(".sourceignore error: %w", err)
- return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- if repository.Spec.Ignore != nil {
- ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*repository.Spec.Ignore), ignoreDomain)...)
- }
- if err := r.Storage.Archive(&artifact, tmpGit, SourceIgnoreFilter(ps, ignoreDomain)); err != nil {
- err = fmt.Errorf("storage archive error: %w", err)
- return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, "SourceIgnoreError", "Failed to load source ignore patterns: %s", err)
+ return ctrl.Result{}, err
}
-
- // update latest symlink
- url, err := r.Storage.Symlink(artifact, "latest.tar.gz")
- if err != nil {
- err = fmt.Errorf("storage symlink error: %w", err)
- return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
+ if obj.Spec.Ignore != nil {
+ ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*obj.Spec.Ignore), nil)...)
}
- message := fmt.Sprintf("Fetched revision: %s", artifact.Revision)
- return sourcev1.GitRepositoryReady(repository, artifact, includedArtifacts, url, sourcev1.GitOperationSucceedReason, message), nil
-}
-
-func (r *GitRepositoryReconciler) reconcileDelete(ctx context.Context, repository sourcev1.GitRepository) (ctrl.Result, error) {
- if err := r.gc(repository); err != nil {
- r.event(ctx, repository, events.EventSeverityError,
- fmt.Sprintf("garbage collection for deleted resource failed: %s", err.Error()))
- // Return the error so we retry the failed garbage collection
+ // Archive directory to storage
+ if err := r.Storage.Archive(&artifact, dir, SourceIgnoreFilter(ps, nil)); err != nil {
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Unable to archive artifact to storage: %s", err)
return ctrl.Result{}, err
}
- // Record deleted status
- r.recordReadiness(ctx, repository)
+ // Record it on the object
+ obj.Status.Artifact = artifact.DeepCopy()
+ obj.Status.IncludedArtifacts = includes
+ conditions.MarkTrue(obj, sourcev1.ArtifactAvailableCondition, "ArchivedArtifact", "Compressed source to artifact with revision %s", artifact.Revision)
+ r.Events.EventWithMetaf(ctx, obj, map[string]string{
+ "revision": obj.GetArtifact().Revision,
+ }, events.EventSeverityInfo, sourcev1.GitOperationSucceedReason, conditions.Get(obj, sourcev1.ArtifactAvailableCondition).Message)
- // Remove our finalizer from the list and update it
- controllerutil.RemoveFinalizer(&repository, sourcev1.SourceFinalizer)
- if err := r.Update(ctx, &repository); err != nil {
- return ctrl.Result{}, err
+ // Update symlink on a "best effort" basis
+ url, err := r.Storage.Symlink(artifact, "latest.tar.gz")
+ if err != nil {
+ r.Events.Eventf(ctx, obj, events.EventSeverityError, sourcev1.StorageOperationFailedReason, "Failed to update status URL symlink: %s", err)
+ }
+ if url != "" {
+ obj.Status.URL = url
}
- // Stop reconciliation as the object is being deleted
- return ctrl.Result{}, nil
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
}
-// resetStatus returns a modified v1beta1.GitRepository and a boolean indicating
-// if the status field has been reset.
-func (r *GitRepositoryReconciler) resetStatus(repository sourcev1.GitRepository) (sourcev1.GitRepository, bool) {
- // We do not have an artifact, or it does no longer exist
- if repository.GetArtifact() == nil || !r.Storage.ArtifactExist(*repository.GetArtifact()) {
- repository = sourcev1.GitRepositoryProgressing(repository)
- repository.Status.Artifact = nil
- return repository, true
- }
- if repository.Generation != repository.Status.ObservedGeneration {
- return sourcev1.GitRepositoryProgressing(repository), true
- }
- return repository, false
-}
+// reconcileInclude reconciles the declared includes from the object
+// by copying their artifact (sub)contents to the declared paths in the
+// given directory.
+// It returns early if an object can not be found, or does not have an
+// artifact.
+// If all includes can be found, an aggregation of all their Ready
+// statuses is recorded in a condition on the given object.
+func (r *GitRepositoryReconciler) reconcileInclude(ctx context.Context, obj *sourcev1.GitRepository, artifacts artifactSet, dir string) (ctrl.Result, error) {
+ includes := make([]conditions.Getter, len(obj.Spec.Include))
+ artifacts = make(artifactSet, len(obj.Spec.Include))
+
+ for i, incl := range obj.Spec.Include {
+ dep := &sourcev1.GitRepository{}
+ if err := r.Get(ctx, types.NamespacedName{Namespace: obj.Namespace, Name: incl.GitRepositoryRef.Name}, dep); err != nil {
+ if apierrors.IsNotFound(err) {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, "IncludeNotFound", "Could not find resource for include %q: %s", incl.GitRepositoryRef.Name, err.Error())
+ }
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, client.IgnoreNotFound(err)
+ }
-// gc performs a garbage collection for the given v1beta1.GitRepository.
-// It removes all but the current artifact except for when the
-// deletion timestamp is set, which will result in the removal of
-// all artifacts for the resource.
-func (r *GitRepositoryReconciler) gc(repository sourcev1.GitRepository) error {
- if !repository.DeletionTimestamp.IsZero() {
- return r.Storage.RemoveAll(r.Storage.NewArtifactFor(repository.Kind, repository.GetObjectMeta(), "", "*"))
- }
- if repository.GetArtifact() != nil {
- return r.Storage.RemoveAllButCurrent(*repository.GetArtifact())
- }
- return nil
-}
+ // Confirm include has an artifact
+ if dep.GetArtifact() == nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, "IncludeUnavailable", "No artifact available for include %q", incl.GitRepositoryRef.Name)
+ return ctrl.Result{RequeueAfter: r.requeueDependency}, nil
+ }
-// event emits a Kubernetes event and forwards the event to notification controller if configured
-func (r *GitRepositoryReconciler) event(ctx context.Context, repository sourcev1.GitRepository, severity, msg string) {
- log := logr.FromContext(ctx)
+ includes[i] = dep.DeepCopy()
- if r.EventRecorder != nil {
- r.EventRecorder.Eventf(&repository, "Normal", severity, msg)
- }
- if r.ExternalEventRecorder != nil {
- objRef, err := reference.GetReference(r.Scheme, &repository)
+ // Copy artifact (sub)contents to configured directory
+ toPath, err := securejoin.SecureJoin(dir, incl.GetToPath())
if err != nil {
- log.Error(err, "unable to send event")
- return
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, "IncludeFailure", "Failed to calculate path for include %q: %s", incl.GitRepositoryRef.Name, err.Error())
+ return ctrl.Result{}, err
}
-
- if err := r.ExternalEventRecorder.Eventf(*objRef, nil, severity, severity, msg); err != nil {
- log.Error(err, "unable to send event")
- return
+ if err = r.Storage.CopyToPath(dep.GetArtifact(), incl.GetFromPath(), toPath); err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, "IncludeCopyFailure", "Failed to copy %q include from %s to %s: %s", incl.GitRepositoryRef.Name, incl.GetFromPath(), incl.GetToPath(), err.Error())
+ return ctrl.Result{}, err
}
+
+ artifacts[i] = dep.GetArtifact().DeepCopy()
}
+
+ // Record an aggregation of all includes Stalled or Ready state to
+ // the object condition
+ conditions.SetAggregate(obj, sourcev1.SourceAvailableCondition, includes,
+ conditions.WithConditions(meta.StalledCondition, meta.ReadyCondition),
+ conditions.WithNegativePolarityConditions(meta.StalledCondition),
+ conditions.WithSourceRefIf(meta.StalledCondition),
+ conditions.WithCounter(),
+ conditions.WithCounterIfOnly(meta.ReadyCondition))
+
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
}
-func (r *GitRepositoryReconciler) recordReadiness(ctx context.Context, repository sourcev1.GitRepository) {
- log := logr.FromContext(ctx)
- if r.MetricsRecorder == nil {
- return
- }
- objRef, err := reference.GetReference(r.Scheme, &repository)
- if err != nil {
- log.Error(err, "unable to record readiness metric")
- return
- }
- if rc := apimeta.FindStatusCondition(repository.Status.Conditions, meta.ReadyCondition); rc != nil {
- r.MetricsRecorder.RecordCondition(*objRef, *rc, !repository.DeletionTimestamp.IsZero())
- } else {
- r.MetricsRecorder.RecordCondition(*objRef, metav1.Condition{
- Type: meta.ReadyCondition,
- Status: metav1.ConditionUnknown,
- }, !repository.DeletionTimestamp.IsZero())
+// reconcileDelete reconciles the delete of an object by garbage
+// collecting all artifacts for the object in the artifact storage,
+// if successful, the finalizer is removed from the object.
+func (r *GitRepositoryReconciler) reconcileDelete(ctx context.Context, obj *sourcev1.GitRepository) (ctrl.Result, error) {
+ // Garbage collect the resource's artifacts
+ if err := r.garbageCollect(obj); err != nil {
+ r.Events.Eventf(ctx, obj, events.EventSeverityError, "GarbageCollectionFailed", "Garbage collection for deleted resource failed: %s", err)
+ // Return the error so we retry the failed garbage collection
+ return ctrl.Result{}, err
}
+
+ // Remove our finalizer from the list
+ controllerutil.RemoveFinalizer(obj, sourcev1.SourceFinalizer)
+
+ // Stop reconciliation as the object is being deleted
+ return ctrl.Result{}, nil
}
-func (r *GitRepositoryReconciler) recordSuspension(ctx context.Context, gitrepository sourcev1.GitRepository) {
- if r.MetricsRecorder == nil {
- return
+// verifyCommitSignature verifies the signature of the given commit if
+// a verification mode is configured on the object.
+func (r *GitRepositoryReconciler) verifyCommitSignature(ctx context.Context, obj *sourcev1.GitRepository, commit git.Commit) (ctrl.Result, error) {
+ // Check if there is a commit verification is configured,
+ // and remove old observation if there is none
+ if obj.Spec.Verification == nil || obj.Spec.Verification.Mode == "" {
+ conditions.Delete(obj, sourcev1.SourceVerifiedCondition)
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
}
- log := logr.FromContext(ctx)
- objRef, err := reference.GetReference(r.Scheme, &gitrepository)
- if err != nil {
- log.Error(err, "unable to record suspended metric")
- return
+ // Get secret with GPG data
+ publicKeySecret := types.NamespacedName{
+ Namespace: obj.Namespace,
+ Name: obj.Spec.Verification.SecretRef.Name,
}
-
- if !gitrepository.DeletionTimestamp.IsZero() {
- r.MetricsRecorder.RecordSuspend(*objRef, false)
- } else {
- r.MetricsRecorder.RecordSuspend(*objRef, gitrepository.Spec.Suspend)
+ var secret corev1.Secret
+ if err := r.Client.Get(ctx, publicKeySecret, &secret); err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceVerifiedCondition, "FailedToGetSecret", "PGP public keys secret error: %s", err)
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, client.IgnoreNotFound(err)
}
-}
-func (r *GitRepositoryReconciler) updateStatus(ctx context.Context, req ctrl.Request, newStatus sourcev1.GitRepositoryStatus) error {
- var repository sourcev1.GitRepository
- if err := r.Get(ctx, req.NamespacedName, &repository); err != nil {
- return err
+ // Verify commit with GPG data from secret
+ if err := commit.Verify(secret); err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceVerifiedCondition, "InvalidCommitSignature", "Commit signature verification failed: %s", err)
+ // We will not be able to recover from this error but HEAD
+ // may change in the future
+ logr.FromContext(ctx).Error(err, "PGP commit verification failed")
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
}
+ conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, "ValidCommitSignature", "Verified signature of commit %q", commit.Hash())
- patch := client.MergeFrom(repository.DeepCopy())
- repository.Status = newStatus
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+}
- return r.Status().Patch(ctx, &repository, patch)
+// garbageCollect performs a garbage collection for the given
+// v1beta1.GitRepository. It removes all but the current artifact
+// except for when the deletion timestamp is set, which will result
+// in the removal of all artifacts for the resource.
+func (r *GitRepositoryReconciler) garbageCollect(obj *sourcev1.GitRepository) error {
+ if !obj.DeletionTimestamp.IsZero() {
+ if err := r.Storage.RemoveAll(r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), "", "*")); err != nil {
+ return err
+ }
+ obj.Status.Artifact = nil
+ return nil
+ }
+ if obj.GetArtifact() != nil {
+ return r.Storage.RemoveAllButCurrent(*obj.GetArtifact())
+ }
+ return nil
}
diff --git a/controllers/gitrepository_controller_test.go b/controllers/gitrepository_controller_test.go
index 5be6c2ffd..fdb07fcd9 100644
--- a/controllers/gitrepository_controller_test.go
+++ b/controllers/gitrepository_controller_test.go
@@ -17,757 +17,1132 @@ limitations under the License.
package controllers
import (
- "context"
- "crypto/tls"
"fmt"
- "io/ioutil"
- "net/http"
"net/url"
"os"
-
- "os/exec"
- "path"
"path/filepath"
-
"strings"
+ "testing"
"time"
"github.com/go-git/go-billy/v5/memfs"
- "github.com/go-git/go-git/v5"
+ gogit "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
-
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
- "github.com/go-git/go-git/v5/plumbing/transport/client"
- httptransport "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/storage/memory"
- . "github.com/onsi/ginkgo"
-
- . "github.com/onsi/ginkgo/extensions/table"
+ "github.com/go-logr/logr"
. "github.com/onsi/gomega"
+ sshtestdata "golang.org/x/crypto/ssh/testdata"
corev1 "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/types"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/utils/pointer"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+ "sigs.k8s.io/controller-runtime/pkg/log"
"github.com/fluxcd/pkg/apis/meta"
"github.com/fluxcd/pkg/gittestserver"
- "github.com/fluxcd/pkg/untar"
+ "github.com/fluxcd/pkg/runtime/conditions"
+ "github.com/fluxcd/pkg/ssh"
+ "github.com/fluxcd/pkg/testserver"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+ "github.com/fluxcd/source-controller/pkg/git"
+ "github.com/fluxcd/source-controller/pkg/git/fake"
)
-var _ = Describe("GitRepositoryReconciler", func() {
+var (
+ testGitImplementations = []string{sourcev1.GoGitImplementation, sourcev1.LibGit2Implementation}
+)
- const (
- timeout = time.Second * 30
- interval = time.Second * 1
- indexInterval = time.Second * 1
- )
+func TestGitRepositoryReconciler_Reconcile(t *testing.T) {
+ g := NewWithT(t)
- Context("GitRepository", func() {
- var (
- namespace *corev1.Namespace
- gitServer *gittestserver.GitServer
- err error
- )
+ obj := &sourcev1.GitRepository{
+ ObjectMeta: metav1.ObjectMeta{
+ GenerateName: "gitrepository-reconcile-",
+ Namespace: "default",
+ },
+ Spec: sourcev1.GitRepositorySpec{
+ URL: "https://github.com/stefanprodan/podinfo.git",
+ },
+ }
+ g.Expect(env.Create(ctx, obj)).To(Succeed())
- BeforeEach(func() {
- namespace = &corev1.Namespace{
- ObjectMeta: metav1.ObjectMeta{Name: "git-repository-test" + randStringRunes(5)},
- }
- err = k8sClient.Create(context.Background(), namespace)
- Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
+ key := client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace}
+
+ // Wait for finalizer to be set
+ g.Eventually(func() bool {
+ if err := env.Get(ctx, key, obj); err != nil {
+ return false
+ }
+ return len(obj.Finalizers) > 0
+ }, timeout).Should(BeTrue())
+
+ // Wait for GitRepository to be Ready
+ g.Eventually(func() bool {
+ if err := env.Get(ctx, key, obj); err != nil {
+ return false
+ }
+
+ if !conditions.Has(obj, sourcev1.ArtifactAvailableCondition) ||
+ !conditions.Has(obj, sourcev1.SourceAvailableCondition) ||
+ !conditions.Has(obj, meta.ReadyCondition) ||
+ obj.Status.Artifact == nil {
+ return false
+ }
+
+ readyCondition := conditions.Get(obj, meta.ReadyCondition)
+
+ return readyCondition.Status == metav1.ConditionTrue &&
+ obj.Generation == readyCondition.ObservedGeneration
+ }, timeout).Should(BeTrue())
+
+ g.Expect(env.Delete(ctx, obj)).To(Succeed())
- cert := corev1.Secret{
+ // Wait for GitRepository to be deleted
+ g.Eventually(func() bool {
+ if err := env.Get(ctx, key, obj); err != nil {
+ return apierrors.IsNotFound(err)
+ }
+ return false
+ }, timeout).Should(BeTrue())
+}
+
+func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
+ type options struct {
+ username string
+ password string
+ publicKey []byte
+ privateKey []byte
+ ca []byte
+ }
+
+ tests := []struct {
+ name string
+ skipForImplementation string
+ protocol string
+ server options
+ secret *corev1.Secret
+ beforeFunc func(obj *sourcev1.GitRepository)
+ want ctrl.Result
+ wantErr bool
+ assertConditions []metav1.Condition
+ }{
+ {
+ name: "HTTP",
+ protocol: "http",
+ want: ctrl.Result{RequeueAfter: interval},
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.SourceAvailableCondition, sourcev1.GitOperationSucceedReason, "Checked out revision master/"),
+ },
+ },
+ {
+ name: "HTTP with BasicAuth",
+ protocol: "http",
+ server: options{
+ username: "git",
+ password: "1234",
+ },
+ secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
- Name: "cert",
- Namespace: namespace.Name,
+ Name: "basic-auth",
},
Data: map[string][]byte{
- "caFile": exampleCA,
+ "username": []byte("git"),
+ "password": []byte("1234"),
},
- }
- err = k8sClient.Create(context.Background(), &cert)
- Expect(err).NotTo(HaveOccurred())
+ },
+ beforeFunc: func(obj *sourcev1.GitRepository) {
+ obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "basic-auth"}
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.SourceAvailableCondition, sourcev1.GitOperationSucceedReason, "Checked out revision master/"),
+ },
+ },
+ {
+ name: "HTTPS with CAFile",
+ protocol: "https",
+ server: options{
+ publicKey: tlsPublicKey,
+ privateKey: tlsPrivateKey,
+ ca: tlsCA,
+ },
+ secret: &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "ca-file",
+ },
+ Data: map[string][]byte{
+ "caFile": tlsCA,
+ },
+ },
+ beforeFunc: func(obj *sourcev1.GitRepository) {
+ obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "ca-file"}
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.SourceAvailableCondition, sourcev1.GitOperationSucceedReason, "Checked out revision master/"),
+ },
+ },
+ {
+ name: "HTTPS with invalid CAFile (go-git)",
+ skipForImplementation: sourcev1.LibGit2Implementation,
+ protocol: "https",
+ server: options{
+ publicKey: tlsPublicKey,
+ privateKey: tlsPrivateKey,
+ ca: tlsCA,
+ },
+ secret: &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "invalid-ca",
+ },
+ Data: map[string][]byte{
+ "caFile": []byte("invalid"),
+ },
+ },
+ beforeFunc: func(obj *sourcev1.GitRepository) {
+ obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "invalid-ca"}
+ },
+ wantErr: true,
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceAvailableCondition, "GitOperationFailed", "Failed to checkout and determine HEAD revision: unable to clone '', error: Get \"/info/refs?service=git-upload-pack\": x509: certificate signed by unknown authority"),
+ },
+ },
+ {
+ name: "HTTPS with invalid CAFile (libgit2)",
+ skipForImplementation: sourcev1.GoGitImplementation,
+ protocol: "https",
+ server: options{
+ publicKey: tlsPublicKey,
+ privateKey: tlsPrivateKey,
+ ca: tlsCA,
+ },
+ secret: &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "invalid-ca",
+ },
+ Data: map[string][]byte{
+ "caFile": []byte("invalid"),
+ },
+ },
+ beforeFunc: func(obj *sourcev1.GitRepository) {
+ obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "invalid-ca"}
+ },
+ wantErr: true,
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceAvailableCondition, "GitOperationFailed", "Failed to checkout and determine HEAD revision: unable to clone '', error: Certificate"),
+ },
+ },
+ {
+ name: "SSH with private key",
+ protocol: "ssh",
+ server: options{
+ username: "git",
+ },
+ secret: &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "private-key",
+ },
+ Data: map[string][]byte{
+ "username": []byte("git"),
+ "identity": sshtestdata.PEMBytes["rsa"],
+ },
+ },
+ beforeFunc: func(obj *sourcev1.GitRepository) {
+ obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "private-key"}
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.SourceAvailableCondition, sourcev1.GitOperationSucceedReason, "Checked out revision master/"),
+ },
+ },
+ {
+ name: "SSH with password protected private key",
+ protocol: "ssh",
+ server: options{
+ username: "git",
+ },
+ secret: &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "private-key",
+ },
+ Data: map[string][]byte{
+ "username": []byte("git"),
+ "identity": sshtestdata.PEMEncryptedKeys[2].PEMBytes,
+ "password": []byte("password"),
+ },
+ },
+ beforeFunc: func(obj *sourcev1.GitRepository) {
+ obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "private-key"}
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.SourceAvailableCondition, sourcev1.GitOperationSucceedReason, "Checked out revision master/"),
+ },
+ },
+ {
+ name: "Missing secret",
+ protocol: "http",
+ server: options{
+ username: "git",
+ },
+ beforeFunc: func(obj *sourcev1.GitRepository) {
+ obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "non-existing"}
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceAvailableCondition, "AuthenticationFailed", "Failed to get secret /non-existing: secrets \"non-existing\" not found"),
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ obj := &sourcev1.GitRepository{
+ ObjectMeta: metav1.ObjectMeta{
+ GenerateName: "auth-strategy-",
+ },
+ Spec: sourcev1.GitRepositorySpec{
+ Interval: metav1.Duration{Duration: interval},
+ Timeout: &metav1.Duration{Duration: interval},
+ },
+ }
- gitServer, err = gittestserver.NewTempGitServer()
- Expect(err).NotTo(HaveOccurred())
- gitServer.AutoCreate()
- })
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
- AfterEach(func() {
- os.RemoveAll(gitServer.Root())
+ server, err := gittestserver.NewTempGitServer()
+ g.Expect(err).NotTo(HaveOccurred())
+ defer os.RemoveAll(server.Root())
+ server.AutoCreate()
- err = k8sClient.Delete(context.Background(), namespace)
- Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
- })
+ repoPath := "/test.git"
+ localRepo, err := initGitRepo(server, "testdata/git/repository", git.DefaultBranch, repoPath)
+ g.Expect(err).NotTo(HaveOccurred())
- type refTestCase struct {
- reference *sourcev1.GitRepositoryRef
- createRefs []string
+ if len(tt.server.username+tt.server.password) > 0 {
+ server.Auth(tt.server.username, tt.server.password)
+ }
- waitForReason string
+ secret := tt.secret.DeepCopy()
+ switch tt.protocol {
+ case "http":
+ g.Expect(server.StartHTTP()).To(Succeed())
+ defer server.StopHTTP()
+ obj.Spec.URL = server.HTTPAddress() + repoPath
+ case "https":
+ g.Expect(server.StartHTTPS(tt.server.publicKey, tt.server.privateKey, tt.server.ca, "example.com")).To(Succeed())
+ obj.Spec.URL = server.HTTPAddress() + repoPath
+ case "ssh":
+ server.KeyDir(filepath.Join(server.Root(), "keys"))
+
+ g.Expect(server.ListenSSH()).To(Succeed())
+ obj.Spec.URL = server.SSHAddress() + repoPath
+
+ go func() {
+ server.StartSSH()
+ }()
+ defer server.StopSSH()
+
+ if secret != nil && len(secret.Data["known_hosts"]) == 0 {
+ u, err := url.Parse(obj.Spec.URL)
+ g.Expect(err).NotTo(HaveOccurred())
+ g.Expect(u.Host).ToNot(BeEmpty())
+ knownHosts, err := ssh.ScanHostKey(u.Host, timeout)
+ g.Expect(err).NotTo(HaveOccurred())
+ secret.Data["known_hosts"] = knownHosts
+ }
+ default:
+ t.Fatalf("unsupported protocol %q", tt.protocol)
+ }
- expectStatus metav1.ConditionStatus
- expectMessage string
- expectRevision string
+ if tt.beforeFunc != nil {
+ tt.beforeFunc(obj)
+ }
- secretRef *meta.LocalObjectReference
- gitImplementation string
- }
+ builder := fakeclient.NewClientBuilder().WithScheme(env.GetScheme())
+ if secret != nil {
+ builder.WithObjects(secret.DeepCopy())
+ }
- DescribeTable("Git references tests", func(t refTestCase) {
- err = gitServer.StartHTTP()
- defer gitServer.StopHTTP()
- Expect(err).NotTo(HaveOccurred())
-
- u, err := url.Parse(gitServer.HTTPAddress())
- Expect(err).NotTo(HaveOccurred())
- u.Path = path.Join(u.Path, fmt.Sprintf("repository-%s.git", randStringRunes(5)))
-
- fs := memfs.New()
- gitrepo, err := git.Init(memory.NewStorage(), fs)
- Expect(err).NotTo(HaveOccurred())
-
- wt, err := gitrepo.Worktree()
- Expect(err).NotTo(HaveOccurred())
-
- ff, _ := fs.Create("fixture")
- _ = ff.Close()
- _, err = wt.Add(fs.Join("fixture"))
- Expect(err).NotTo(HaveOccurred())
-
- commit, err := wt.Commit("Sample", &git.CommitOptions{Author: &object.Signature{
- Name: "John Doe",
- Email: "john@example.com",
- When: time.Now(),
- }})
- Expect(err).NotTo(HaveOccurred())
-
- for _, ref := range t.createRefs {
- hRef := plumbing.NewHashReference(plumbing.ReferenceName(ref), commit)
- err = gitrepo.Storer.SetReference(hRef)
- Expect(err).NotTo(HaveOccurred())
+ r := &GitRepositoryReconciler{
+ Client: builder.Build(),
+ Storage: storage,
}
- remote, err := gitrepo.CreateRemote(&config.RemoteConfig{
- Name: "origin",
- URLs: []string{u.String()},
- })
- Expect(err).NotTo(HaveOccurred())
+ for _, i := range testGitImplementations {
+ t.Run(i, func(t *testing.T) {
+ g := NewWithT(t)
+
+ if tt.skipForImplementation == i {
+ t.Skipf("Skipped for Git implementation %q", i)
+ }
+
+ tmpDir, err := os.MkdirTemp("", "auth-strategy-")
+ g.Expect(err).To(BeNil())
+ defer os.RemoveAll(tmpDir)
- err = remote.Push(&git.PushOptions{
- RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"},
- })
- Expect(err).NotTo(HaveOccurred())
+ obj := obj.DeepCopy()
+ obj.Spec.GitImplementation = i
- t.reference.Commit = strings.Replace(t.reference.Commit, "", commit.String(), 1)
+ head, _ := localRepo.Head()
+ assertConditions := tt.assertConditions
+ for k := range assertConditions {
+ assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "", head.Hash().String())
+ assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "", obj.Spec.URL)
+ }
- key := types.NamespacedName{
- Name: fmt.Sprintf("git-ref-test-%s", randStringRunes(5)),
- Namespace: namespace.Name,
+ var artifact sourcev1.Artifact
+ got, err := r.reconcileSource(logr.NewContext(ctx, log.NullLogger{}), obj, &artifact, tmpDir)
+ g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions))
+ g.Expect(err != nil).To(Equal(tt.wantErr))
+ g.Expect(got).To(Equal(tt.want))
+ g.Expect(artifact).ToNot(BeNil())
+ })
}
- created := &sourcev1.GitRepository{
+ })
+ }
+}
+
+func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T) {
+ g := NewWithT(t)
+
+ branches := []string{"staging"}
+ tags := []string{"non-semver-tag", "v0.1.0", "0.2.0", "v0.2.1", "v1.0.0-alpha", "v1.1.0", "v2.0.0"}
+
+ tests := []struct {
+ name string
+ reference *sourcev1.GitRepositoryRef
+ want ctrl.Result
+ wantErr bool
+ wantRevision string
+ }{
+ {
+ name: "Nil reference (default branch)",
+ want: ctrl.Result{RequeueAfter: interval},
+ wantRevision: "master/",
+ },
+ {
+ name: "Branch",
+ reference: &sourcev1.GitRepositoryRef{
+ Branch: "staging",
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ wantRevision: "staging/",
+ },
+ {
+ name: "Tag",
+ reference: &sourcev1.GitRepositoryRef{
+ Tag: "v0.1.0",
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ wantRevision: "v0.1.0/",
+ },
+ {
+ name: "Branch commit",
+ reference: &sourcev1.GitRepositoryRef{
+ Branch: "staging",
+ Commit: "",
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ wantRevision: "staging/",
+ },
+ {
+ name: "SemVer",
+ reference: &sourcev1.GitRepositoryRef{
+ SemVer: "*",
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ wantRevision: "v2.0.0/",
+ },
+ {
+ name: "SemVer range",
+ reference: &sourcev1.GitRepositoryRef{
+ SemVer: "",
+ },
+ {
+ name: "SemVer prerelease",
+ reference: &sourcev1.GitRepositoryRef{
+ SemVer: ">=1.0.0-0 <1.1.0-0",
+ },
+ wantRevision: "v1.0.0-alpha/",
+ want: ctrl.Result{RequeueAfter: interval},
+ },
+ }
+
+ server, err := gittestserver.NewTempGitServer()
+ g.Expect(err).To(BeNil())
+ server.AutoCreate()
+ g.Expect(server.StartHTTP()).To(Succeed())
+ defer server.StopHTTP()
+
+ repoPath := "/test.git"
+ localRepo, err := initGitRepo(server, "testdata/git/repository", git.DefaultBranch, repoPath)
+ g.Expect(err).NotTo(HaveOccurred())
+
+ headRef, err := localRepo.Head()
+ g.Expect(err).NotTo(HaveOccurred())
+
+ for _, branch := range branches {
+ g.Expect(remoteBranchForHead(localRepo, headRef, branch)).To(Succeed())
+ }
+ for _, tag := range tags {
+ g.Expect(remoteTagForHead(localRepo, headRef, tag)).To(Succeed())
+ }
+
+ r := &GitRepositoryReconciler{
+ Client: fakeclient.NewClientBuilder().WithScheme(runtime.NewScheme()).Build(),
+ Storage: storage,
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ obj := &sourcev1.GitRepository{
ObjectMeta: metav1.ObjectMeta{
- Name: key.Name,
- Namespace: key.Namespace,
+ GenerateName: "checkout-strategy-",
},
Spec: sourcev1.GitRepositorySpec{
- URL: u.String(),
- Interval: metav1.Duration{Duration: indexInterval},
- Reference: t.reference,
+ Interval: metav1.Duration{Duration: interval},
+ Timeout: &metav1.Duration{Duration: interval},
+ URL: server.HTTPAddress() + repoPath,
+ Reference: tt.reference,
},
}
- Expect(k8sClient.Create(context.Background(), created)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), created)
-
- got := &sourcev1.GitRepository{}
- var cond metav1.Condition
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- for _, c := range got.Status.Conditions {
- if c.Reason == t.waitForReason {
- cond = c
- return true
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
-
- Expect(cond.Status).To(Equal(t.expectStatus))
- Expect(cond.Message).To(ContainSubstring(t.expectMessage))
- Expect(got.Status.Artifact == nil).To(Equal(t.expectRevision == ""))
- if t.expectRevision != "" {
- Expect(got.Status.Artifact.Revision).To(Equal(t.expectRevision + "/" + commit.String()))
- }
- },
- Entry("branch", refTestCase{
- reference: &sourcev1.GitRepositoryRef{Branch: "some-branch"},
- createRefs: []string{"refs/heads/some-branch"},
- waitForReason: sourcev1.GitOperationSucceedReason,
- expectStatus: metav1.ConditionTrue,
- expectRevision: "some-branch",
- }),
- Entry("branch non existing", refTestCase{
- reference: &sourcev1.GitRepositoryRef{Branch: "invalid-branch"},
- waitForReason: sourcev1.GitOperationFailedReason,
- expectStatus: metav1.ConditionFalse,
- expectMessage: "couldn't find remote ref",
- }),
- Entry("tag", refTestCase{
- reference: &sourcev1.GitRepositoryRef{Tag: "some-tag"},
- createRefs: []string{"refs/tags/some-tag"},
- waitForReason: sourcev1.GitOperationSucceedReason,
- expectStatus: metav1.ConditionTrue,
- expectRevision: "some-tag",
- }),
- Entry("tag non existing", refTestCase{
- reference: &sourcev1.GitRepositoryRef{Tag: "invalid-tag"},
- waitForReason: sourcev1.GitOperationFailedReason,
- expectStatus: metav1.ConditionFalse,
- expectMessage: "couldn't find remote ref",
- }),
- Entry("semver", refTestCase{
- reference: &sourcev1.GitRepositoryRef{SemVer: "1.0.0"},
- createRefs: []string{"refs/tags/v1.0.0"},
- waitForReason: sourcev1.GitOperationSucceedReason,
- expectStatus: metav1.ConditionTrue,
- expectRevision: "v1.0.0",
- }),
- Entry("semver range", refTestCase{
- reference: &sourcev1.GitRepositoryRef{SemVer: ">=0.1.0 <1.0.0"},
- createRefs: []string{"refs/tags/0.1.0", "refs/tags/0.1.1", "refs/tags/0.2.0", "refs/tags/1.0.0"},
- waitForReason: sourcev1.GitOperationSucceedReason,
- expectStatus: metav1.ConditionTrue,
- expectRevision: "0.2.0",
- }),
- Entry("mixed semver range", refTestCase{
- reference: &sourcev1.GitRepositoryRef{SemVer: ">=0.1.0 <1.0.0"},
- createRefs: []string{"refs/tags/0.1.0", "refs/tags/v0.1.1", "refs/tags/v0.2.0", "refs/tags/1.0.0"},
- waitForReason: sourcev1.GitOperationSucceedReason,
- expectStatus: metav1.ConditionTrue,
- expectRevision: "v0.2.0",
- }),
- Entry("semver invalid", refTestCase{
- reference: &sourcev1.GitRepositoryRef{SemVer: "1.2.3.4"},
- waitForReason: sourcev1.GitOperationFailedReason,
- expectStatus: metav1.ConditionFalse,
- expectMessage: "semver parse range error: improper constraint: 1.2.3.4",
- }),
- Entry("semver no match", refTestCase{
- reference: &sourcev1.GitRepositoryRef{SemVer: "1.0.0"},
- waitForReason: sourcev1.GitOperationFailedReason,
- expectStatus: metav1.ConditionFalse,
- expectMessage: "no match found for semver: 1.0.0",
- }),
- Entry("commit", refTestCase{
- reference: &sourcev1.GitRepositoryRef{
- Commit: "",
- },
- waitForReason: sourcev1.GitOperationSucceedReason,
- expectStatus: metav1.ConditionTrue,
- expectRevision: "master",
- }),
- Entry("commit in branch", refTestCase{
- reference: &sourcev1.GitRepositoryRef{
- Branch: "some-branch",
- Commit: "",
- },
- createRefs: []string{"refs/heads/some-branch"},
- waitForReason: sourcev1.GitOperationSucceedReason,
- expectStatus: metav1.ConditionTrue,
- expectRevision: "some-branch",
- }),
- Entry("invalid commit", refTestCase{
- reference: &sourcev1.GitRepositoryRef{
- Branch: "master",
- Commit: "invalid",
- },
- waitForReason: sourcev1.GitOperationFailedReason,
- expectStatus: metav1.ConditionFalse,
- expectMessage: "git commit 'invalid' not found: object not found",
- }),
- )
-
- DescribeTable("Git self signed cert tests", func(t refTestCase) {
- err = gitServer.StartHTTPS(examplePublicKey, examplePrivateKey, exampleCA, "example.com")
- defer gitServer.StopHTTP()
- Expect(err).NotTo(HaveOccurred())
-
- u, err := url.Parse(gitServer.HTTPAddress())
- Expect(err).NotTo(HaveOccurred())
- u.Path = path.Join(u.Path, fmt.Sprintf("repository-%s.git", randStringRunes(5)))
-
- var transport = httptransport.NewClient(&http.Client{
- Transport: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
- },
- })
- client.InstallProtocol("https", transport)
-
- fs := memfs.New()
- gitrepo, err := git.Init(memory.NewStorage(), fs)
- Expect(err).NotTo(HaveOccurred())
-
- wt, err := gitrepo.Worktree()
- Expect(err).NotTo(HaveOccurred())
-
- ff, _ := fs.Create("fixture")
- _ = ff.Close()
- _, err = wt.Add(fs.Join("fixture"))
- Expect(err).NotTo(HaveOccurred())
-
- commit, err := wt.Commit("Sample", &git.CommitOptions{Author: &object.Signature{
- Name: "John Doe",
- Email: "john@example.com",
- When: time.Now(),
- }})
- Expect(err).NotTo(HaveOccurred())
-
- for _, ref := range t.createRefs {
- hRef := plumbing.NewHashReference(plumbing.ReferenceName(ref), commit)
- err = gitrepo.Storer.SetReference(hRef)
- Expect(err).NotTo(HaveOccurred())
+
+ if obj.Spec.Reference != nil && obj.Spec.Reference.Commit == "" {
+ obj.Spec.Reference.Commit = headRef.Hash().String()
}
- remote, err := gitrepo.CreateRemote(&config.RemoteConfig{
- Name: "origin",
- URLs: []string{u.String()},
- })
- Expect(err).NotTo(HaveOccurred())
+ for _, i := range testGitImplementations {
+ t.Run(i, func(t *testing.T) {
+ g := NewWithT(t)
- err = remote.Push(&git.PushOptions{
- RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"},
- })
- Expect(err).NotTo(HaveOccurred())
+ tmpDir, err := os.MkdirTemp("", "checkout-strategy-")
+ g.Expect(err).NotTo(HaveOccurred())
- t.reference.Commit = strings.Replace(t.reference.Commit, "", commit.String(), 1)
+ obj := obj.DeepCopy()
+ obj.Spec.GitImplementation = i
- client.InstallProtocol("https", httptransport.DefaultClient)
+ var artifact sourcev1.Artifact
+ got, err := r.reconcileSource(ctx, obj, &artifact, tmpDir)
+ if err != nil {
+ println(err.Error())
+ }
+ g.Expect(err != nil).To(Equal(tt.wantErr))
+ g.Expect(got).To(Equal(tt.want))
+ if tt.wantRevision != "" {
+ revision := strings.ReplaceAll(tt.wantRevision, "", headRef.Hash().String())
+ g.Expect(artifact.Revision).To(Equal(revision))
+ }
+ })
+ }
+ })
+ }
+}
+
+func TestGitRepositoryReconciler_reconcileArtifact(t *testing.T) {
+ tests := []struct {
+ name string
+ dir string
+ beforeFunc func(obj *sourcev1.GitRepository)
+ afterFunc func(t *WithT, obj *sourcev1.GitRepository, artifact sourcev1.Artifact)
+ want ctrl.Result
+ wantErr bool
+ assertConditions []metav1.Condition
+ }{
+ {
+ name: "Archive artifact",
+ dir: "testdata/git/repository",
+ beforeFunc: func(obj *sourcev1.GitRepository) {
+ obj.Spec.Interval = metav1.Duration{Duration: interval}
+ },
+ afterFunc: func(t *WithT, obj *sourcev1.GitRepository, artifact sourcev1.Artifact) {
+ t.Expect(obj.GetArtifact()).ToNot(BeNil())
+ t.Expect(obj.GetArtifact().Checksum).NotTo(BeEmpty())
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.ArtifactAvailableCondition, "ArchivedArtifact", "Compressed source to artifact with revision main/revision"),
+ },
+ },
+ {
+ name: "Invalid directory",
+ dir: "/a/random/invalid/path",
+ afterFunc: func(t *WithT, obj *sourcev1.GitRepository, artifact sourcev1.Artifact) {
+ t.Expect(obj.GetArtifact()).To(BeNil())
+ },
+ wantErr: true,
+ assertConditions: []metav1.Condition{
+ {
+ Type: sourcev1.ArtifactAvailableCondition,
+ Status: metav1.ConditionFalse,
+ Reason: sourcev1.StorageOperationFailedReason,
+ Message: "Failed to stat source path: stat /a/random/invalid/path: no such file or directory",
+ },
+ },
+ },
+ {
+ name: "Spec ignore overwrite",
+ dir: "testdata/git/repository",
+ beforeFunc: func(obj *sourcev1.GitRepository) {
+ obj.Spec.Interval = metav1.Duration{Duration: interval}
+ obj.Spec.Ignore = pointer.StringPtr("!**.txt\n")
+ },
+ afterFunc: func(t *WithT, obj *sourcev1.GitRepository, artifact sourcev1.Artifact) {
+ t.Expect(obj.GetArtifact()).ToNot(BeNil())
+ t.Expect(obj.GetArtifact().Checksum).NotTo(BeEmpty())
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.ArtifactAvailableCondition, "ArchivedArtifact", "Compressed source to artifact with revision main/revision"),
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
- key := types.NamespacedName{
- Name: fmt.Sprintf("git-ref-test-%s", randStringRunes(5)),
- Namespace: namespace.Name,
+ r := &GitRepositoryReconciler{
+ Storage: storage,
}
- created := &sourcev1.GitRepository{
+
+ obj := &sourcev1.GitRepository{
ObjectMeta: metav1.ObjectMeta{
- Name: key.Name,
- Namespace: key.Namespace,
- },
- Spec: sourcev1.GitRepositorySpec{
- URL: u.String(),
- Interval: metav1.Duration{Duration: indexInterval},
- Reference: t.reference,
- GitImplementation: t.gitImplementation,
- SecretRef: t.secretRef,
+ GenerateName: "reconcile-artifact-",
+ Generation: 1,
},
+ Status: sourcev1.GitRepositoryStatus{},
}
- Expect(k8sClient.Create(context.Background(), created)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), created)
-
- got := &sourcev1.GitRepository{}
- var cond metav1.Condition
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- for _, c := range got.Status.Conditions {
- if c.Reason == t.waitForReason {
- cond = c
- return true
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
-
- Expect(cond.Status).To(Equal(t.expectStatus))
- Expect(cond.Message).To(ContainSubstring(t.expectMessage))
- Expect(got.Status.Artifact == nil).To(Equal(t.expectRevision == ""))
- },
- Entry("self signed libgit2 without CA", refTestCase{
- reference: &sourcev1.GitRepositoryRef{Branch: "main"},
- waitForReason: sourcev1.GitOperationFailedReason,
- expectStatus: metav1.ConditionFalse,
- expectMessage: "error: user rejected certificate",
- gitImplementation: sourcev1.LibGit2Implementation,
- }),
- Entry("self signed libgit2 with CA", refTestCase{
- reference: &sourcev1.GitRepositoryRef{Branch: "some-branch"},
- createRefs: []string{"refs/heads/some-branch"},
- waitForReason: sourcev1.GitOperationSucceedReason,
- expectStatus: metav1.ConditionTrue,
- expectRevision: "some-branch",
- secretRef: &meta.LocalObjectReference{Name: "cert"},
- gitImplementation: sourcev1.LibGit2Implementation,
- }),
- Entry("self signed go-git without CA", refTestCase{
- reference: &sourcev1.GitRepositoryRef{Branch: "main"},
- waitForReason: sourcev1.GitOperationFailedReason,
- expectStatus: metav1.ConditionFalse,
- expectMessage: "x509: certificate signed by unknown authority",
- }),
- Entry("self signed go-git with CA", refTestCase{
- reference: &sourcev1.GitRepositoryRef{Branch: "some-branch"},
- createRefs: []string{"refs/heads/some-branch"},
- waitForReason: sourcev1.GitOperationSucceedReason,
- expectStatus: metav1.ConditionTrue,
- expectRevision: "some-branch",
- secretRef: &meta.LocalObjectReference{Name: "cert"},
- gitImplementation: sourcev1.GoGitImplementation,
- }),
- )
-
- Context("recurse submodules", func() {
- It("downloads submodules when asked", func() {
- Expect(gitServer.StartHTTP()).To(Succeed())
- defer gitServer.StopHTTP()
-
- u, err := url.Parse(gitServer.HTTPAddress())
- Expect(err).NotTo(HaveOccurred())
-
- subRepoURL := *u
- subRepoURL.Path = path.Join(u.Path, fmt.Sprintf("subrepository-%s.git", randStringRunes(5)))
-
- // create the git repo to use as a submodule
- fs := memfs.New()
- subRepo, err := git.Init(memory.NewStorage(), fs)
- Expect(err).NotTo(HaveOccurred())
-
- wt, err := subRepo.Worktree()
- Expect(err).NotTo(HaveOccurred())
-
- ff, _ := fs.Create("fixture")
- _ = ff.Close()
- _, err = wt.Add(fs.Join("fixture"))
- Expect(err).NotTo(HaveOccurred())
-
- _, err = wt.Commit("Sample", &git.CommitOptions{Author: &object.Signature{
- Name: "John Doe",
- Email: "john@example.com",
- When: time.Now(),
- }})
- Expect(err).NotTo(HaveOccurred())
-
- remote, err := subRepo.CreateRemote(&config.RemoteConfig{
- Name: "origin",
- URLs: []string{subRepoURL.String()},
- })
- Expect(err).NotTo(HaveOccurred())
- err = remote.Push(&git.PushOptions{
- RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"},
- })
- Expect(err).NotTo(HaveOccurred())
-
- // this one is linked to a real directory, so that I can
- // exec `git submodule add` later
- tmp, err := ioutil.TempDir("", "flux-test")
- Expect(err).NotTo(HaveOccurred())
- defer os.RemoveAll(tmp)
-
- repoDir := filepath.Join(tmp, "git")
- repo, err := git.PlainInit(repoDir, false)
- Expect(err).NotTo(HaveOccurred())
-
- wt, err = repo.Worktree()
- Expect(err).NotTo(HaveOccurred())
- _, err = wt.Commit("Initial revision", &git.CommitOptions{
- Author: &object.Signature{
- Name: "John Doe",
- Email: "john@example.com",
- When: time.Now(),
- }})
- Expect(err).NotTo(HaveOccurred())
-
- submodAdd := exec.Command("git", "submodule", "add", "-b", "master", subRepoURL.String(), "sub")
- submodAdd.Dir = repoDir
- out, err := submodAdd.CombinedOutput()
- os.Stdout.Write(out)
- Expect(err).NotTo(HaveOccurred())
-
- _, err = wt.Commit("Add submodule", &git.CommitOptions{
- Author: &object.Signature{
- Name: "John Doe",
- Email: "john@example.com",
- When: time.Now(),
- }})
- Expect(err).NotTo(HaveOccurred())
-
- mainRepoURL := *u
- mainRepoURL.Path = path.Join(u.Path, fmt.Sprintf("repository-%s.git", randStringRunes(5)))
- remote, err = repo.CreateRemote(&config.RemoteConfig{
- Name: "origin",
- URLs: []string{mainRepoURL.String()},
- })
- Expect(err).NotTo(HaveOccurred())
+ if tt.beforeFunc != nil {
+ tt.beforeFunc(obj)
+ }
- err = remote.Push(&git.PushOptions{
- RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"},
- })
- Expect(err).NotTo(HaveOccurred())
+ artifact := storage.NewArtifactFor(obj.Kind, obj, "main/revision", "checksum.tar.gz")
- key := types.NamespacedName{
- Name: fmt.Sprintf("git-ref-test-%s", randStringRunes(5)),
- Namespace: namespace.Name,
- }
- created := &sourcev1.GitRepository{
+ got, err := r.reconcileArtifact(ctx, obj, artifact, nil, tt.dir)
+ g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions))
+ g.Expect(err != nil).To(Equal(tt.wantErr))
+ g.Expect(got).To(Equal(tt.want))
+
+ if tt.afterFunc != nil {
+ tt.afterFunc(g, obj, artifact)
+ }
+ })
+ }
+}
+
+func TestGitRepositoryReconciler_reconcileInclude(t *testing.T) {
+ g := NewWithT(t)
+
+ server, err := testserver.NewTempArtifactServer()
+ g.Expect(err).NotTo(HaveOccurred())
+ storage, err := newTestStorage(server.HTTPServer)
+ g.Expect(err).NotTo(HaveOccurred())
+ defer os.RemoveAll(storage.BasePath)
+
+ dependencyInterval := 5 * time.Second
+
+ type dependency struct {
+ name string
+ withArtifact bool
+ conditions []metav1.Condition
+ }
+
+ type include struct {
+ name string
+ fromPath string
+ toPath string
+ shouldExist bool
+ }
+
+ tests := []struct {
+ name string
+ dependencies []dependency
+ includes []include
+ want ctrl.Result
+ wantErr bool
+ assertConditions []metav1.Condition
+ }{
+ {
+ name: "Includes artifacts",
+ dependencies: []dependency{
+ {
+ name: "a",
+ withArtifact: true,
+ conditions: []metav1.Condition{
+ *conditions.TrueCondition(meta.ReadyCondition, "Foo", "foo ready"),
+ },
+ },
+ {
+ name: "b",
+ withArtifact: true,
+ conditions: []metav1.Condition{
+ *conditions.TrueCondition(meta.ReadyCondition, "Bar", "bar ready"),
+ },
+ },
+ },
+ includes: []include{
+ {name: "a", toPath: "a/"},
+ {name: "b", toPath: "b/"},
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.SourceAvailableCondition, "Foo", "2 of 2 Ready"),
+ },
+ },
+ {
+ name: "Non existing artifact",
+ includes: []include{
+ {name: "a", toPath: "a/"},
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ wantErr: false,
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceAvailableCondition, "IncludeNotFound", "Could not find resource for include \"a\": gitrepositories.source.toolkit.fluxcd.io \"a\" not found"),
+ },
+ },
+ {
+ name: "Missing artifact",
+ dependencies: []dependency{
+ {
+ name: "a",
+ withArtifact: false,
+ conditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceAvailableCondition, "Foo", "foo unavailable"),
+ },
+ },
+ },
+ includes: []include{
+ {name: "a", toPath: "a/"},
+ },
+ want: ctrl.Result{RequeueAfter: dependencyInterval},
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceAvailableCondition, "IncludeUnavailable", "No artifact available for include \"a\""),
+ },
+ },
+ {
+ name: "Invalid FromPath",
+ dependencies: []dependency{
+ {
+ name: "a",
+ withArtifact: true,
+ },
+ },
+ includes: []include{
+ {name: "a", fromPath: "../../../path", shouldExist: false},
+ },
+ wantErr: true,
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceAvailableCondition, "IncludeCopyFailure", "Failed to copy \"a\" include from ../../../path to a"),
+ },
+ },
+ {
+ name: "Stalled include",
+ dependencies: []dependency{
+ {
+ name: "a",
+ withArtifact: true,
+ conditions: []metav1.Condition{
+ *conditions.TrueCondition(meta.ReadyCondition, "Foo", "foo ready"),
+ },
+ },
+ {
+ name: "b",
+ withArtifact: true,
+ conditions: []metav1.Condition{
+ *conditions.TrueCondition(meta.StalledCondition, "Bar", "bar stalled"),
+ },
+ },
+ },
+ includes: []include{
+ {name: "a", toPath: "a/"},
+ {name: "b", toPath: "b/"},
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceAvailableCondition, "Bar @ GitRepository/a", "bar stalled"),
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ var depObjs []client.Object
+ for _, d := range tt.dependencies {
+ obj := &sourcev1.GitRepository{
ObjectMeta: metav1.ObjectMeta{
- Name: key.Name,
- Namespace: key.Namespace,
+ Name: d.name,
},
- Spec: sourcev1.GitRepositorySpec{
- URL: mainRepoURL.String(),
- Interval: metav1.Duration{Duration: indexInterval},
- Reference: &sourcev1.GitRepositoryRef{Branch: "master"},
- GitImplementation: sourcev1.GoGitImplementation, // only works with go-git
- RecurseSubmodules: true,
+ Status: sourcev1.GitRepositoryStatus{
+ Conditions: d.conditions,
},
}
- Expect(k8sClient.Create(context.Background(), created)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), created)
-
- got := &sourcev1.GitRepository{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- for _, c := range got.Status.Conditions {
- if c.Reason == sourcev1.GitOperationSucceedReason {
- return true
- }
+ if d.withArtifact {
+ obj.Status.Artifact = &sourcev1.Artifact{
+ Path: d.name + ".tar.gz",
+ Revision: d.name,
+ LastUpdateTime: metav1.Now(),
}
- return false
- }, timeout, interval).Should(BeTrue())
-
- // check that the downloaded artifact includes the
- // file from the submodule
- res, err := http.Get(got.Status.URL)
- Expect(err).NotTo(HaveOccurred())
- Expect(res.StatusCode).To(Equal(http.StatusOK))
-
- _, err = untar.Untar(res.Body, filepath.Join(tmp, "tar"))
- Expect(err).NotTo(HaveOccurred())
- Expect(filepath.Join(tmp, "tar", "sub", "fixture")).To(BeAnExistingFile())
- })
- })
-
- type includeTestCase struct {
- fromPath string
- toPath string
- createFiles []string
- checkFiles []string
- }
-
- DescribeTable("Include git repositories", func(t includeTestCase) {
- Expect(gitServer.StartHTTP()).To(Succeed())
- defer gitServer.StopHTTP()
-
- u, err := url.Parse(gitServer.HTTPAddress())
- Expect(err).NotTo(HaveOccurred())
-
- // create the main git repository
- mainRepoURL := *u
- mainRepoURL.Path = path.Join(u.Path, fmt.Sprintf("repository-%s.git", randStringRunes(5)))
-
- mainFs := memfs.New()
- mainRepo, err := git.Init(memory.NewStorage(), mainFs)
- Expect(err).NotTo(HaveOccurred())
-
- mainWt, err := mainRepo.Worktree()
- Expect(err).NotTo(HaveOccurred())
-
- ff, _ := mainFs.Create("fixture")
- _ = ff.Close()
- _, err = mainWt.Add(mainFs.Join("fixture"))
- Expect(err).NotTo(HaveOccurred())
-
- _, err = mainWt.Commit("Sample", &git.CommitOptions{Author: &object.Signature{
- Name: "John Doe",
- Email: "john@example.com",
- When: time.Now(),
- }})
- Expect(err).NotTo(HaveOccurred())
-
- mainRemote, err := mainRepo.CreateRemote(&config.RemoteConfig{
- Name: "origin",
- URLs: []string{mainRepoURL.String()},
- })
- Expect(err).NotTo(HaveOccurred())
-
- err = mainRemote.Push(&git.PushOptions{
- RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"},
- })
- Expect(err).NotTo(HaveOccurred())
-
- // create the sub git repository
- subRepoURL := *u
- subRepoURL.Path = path.Join(u.Path, fmt.Sprintf("subrepository-%s.git", randStringRunes(5)))
-
- subFs := memfs.New()
- subRepo, err := git.Init(memory.NewStorage(), subFs)
- Expect(err).NotTo(HaveOccurred())
-
- subWt, err := subRepo.Worktree()
- Expect(err).NotTo(HaveOccurred())
-
- for _, v := range t.createFiles {
- if dir := filepath.Base(v); dir != v {
- err := subFs.MkdirAll(dir, 0700)
- Expect(err).NotTo(HaveOccurred())
+ g.Expect(storage.Archive(obj.GetArtifact(), "testdata/git/repository", nil)).To(Succeed())
}
- ff, err := subFs.Create(v)
- Expect(err).NotTo(HaveOccurred())
- _ = ff.Close()
- _, err = subWt.Add(subFs.Join(v))
- Expect(err).NotTo(HaveOccurred())
+ depObjs = append(depObjs, obj)
}
- _, err = subWt.Commit("Sample", &git.CommitOptions{Author: &object.Signature{
- Name: "John Doe",
- Email: "john@example.com",
- When: time.Now(),
- }})
- Expect(err).NotTo(HaveOccurred())
-
- subRemote, err := subRepo.CreateRemote(&config.RemoteConfig{
- Name: "origin",
- URLs: []string{subRepoURL.String()},
- })
- Expect(err).NotTo(HaveOccurred())
-
- err = subRemote.Push(&git.PushOptions{
- RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"},
- })
- Expect(err).NotTo(HaveOccurred())
-
- // create main and sub resetRepositories
- subKey := types.NamespacedName{
- Name: fmt.Sprintf("git-ref-test-%s", randStringRunes(5)),
- Namespace: namespace.Name,
+ builder := fakeclient.NewClientBuilder().WithScheme(env.GetScheme())
+ if len(tt.dependencies) > 0 {
+ builder.WithObjects(depObjs...)
}
- subCreated := &sourcev1.GitRepository{
+
+ r := &GitRepositoryReconciler{
+ Client: builder.Build(),
+ Storage: storage,
+ requeueDependency: dependencyInterval,
+ }
+
+ obj := &sourcev1.GitRepository{
ObjectMeta: metav1.ObjectMeta{
- Name: subKey.Name,
- Namespace: subKey.Namespace,
+ Name: "reconcile-include",
},
Spec: sourcev1.GitRepositorySpec{
- URL: subRepoURL.String(),
- Interval: metav1.Duration{Duration: indexInterval},
- Reference: &sourcev1.GitRepositoryRef{Branch: "master"},
+ Interval: metav1.Duration{Duration: interval},
},
}
- Expect(k8sClient.Create(context.Background(), subCreated)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), subCreated)
- mainKey := types.NamespacedName{
- Name: fmt.Sprintf("git-ref-test-%s", randStringRunes(5)),
- Namespace: namespace.Name,
+ for i, incl := range tt.includes {
+ incl := sourcev1.GitRepositoryInclude{
+ GitRepositoryRef: meta.LocalObjectReference{Name: incl.name},
+ FromPath: incl.fromPath,
+ ToPath: incl.toPath,
+ }
+ tt.includes[i].fromPath = incl.GetFromPath()
+ tt.includes[i].toPath = incl.GetToPath()
+ obj.Spec.Include = append(obj.Spec.Include, incl)
+ }
+
+ tmpDir, err := os.MkdirTemp("", "include-")
+ g.Expect(err).NotTo(HaveOccurred())
+
+ var artifacts artifactSet
+ got, err := r.reconcileInclude(ctx, obj, artifacts, tmpDir)
+ g.Expect(obj.GetConditions()).To(conditions.MatchConditions(tt.assertConditions))
+ g.Expect(err != nil).To(Equal(tt.wantErr))
+ g.Expect(got).To(Equal(tt.want))
+ for _, i := range tt.includes {
+ if i.toPath != "" {
+ expect := g.Expect(filepath.Join(storage.BasePath, i.toPath))
+ if i.shouldExist {
+ expect.To(BeADirectory())
+ } else {
+ expect.NotTo(BeADirectory())
+ }
+ }
+ if i.shouldExist {
+ g.Expect(filepath.Join(storage.BasePath, i.toPath)).Should(BeADirectory())
+ } else {
+ g.Expect(filepath.Join(storage.BasePath, i.toPath)).ShouldNot(BeADirectory())
+ }
}
- mainCreated := &sourcev1.GitRepository{
+ })
+ }
+}
+
+func TestGitRepositoryReconciler_reconcileDelete(t *testing.T) {
+ g := NewWithT(t)
+
+ r := &GitRepositoryReconciler{
+ Storage: storage,
+ }
+
+ obj := &sourcev1.GitRepository{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "reconcile-delete-",
+ DeletionTimestamp: &metav1.Time{Time: time.Now()},
+ Finalizers: []string{
+ sourcev1.SourceFinalizer,
+ },
+ },
+ Status: sourcev1.GitRepositoryStatus{},
+ }
+
+ artifact := storage.NewArtifactFor(sourcev1.GitRepositoryKind, obj.GetObjectMeta(), "revision", "foo.txt")
+ obj.Status.Artifact = &artifact
+
+ got, err := r.reconcileDelete(ctx, obj)
+ g.Expect(err).NotTo(HaveOccurred())
+ g.Expect(got).To(Equal(ctrl.Result{}))
+ g.Expect(controllerutil.ContainsFinalizer(obj, sourcev1.SourceFinalizer)).To(BeFalse())
+ g.Expect(obj.Status.Artifact).To(BeNil())
+}
+
+func TestGitRepositoryReconciler_verifyCommitSignature(t *testing.T) {
+ tests := []struct {
+ name string
+ secret *corev1.Secret
+ commit git.Commit
+ beforeFunc func(obj *sourcev1.GitRepository)
+ want ctrl.Result
+ wantErr bool
+ assertConditions []metav1.Condition
+ }{
+ {
+ name: "Valid commit",
+ secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
- Name: mainKey.Name,
- Namespace: mainKey.Namespace,
+ Name: "existing",
},
- Spec: sourcev1.GitRepositorySpec{
- URL: mainRepoURL.String(),
- Interval: metav1.Duration{Duration: indexInterval},
- Reference: &sourcev1.GitRepositoryRef{Branch: "master"},
- Include: []sourcev1.GitRepositoryInclude{
- {
- GitRepositoryRef: meta.LocalObjectReference{
- Name: subKey.Name,
- },
- FromPath: t.fromPath,
- ToPath: t.toPath,
- },
+ },
+ commit: fake.NewCommit(true, "shasum"),
+ beforeFunc: func(obj *sourcev1.GitRepository) {
+ obj.Spec.Interval = metav1.Duration{Duration: interval}
+ obj.Spec.Verification = &sourcev1.GitRepositoryVerification{
+ Mode: "head",
+ SecretRef: meta.LocalObjectReference{
+ Name: "existing",
},
+ }
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.SourceVerifiedCondition, "ValidCommitSignature", "Verified signature of commit \"shasum\""),
+ },
+ },
+ {
+ name: "Invalid commit",
+ secret: &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "existing",
},
- }
- Expect(k8sClient.Create(context.Background(), mainCreated)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), mainCreated)
-
- got := &sourcev1.GitRepository{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), mainKey, got)
- for _, c := range got.Status.Conditions {
- if c.Reason == sourcev1.GitOperationSucceedReason {
- return true
- }
+ },
+ commit: fake.NewCommit(false, "shasum"),
+ beforeFunc: func(obj *sourcev1.GitRepository) {
+ obj.Spec.Interval = metav1.Duration{Duration: interval}
+ obj.Spec.Verification = &sourcev1.GitRepositoryVerification{
+ Mode: "head",
+ SecretRef: meta.LocalObjectReference{
+ Name: "existing",
+ },
+ }
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceVerifiedCondition, "InvalidCommitSignature", "Commit signature verification failed: invalid signature"),
+ },
+ },
+ {
+ name: "Non existing secret",
+ beforeFunc: func(obj *sourcev1.GitRepository) {
+ obj.Spec.Interval = metav1.Duration{Duration: interval}
+ obj.Spec.Verification = &sourcev1.GitRepositoryVerification{
+ Mode: "head",
+ SecretRef: meta.LocalObjectReference{
+ Name: "none-existing",
+ },
}
- return false
- }, timeout, interval).Should(BeTrue())
-
- // check the contents of the repository
- res, err := http.Get(got.Status.URL)
- Expect(err).NotTo(HaveOccurred())
- Expect(res.StatusCode).To(Equal(http.StatusOK))
- tmp, err := ioutil.TempDir("", "flux-test")
- Expect(err).NotTo(HaveOccurred())
- defer os.RemoveAll(tmp)
- _, err = untar.Untar(res.Body, filepath.Join(tmp, "tar"))
- Expect(err).NotTo(HaveOccurred())
- for _, v := range t.checkFiles {
- Expect(filepath.Join(tmp, "tar", v)).To(BeAnExistingFile())
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceVerifiedCondition, "FailedToGetSecret", "PGP public keys secret error: secrets \"none-existing\" not found"),
+ },
+ },
+ {
+ name: "Nil verification in spec deletes SourceVerified condition",
+ beforeFunc: func(obj *sourcev1.GitRepository) {
+ obj.Spec.Interval = metav1.Duration{Duration: interval}
+ conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, "Foo", "")
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ assertConditions: []metav1.Condition{},
+ },
+ {
+ name: "Empty verification mode in spec deletes SourceVerified condition",
+ beforeFunc: func(obj *sourcev1.GitRepository) {
+ obj.Spec.Interval = metav1.Duration{Duration: interval}
+ obj.Spec.Verification = &sourcev1.GitRepositoryVerification{}
+ conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, "Foo", "")
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ assertConditions: []metav1.Condition{},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ builder := fakeclient.NewClientBuilder().WithScheme(env.GetScheme())
+ if tt.secret != nil {
+ builder.WithObjects(tt.secret)
}
- // add new file to check that the change is reconciled
- ff, err = subFs.Create(subFs.Join(t.fromPath, "test"))
- Expect(err).NotTo(HaveOccurred())
- err = ff.Close()
- Expect(err).NotTo(HaveOccurred())
- _, err = subWt.Add(subFs.Join(t.fromPath, "test"))
- Expect(err).NotTo(HaveOccurred())
-
- hash, err := subWt.Commit("Sample", &git.CommitOptions{Author: &object.Signature{
- Name: "John Doe",
- Email: "john@example.com",
- When: time.Now(),
- }})
- Expect(err).NotTo(HaveOccurred())
-
- err = subRemote.Push(&git.PushOptions{
- RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"},
- })
- Expect(err).NotTo(HaveOccurred())
-
- got = &sourcev1.GitRepository{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), mainKey, got)
- if got.Status.IncludedArtifacts[0].Revision == fmt.Sprintf("master/%s", hash.String()) {
- for _, c := range got.Status.Conditions {
- if c.Reason == sourcev1.GitOperationSucceedReason {
- return true
- }
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
-
- // get the main repository artifact
- res, err = http.Get(got.Status.URL)
- Expect(err).NotTo(HaveOccurred())
- Expect(res.StatusCode).To(Equal(http.StatusOK))
- tmp, err = ioutil.TempDir("", "flux-test")
- Expect(err).NotTo(HaveOccurred())
- defer os.RemoveAll(tmp)
- _, err = untar.Untar(res.Body, filepath.Join(tmp, "tar"))
- Expect(err).NotTo(HaveOccurred())
- Expect(filepath.Join(tmp, "tar", t.toPath, "test")).To(BeAnExistingFile())
- },
- Entry("only to path", includeTestCase{
- fromPath: "",
- toPath: "sub",
- createFiles: []string{"dir1", "dir2"},
- checkFiles: []string{"sub/dir1", "sub/dir2"},
- }),
- Entry("to nested path", includeTestCase{
- fromPath: "",
- toPath: "sub/nested",
- createFiles: []string{"dir1", "dir2"},
- checkFiles: []string{"sub/nested/dir1", "sub/nested/dir2"},
- }),
- Entry("from and to path", includeTestCase{
- fromPath: "nested",
- toPath: "sub",
- createFiles: []string{"dir1", "nested/dir2", "nested/dir3", "nested/foo/bar"},
- checkFiles: []string{"sub/dir2", "sub/dir3", "sub/foo/bar"},
- }),
- )
+ r := &GitRepositoryReconciler{
+ Client: builder.Build(),
+ }
+
+ obj := &sourcev1.GitRepository{
+ ObjectMeta: metav1.ObjectMeta{
+ GenerateName: "verify-commit-",
+ Generation: 1,
+ },
+ Status: sourcev1.GitRepositoryStatus{},
+ }
+
+ if tt.beforeFunc != nil {
+ tt.beforeFunc(obj)
+ }
+
+ got, err := r.verifyCommitSignature(logr.NewContext(ctx, log.NullLogger{}), obj, tt.commit)
+ g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions))
+ g.Expect(err != nil).To(Equal(tt.wantErr))
+ g.Expect(got).To(Equal(tt.want))
+ })
+ }
+}
+
+// helpers
+
+func initGitRepo(server *gittestserver.GitServer, fixture, branch, repositoryPath string) (*gogit.Repository, error) {
+ fs := memfs.New()
+ repo, err := gogit.Init(memory.NewStorage(), fs)
+ if err != nil {
+ return nil, err
+ }
+
+ branchRef := plumbing.NewBranchReferenceName(branch)
+ if err = repo.CreateBranch(&config.Branch{
+ Name: branch,
+ Remote: gogit.DefaultRemoteName,
+ Merge: branchRef,
+ }); err != nil {
+ return nil, err
+ }
+
+ err = commitFromFixture(repo, fixture)
+ if err != nil {
+ return nil, err
+ }
+
+ if server.HTTPAddress() == "" {
+ if err = server.StartHTTP(); err != nil {
+ return nil, err
+ }
+ defer server.StopHTTP()
+ }
+ if _, err = repo.CreateRemote(&config.RemoteConfig{
+ Name: gogit.DefaultRemoteName,
+ URLs: []string{server.HTTPAddressWithCredentials() + repositoryPath},
+ }); err != nil {
+ return nil, err
+ }
+
+ if err = repo.Push(&gogit.PushOptions{
+ RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*"},
+ }); err != nil {
+ return nil, err
+ }
+
+ return repo, nil
+}
+
+func Test_commitFromFixture(t *testing.T) {
+ g := NewWithT(t)
+
+ repo, err := gogit.Init(memory.NewStorage(), memfs.New())
+ g.Expect(err).ToNot(HaveOccurred())
+
+ err = commitFromFixture(repo, "testdata/git/repository")
+ g.Expect(err).ToNot(HaveOccurred())
+}
+
+func commitFromFixture(repo *gogit.Repository, fixture string) error {
+ working, err := repo.Worktree()
+ if err != nil {
+ return err
+ }
+ fs := working.Filesystem
+
+ if err = filepath.Walk(fixture, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if info.IsDir() {
+ return fs.MkdirAll(fs.Join(path[len(fixture):]), info.Mode())
+ }
+
+ fileBytes, err := os.ReadFile(path)
+ if err != nil {
+ return err
+ }
+
+ ff, err := fs.Create(path[len(fixture):])
+ if err != nil {
+ return err
+ }
+ defer ff.Close()
+
+ _, err = ff.Write(fileBytes)
+ return err
+ }); err != nil {
+ return err
+ }
+
+ _, err = working.Add(".")
+ if err != nil {
+ return err
+ }
+
+ if _, err = working.Commit("Fixtures from "+fixture, &gogit.CommitOptions{
+ Author: &object.Signature{
+ Name: "Jane Doe",
+ Email: "jane@example.com",
+ When: time.Now(),
+ },
+ }); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func remoteBranchForHead(repo *gogit.Repository, head *plumbing.Reference, branch string) error {
+ refSpec := fmt.Sprintf("%s:refs/heads/%s", head.Name(), branch)
+ return repo.Push(&gogit.PushOptions{
+ RemoteName: "origin",
+ RefSpecs: []config.RefSpec{config.RefSpec(refSpec)},
+ Force: true,
+ })
+}
+
+func remoteTagForHead(repo *gogit.Repository, head *plumbing.Reference, tag string) error {
+ if _, err := repo.CreateTag(tag, head.Hash(), &gogit.CreateTagOptions{
+ // Not setting this seems to make things flaky
+ // Expected success, but got an error:
+ // <*errors.errorString | 0xc0000f6350>: {
+ // s: "tagger field is required",
+ // }
+ // tagger field is required
+ Tagger: &object.Signature{
+ Name: "Jane Doe",
+ Email: "jane@example.com",
+ When: time.Now(),
+ },
+ Message: tag,
+ }); err != nil {
+ return err
+ }
+ refSpec := fmt.Sprintf("refs/tags/%[1]s:refs/tags/%[1]s", tag)
+ return repo.Push(&gogit.PushOptions{
+ RefSpecs: []config.RefSpec{config.RefSpec(refSpec)},
})
-})
+}
diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go
index 8260a0f0a..479e9aa37 100644
--- a/controllers/helmchart_controller.go
+++ b/controllers/helmchart_controller.go
@@ -19,28 +19,20 @@ package controllers
import (
"context"
"fmt"
- "io"
- "io/ioutil"
- "net/url"
"os"
- "path/filepath"
"regexp"
- "strings"
"time"
- securejoin "github.com/cyphar/filepath-securejoin"
+ "github.com/fluxcd/pkg/apis/meta"
+ "github.com/fluxcd/pkg/runtime/conditions"
+ helper "github.com/fluxcd/pkg/runtime/controller"
+ "github.com/fluxcd/pkg/runtime/events"
+ "github.com/fluxcd/pkg/runtime/patch"
+ "github.com/fluxcd/pkg/runtime/predicates"
"github.com/go-logr/logr"
- helmchart "helm.sh/helm/v3/pkg/chart"
- "helm.sh/helm/v3/pkg/chart/loader"
- "helm.sh/helm/v3/pkg/chartutil"
"helm.sh/helm/v3/pkg/getter"
- corev1 "k8s.io/api/core/v1"
- apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/runtime"
- "k8s.io/apimachinery/pkg/types"
- kuberecorder "k8s.io/client-go/tools/record"
- "k8s.io/client-go/tools/reference"
+ kerrors "k8s.io/apimachinery/pkg/util/errors"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -50,14 +42,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
- "sigs.k8s.io/yaml"
-
- "github.com/fluxcd/pkg/apis/meta"
- "github.com/fluxcd/pkg/runtime/events"
- "github.com/fluxcd/pkg/runtime/metrics"
- "github.com/fluxcd/pkg/runtime/predicates"
- "github.com/fluxcd/pkg/runtime/transform"
- "github.com/fluxcd/pkg/untar"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
"github.com/fluxcd/source-controller/internal/helm"
@@ -71,12 +55,24 @@ import (
// HelmChartReconciler reconciles a HelmChart object
type HelmChartReconciler struct {
client.Client
- Scheme *runtime.Scheme
- Storage *Storage
- Getters getter.Providers
- EventRecorder kuberecorder.EventRecorder
- ExternalEventRecorder *events.Recorder
- MetricsRecorder *metrics.Recorder
+ helper.Events
+ helper.Metrics
+
+ Getters getter.Providers
+ Storage *Storage
+}
+
+type HelmChartReconcilerOptions struct {
+ MaxConcurrentReconciles int
+}
+
+type unsupportedSourceKindError struct {
+ Kind string
+ Supported []string
+}
+
+func (e unsupportedSourceKindError) Error() string {
+ return fmt.Sprintf("unsupported source kind %q, must be one of: %v", e.Kind, e.Supported)
}
func (r *HelmChartReconciler) SetupWithManager(mgr ctrl.Manager) error {
@@ -116,668 +112,231 @@ func (r *HelmChartReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, opts
Complete(r)
}
-func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, retErr error) {
start := time.Now()
log := logr.FromContext(ctx)
- var chart sourcev1.HelmChart
- if err := r.Get(ctx, req.NamespacedName, &chart); err != nil {
- return ctrl.Result{Requeue: true}, client.IgnoreNotFound(err)
+ // Fetch the HelmChart
+ obj := &sourcev1.HelmChart{}
+ if err := r.Get(ctx, req.NamespacedName, obj); err != nil {
+ return ctrl.Result{}, client.IgnoreNotFound(err)
}
// Record suspended status metric
- defer r.recordSuspension(ctx, chart)
-
- // Add our finalizer if it does not exist
- if !controllerutil.ContainsFinalizer(&chart, sourcev1.SourceFinalizer) {
- controllerutil.AddFinalizer(&chart, sourcev1.SourceFinalizer)
- if err := r.Update(ctx, &chart); err != nil {
- log.Error(err, "unable to register finalizer")
- return ctrl.Result{}, err
- }
- }
-
- // Examine if the object is under deletion
- if !chart.ObjectMeta.DeletionTimestamp.IsZero() {
- return r.reconcileDelete(ctx, chart)
- }
+ r.RecordSuspend(ctx, obj, obj.Spec.Suspend)
- // Return early if the object is suspended.
- if chart.Spec.Suspend {
+ // Return early if the object is suspended
+ if obj.Spec.Suspend {
log.Info("Reconciliation is suspended for this object")
return ctrl.Result{}, nil
}
- // Record reconciliation duration
- if r.MetricsRecorder != nil {
- objRef, err := reference.GetReference(r.Scheme, &chart)
- if err != nil {
- return ctrl.Result{}, err
- }
- defer r.MetricsRecorder.RecordDuration(*objRef, start)
- }
-
- // Conditionally set progressing condition in status
- resetChart, changed := r.resetStatus(chart)
- if changed {
- chart = resetChart
- if err := r.updateStatus(ctx, req, chart.Status); err != nil {
- log.Error(err, "unable to update status")
- return ctrl.Result{Requeue: true}, err
- }
- r.recordReadiness(ctx, chart)
- }
-
- // Record the value of the reconciliation request, if any
- // TODO(hidde): would be better to defer this in combination with
- // always patching the status sub-resource after a reconciliation.
- if v, ok := meta.ReconcileAnnotationValue(chart.GetAnnotations()); ok {
- chart.Status.SetLastHandledReconcileRequest(v)
- }
-
- // Purge all but current artifact from storage
- if err := r.gc(chart); err != nil {
- log.Error(err, "unable to purge old artifacts")
- }
-
- // Retrieve the source
- source, err := r.getSource(ctx, chart)
+ // Initialize the patch helper
+ patchHelper, err := patch.NewHelper(obj, r.Client)
if err != nil {
- chart = sourcev1.HelmChartNotReady(*chart.DeepCopy(), sourcev1.ChartPullFailedReason, err.Error())
- if err := r.updateStatus(ctx, req, chart.Status); err != nil {
- log.Error(err, "unable to update status")
- }
- return ctrl.Result{Requeue: true}, err
- }
-
- // Assert source is ready
- if source.GetArtifact() == nil {
- err = fmt.Errorf("no artifact found for source `%s` kind '%s'",
- chart.Spec.SourceRef.Name, chart.Spec.SourceRef.Kind)
- chart = sourcev1.HelmChartNotReady(*chart.DeepCopy(), sourcev1.ChartPullFailedReason, err.Error())
- if err := r.updateStatus(ctx, req, chart.Status); err != nil {
- log.Error(err, "unable to update status")
- }
- r.recordReadiness(ctx, chart)
- return ctrl.Result{Requeue: true}, err
- }
-
- // Perform the reconciliation for the chart source type
- var reconciledChart sourcev1.HelmChart
- var reconcileErr error
- switch typedSource := source.(type) {
- case *sourcev1.HelmRepository:
- // TODO: move this to a validation webhook once the discussion around
- // certificates has settled: https://github.com/fluxcd/image-reflector-controller/issues/69
- if err := validHelmChartName(chart.Spec.Chart); err != nil {
- reconciledChart = sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error())
- log.Error(err, "validation failed")
- if err := r.updateStatus(ctx, req, reconciledChart.Status); err != nil {
- log.Info(fmt.Sprintf("%v", reconciledChart.Status))
- log.Error(err, "unable to update status")
- return ctrl.Result{Requeue: true}, err
- }
- r.event(ctx, reconciledChart, events.EventSeverityError, err.Error())
- r.recordReadiness(ctx, reconciledChart)
- // Do not requeue as there is no chance on recovery.
- return ctrl.Result{Requeue: false}, nil
- }
- reconciledChart, reconcileErr = r.reconcileFromHelmRepository(ctx, *typedSource, *chart.DeepCopy(), changed)
- case *sourcev1.GitRepository, *sourcev1.Bucket:
- reconciledChart, reconcileErr = r.reconcileFromTarballArtifact(ctx, *typedSource.GetArtifact(),
- *chart.DeepCopy(), changed)
- default:
- err := fmt.Errorf("unable to reconcile unsupported source reference kind '%s'", chart.Spec.SourceRef.Kind)
- return ctrl.Result{Requeue: false}, err
- }
-
- // Update status with the reconciliation result
- if err := r.updateStatus(ctx, req, reconciledChart.Status); err != nil {
- log.Error(err, "unable to update status")
- return ctrl.Result{Requeue: true}, err
- }
-
- // If reconciliation failed, record the failure and requeue immediately
- if reconcileErr != nil {
- r.event(ctx, reconciledChart, events.EventSeverityError, reconcileErr.Error())
- r.recordReadiness(ctx, reconciledChart)
- return ctrl.Result{Requeue: true}, reconcileErr
- }
-
- // Emit an event if we did not have an artifact before, or the revision has changed
- if (chart.GetArtifact() == nil && reconciledChart.GetArtifact() != nil) ||
- (chart.GetArtifact() != nil && reconciledChart.GetArtifact() != nil && reconciledChart.GetArtifact().Revision != chart.GetArtifact().Revision) {
- r.event(ctx, reconciledChart, events.EventSeverityInfo, sourcev1.HelmChartReadyMessage(reconciledChart))
+ return ctrl.Result{}, err
}
- r.recordReadiness(ctx, reconciledChart)
-
- log.Info(fmt.Sprintf("Reconciliation finished in %s, next run in %s",
- time.Now().Sub(start).String(),
- chart.GetInterval().Duration.String(),
- ))
- return ctrl.Result{RequeueAfter: chart.GetInterval().Duration}, nil
-}
-
-type HelmChartReconcilerOptions struct {
- MaxConcurrentReconciles int
-}
-func (r *HelmChartReconciler) getSource(ctx context.Context, chart sourcev1.HelmChart) (sourcev1.Source, error) {
- var source sourcev1.Source
- namespacedName := types.NamespacedName{
- Namespace: chart.GetNamespace(),
- Name: chart.Spec.SourceRef.Name,
- }
- switch chart.Spec.SourceRef.Kind {
- case sourcev1.HelmRepositoryKind:
- var repository sourcev1.HelmRepository
- err := r.Client.Get(ctx, namespacedName, &repository)
- if err != nil {
- return source, fmt.Errorf("failed to retrieve source: %w", err)
+ // Always attempt to patch the object and status after each
+ // reconciliation
+ defer func() {
+ // Record the value of the reconciliation request, if any
+ if v, ok := meta.ReconcileAnnotationValue(obj.GetAnnotations()); ok {
+ obj.Status.SetLastHandledReconcileRequest(v)
}
- source = &repository
- case sourcev1.GitRepositoryKind:
- var repository sourcev1.GitRepository
- err := r.Client.Get(ctx, namespacedName, &repository)
- if err != nil {
- return source, fmt.Errorf("failed to retrieve source: %w", err)
- }
- source = &repository
- case sourcev1.BucketKind:
- var bucket sourcev1.Bucket
- err := r.Client.Get(ctx, namespacedName, &bucket)
- if err != nil {
- return source, fmt.Errorf("failed to retrieve source: %w", err)
- }
- source = &bucket
- default:
- return source, fmt.Errorf("source `%s` kind '%s' not supported",
- chart.Spec.SourceRef.Name, chart.Spec.SourceRef.Kind)
- }
- return source, nil
-}
-func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context,
- repository sourcev1.HelmRepository, chart sourcev1.HelmChart, force bool) (sourcev1.HelmChart, error) {
- // Configure ChartRepository getter options
- clientOpts := []getter.Option{
- getter.WithURL(repository.Spec.URL),
- getter.WithTimeout(repository.Spec.Timeout.Duration),
- getter.WithPassCredentialsAll(repository.Spec.PassCredentials),
- }
- if secret, err := r.getHelmRepositorySecret(ctx, &repository); err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err
- } else if secret != nil {
- opts, cleanup, err := helm.ClientOptionsFromSecret(*secret)
- if err != nil {
- err = fmt.Errorf("auth options error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err
+ // Summarize Ready condition
+ conditions.SetSummary(obj,
+ meta.ReadyCondition,
+ conditions.WithConditions(
+ sourcev1.ArtifactAvailableCondition,
+ sourcev1.ChartReconciled,
+ sourcev1.SourceAvailableCondition,
+ ),
+ )
+
+ // Patch the object, ignoring conflicts on the conditions owned by
+ // this controller
+ patchOpts := []patch.Option{
+ patch.WithOwnedConditions{
+ Conditions: []string{
+ sourcev1.ArtifactAvailableCondition,
+ sourcev1.SourceAvailableCondition,
+ sourcev1.ValuesFilesMergedCondition,
+ sourcev1.DependenciesBuildCondition,
+ sourcev1.ChartPackagedCondition,
+ sourcev1.ChartReconciled,
+ meta.ReadyCondition,
+ meta.ReconcilingCondition,
+ meta.StalledCondition,
+ },
+ },
}
- defer cleanup()
- clientOpts = append(clientOpts, opts...)
- }
- // Initialize the chart repository and load the index file
- chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Getters, clientOpts)
- if err != nil {
- switch err.(type) {
- case *url.Error:
- return sourcev1.HelmChartNotReady(chart, sourcev1.URLInvalidReason, err.Error()), err
- default:
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
+ // Determine if the resource is still being reconciled, or if
+ // it has stalled, and record this observation
+ if retErr == nil && (result.IsZero() || !result.Requeue) {
+ // We are no longer reconciling
+ conditions.Delete(obj, meta.ReconcilingCondition)
+
+ // We have now observed this generation
+ patchOpts = append(patchOpts, patch.WithStatusObservedGeneration{})
+
+ readyCondition := conditions.Get(obj, meta.ReadyCondition)
+ switch readyCondition.Status {
+ case metav1.ConditionFalse:
+ // As we are no longer reconciling and the end-state
+ // is not ready, the reconciliation has stalled
+ conditions.MarkStalled(obj, readyCondition.Reason, readyCondition.Message)
+ case metav1.ConditionTrue:
+ // As we are no longer reconciling and the end-state
+ // is ready, the reconciliation is no longer stalled
+ conditions.Delete(obj, meta.StalledCondition)
+ }
}
- }
- indexFile, err := os.Open(r.Storage.LocalPath(*repository.GetArtifact()))
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- b, err := ioutil.ReadAll(indexFile)
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- if err = chartRepo.LoadIndex(b); err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- // Lookup the chart version in the chart repository index
- chartVer, err := chartRepo.Get(chart.Spec.Chart, chart.Spec.Version)
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
-
- // Return early if the revision is still the same as the current artifact
- newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.GetObjectMeta(), chartVer.Version,
- fmt.Sprintf("%s-%s.tgz", chartVer.Name, chartVer.Version))
- if !force && repository.GetArtifact().HasRevision(newArtifact.Revision) {
- if newArtifact.URL != chart.GetArtifact().URL {
- r.Storage.SetArtifactURL(chart.GetArtifact())
- chart.Status.URL = r.Storage.SetHostname(chart.Status.URL)
+ // Finally, patch the resource
+ if err := patchHelper.Patch(ctx, obj, patchOpts...); err != nil {
+ retErr = kerrors.NewAggregate([]error{retErr, err})
}
- return chart, nil
- }
- // Ensure artifact directory exists
- err = r.Storage.MkdirAll(newArtifact)
- if err != nil {
- err = fmt.Errorf("unable to create chart directory: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
+ // Always record readiness and duration metrics
+ r.Metrics.RecordReadiness(ctx, obj)
+ r.Metrics.RecordDuration(ctx, obj, start)
+ }()
- // Acquire a lock for the artifact
- unlock, err := r.Storage.Lock(newArtifact)
- if err != nil {
- err = fmt.Errorf("unable to acquire lock: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ // Add finalizer first if not exist to avoid the race condition
+ // between init and delete
+ if !controllerutil.ContainsFinalizer(obj, sourcev1.SourceFinalizer) {
+ controllerutil.AddFinalizer(obj, sourcev1.SourceFinalizer)
+ return ctrl.Result{Requeue: true}, nil
}
- defer unlock()
- // Attempt to download the chart
- res, err := chartRepo.DownloadChart(chartVer)
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
+ // Examine if the object is under deletion
+ if !obj.ObjectMeta.DeletionTimestamp.IsZero() {
+ return r.reconcileDelete(ctx, obj)
}
- tmpFile, err := ioutil.TempFile("", fmt.Sprintf("%s-%s-", chart.Namespace, chart.Name))
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- defer os.RemoveAll(tmpFile.Name())
- if _, err = io.Copy(tmpFile, res); err != nil {
- tmpFile.Close()
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- tmpFile.Close()
-
- // Check if we need to repackage the chart with the declared defaults files.
- var (
- pkgPath = tmpFile.Name()
- readyReason = sourcev1.ChartPullSucceededReason
- readyMessage = fmt.Sprintf("Fetched revision: %s", newArtifact.Revision)
- )
-
- switch {
- case len(chart.GetValuesFiles()) > 0:
- valuesMap := make(map[string]interface{})
-
- // Load the chart
- helmChart, err := loader.LoadFile(pkgPath)
- if err != nil {
- err = fmt.Errorf("load chart error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- for _, v := range chart.GetValuesFiles() {
- if v == "values.yaml" {
- valuesMap = transform.MergeMaps(valuesMap, helmChart.Values)
- continue
- }
-
- var valuesData []byte
- cfn := filepath.Clean(v)
- for _, f := range helmChart.Files {
- if f.Name == cfn {
- valuesData = f.Data
- break
- }
- }
- if valuesData == nil {
- err = fmt.Errorf("invalid values file path: %s", v)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
-
- yamlMap := make(map[string]interface{})
- err = yaml.Unmarshal(valuesData, &yamlMap)
- if err != nil {
- err = fmt.Errorf("unmarshaling values from %s failed: %w", v, err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
-
- valuesMap = transform.MergeMaps(valuesMap, yamlMap)
- }
-
- yamlBytes, err := yaml.Marshal(valuesMap)
- if err != nil {
- err = fmt.Errorf("marshaling values failed: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err
- }
-
- // Overwrite values file
- if changed, err := helm.OverwriteChartDefaultValues(helmChart, yamlBytes); err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err
- } else if !changed {
- break
- }
-
- // Create temporary working directory
- tmpDir, err := ioutil.TempDir("", fmt.Sprintf("%s-%s-", chart.Namespace, chart.Name))
- if err != nil {
- err = fmt.Errorf("tmp dir error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- defer os.RemoveAll(tmpDir)
-
- // Package the chart with the new default values
- pkgPath, err = chartutil.Save(helmChart, tmpDir)
- if err != nil {
- err = fmt.Errorf("chart package error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err
- }
+ // Reconcile actual object
+ return r.reconcile(ctx, obj)
+}
- // Copy the packaged chart to the artifact path
- if err := r.Storage.CopyFromPath(&newArtifact, pkgPath); err != nil {
- err = fmt.Errorf("failed to write chart package to storage: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
+func (r *HelmChartReconciler) reconcile(ctx context.Context, obj *sourcev1.HelmChart) (ctrl.Result, error) {
+ // Mark the resource as under reconciliation
+ conditions.MarkReconciling(obj, "Reconciling", "")
- readyMessage = fmt.Sprintf("Fetched and packaged revision: %s", newArtifact.Revision)
- readyReason = sourcev1.ChartPackageSucceededReason
+ // Reconcile the storage data
+ if result, err := r.reconcileStorage(ctx, obj); err != nil {
+ return result, err
}
- // Write artifact to storage
- if err := r.Storage.CopyFromPath(&newArtifact, pkgPath); err != nil {
- err = fmt.Errorf("unable to write chart file: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ // Reconcile the source
+ var sourcePath string
+ defer os.RemoveAll(sourcePath)
+ if result, err := r.reconcileSource(ctx, obj, &sourcePath); err != nil || conditions.IsFalse(obj, sourcev1.SourceAvailableCondition) {
+ return result, err
}
- // Update symlink
- chartUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", chartVer.Name))
- if err != nil {
- err = fmt.Errorf("storage error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ // Reconcile the chart using the source data
+ var artifact sourcev1.Artifact
+ var resultPath string
+ defer os.RemoveAll(resultPath)
+ if result, err := r.reconcileChart(ctx, obj, sourcePath, &artifact, &resultPath); err != nil {
+ return result, err
}
- return sourcev1.HelmChartReady(chart, newArtifact, chartUrl, readyReason, readyMessage), nil
+ // Reconcile artifact to storage
+ return r.reconcileArtifact(ctx, obj, artifact, resultPath)
}
-func (r *HelmChartReconciler) reconcileFromTarballArtifact(ctx context.Context,
- artifact sourcev1.Artifact, chart sourcev1.HelmChart, force bool) (sourcev1.HelmChart, error) {
- // Create temporary working directory
- tmpDir, err := ioutil.TempDir("", fmt.Sprintf("%s-%s-", chart.Namespace, chart.Name))
- if err != nil {
- err = fmt.Errorf("tmp dir error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+// reconcileStorage reconciles the storage data for the given object
+// by garbage collecting previous advertised artifact(s) from storage,
+// observing if the artifact in the status still exists, and
+// ensuring the URLs are up-to-date with the current hostname
+// configuration.
+func (r *HelmChartReconciler) reconcileStorage(ctx context.Context, obj *sourcev1.HelmChart) (ctrl.Result, error) {
+ // Garbage collect previous advertised artifact(s) from storage
+ if err := r.garbageCollect(obj); err != nil {
+ r.Events.Eventf(ctx, obj, events.EventSeverityError, "GarbageCollectionFailed", "Garbage collection failed: %s", err)
}
- defer os.RemoveAll(tmpDir)
- // Open the tarball artifact file and untar files into working directory
- f, err := os.Open(r.Storage.LocalPath(artifact))
- if err != nil {
- err = fmt.Errorf("artifact open error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- if _, err = untar.Untar(f, tmpDir); err != nil {
- f.Close()
- err = fmt.Errorf("artifact untar error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ // Determine if the advertised artifact is still in storage
+ if artifact := obj.GetArtifact(); artifact != nil && !r.Storage.ArtifactExist(*artifact) {
+ obj.Status.Artifact = nil
+ obj.Status.URL = ""
}
- f.Close()
- // Load the chart
- chartPath, err := securejoin.SecureJoin(tmpDir, chart.Spec.Chart)
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- chartFileInfo, err := os.Stat(chartPath)
- if err != nil {
- err = fmt.Errorf("chart location read error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ // Record that we do not have an artifact
+ if obj.GetArtifact() == nil {
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, "NoArtifactFound", "No artifact for resource in storage")
+ return ctrl.Result{Requeue: true}, nil
}
- helmChart, err := loader.Load(chartPath)
- if err != nil {
- err = fmt.Errorf("load chart error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
-
- // Return early if the revision is still the same as the current chart artifact
- newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.ObjectMeta.GetObjectMeta(), helmChart.Metadata.Version,
- fmt.Sprintf("%s-%s.tgz", helmChart.Metadata.Name, helmChart.Metadata.Version))
- if !force && apimeta.IsStatusConditionTrue(chart.Status.Conditions, meta.ReadyCondition) && chart.GetArtifact().HasRevision(newArtifact.Revision) {
- if newArtifact.URL != artifact.URL {
- r.Storage.SetArtifactURL(chart.GetArtifact())
- chart.Status.URL = r.Storage.SetHostname(chart.Status.URL)
- }
- return chart, nil
- }
-
- // Either (re)package the chart with the declared default values file,
- // or write the chart directly to storage.
- pkgPath := chartPath
- isValuesFileOverriden := false
- if len(chart.GetValuesFiles()) > 0 {
- valuesMap := make(map[string]interface{})
- for _, v := range chart.GetValuesFiles() {
- srcPath, err := securejoin.SecureJoin(tmpDir, v)
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- if f, err := os.Stat(srcPath); os.IsNotExist(err) || !f.Mode().IsRegular() {
- err = fmt.Errorf("invalid values file path: %s", v)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
-
- valuesData, err := ioutil.ReadFile(srcPath)
- if err != nil {
- err = fmt.Errorf("failed to read from values file '%s': %w", v, err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
-
- yamlMap := make(map[string]interface{})
- err = yaml.Unmarshal(valuesData, &yamlMap)
- if err != nil {
- err = fmt.Errorf("unmarshaling values from %s failed: %w", v, err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
-
- valuesMap = transform.MergeMaps(valuesMap, yamlMap)
- }
-
- yamlBytes, err := yaml.Marshal(valuesMap)
- if err != nil {
- err = fmt.Errorf("marshaling values failed: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err
- }
-
- isValuesFileOverriden, err = helm.OverwriteChartDefaultValues(helmChart, yamlBytes)
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err
- }
- }
-
- isDir := chartFileInfo.IsDir()
- switch {
- case isDir:
- // Determine chart dependencies
- deps := helmChart.Dependencies()
- reqs := helmChart.Metadata.Dependencies
- lock := helmChart.Lock
- if lock != nil {
- // Load from lockfile if exists
- reqs = lock.Dependencies
- }
- var dwr []*helm.DependencyWithRepository
- for _, dep := range reqs {
- // Exclude existing dependencies
- found := false
- for _, existing := range deps {
- if existing.Name() == dep.Name {
- found = true
- }
- }
- if found {
- continue
- }
-
- // Continue loop if file scheme detected
- if dep.Repository == "" || strings.HasPrefix(dep.Repository, "file://") {
- dwr = append(dwr, &helm.DependencyWithRepository{
- Dependency: dep,
- Repository: nil,
- })
- continue
- }
-
- // Discover existing HelmRepository by URL
- repository, err := r.resolveDependencyRepository(ctx, dep, chart.Namespace)
- if err != nil {
- repository = &sourcev1.HelmRepository{
- Spec: sourcev1.HelmRepositorySpec{
- URL: dep.Repository,
- Timeout: &metav1.Duration{Duration: 60 * time.Second},
- },
- }
- }
- // Configure ChartRepository getter options
- clientOpts := []getter.Option{
- getter.WithURL(repository.Spec.URL),
- getter.WithTimeout(repository.Spec.Timeout.Duration),
- getter.WithPassCredentialsAll(repository.Spec.PassCredentials),
- }
- if secret, err := r.getHelmRepositorySecret(ctx, repository); err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err
- } else if secret != nil {
- opts, cleanup, err := helm.ClientOptionsFromSecret(*secret)
- if err != nil {
- err = fmt.Errorf("auth options error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err
- }
- defer cleanup()
- clientOpts = append(clientOpts, opts...)
- }
-
- // Initialize the chart repository and load the index file
- chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Getters, clientOpts)
- if err != nil {
- switch err.(type) {
- case *url.Error:
- return sourcev1.HelmChartNotReady(chart, sourcev1.URLInvalidReason, err.Error()), err
- default:
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- }
- if repository.Status.Artifact != nil {
- indexFile, err := os.Open(r.Storage.LocalPath(*repository.GetArtifact()))
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- b, err := ioutil.ReadAll(indexFile)
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- if err = chartRepo.LoadIndex(b); err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- } else {
- // Download index
- err = chartRepo.DownloadIndex()
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- }
-
- dwr = append(dwr, &helm.DependencyWithRepository{
- Dependency: dep,
- Repository: chartRepo,
- })
- }
+ // Always update URLs to ensure hostname is up-to-date
+ r.Storage.SetArtifactURL(obj.GetArtifact())
+ obj.Status.URL = r.Storage.SetHostname(obj.Status.URL)
- // Construct dependencies for chart if any
- if len(dwr) > 0 {
- dm := &helm.DependencyManager{
- WorkingDir: tmpDir,
- ChartPath: chart.Spec.Chart,
- Chart: helmChart,
- Dependencies: dwr,
- }
- err = dm.Build(ctx)
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- }
-
- fallthrough
- case isValuesFileOverriden:
- pkgPath, err = chartutil.Save(helmChart, tmpDir)
- if err != nil {
- err = fmt.Errorf("chart package error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err
- }
- }
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+}
- // Ensure artifact directory exists
- err = r.Storage.MkdirAll(newArtifact)
- if err != nil {
- err = fmt.Errorf("unable to create artifact directory: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+func (r *HelmChartReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.HelmChart, artifact sourcev1.Artifact, path string) (ctrl.Result, error) {
+ // Ensure artifact directory exists and acquire lock
+ if err := r.Storage.MkdirAll(artifact); err != nil {
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to create directory: %s", err.Error())
+ return ctrl.Result{}, err
}
-
- // Acquire a lock for the artifact
- unlock, err := r.Storage.Lock(newArtifact)
+ unlock, err := r.Storage.Lock(artifact)
if err != nil {
- err = fmt.Errorf("unable to acquire lock: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to acquire lock: %s", err.Error())
+ return ctrl.Result{}, err
}
defer unlock()
// Copy the packaged chart to the artifact path
- if err := r.Storage.CopyFromPath(&newArtifact, pkgPath); err != nil {
- err = fmt.Errorf("failed to write chart package to storage: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ if err := r.Storage.CopyFromPath(&artifact, path); err != nil {
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Unable to write chart to storage: %s", err.Error())
+ return ctrl.Result{}, err
}
- // Update symlink
- cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", helmChart.Metadata.Name))
+ // Record it on the object
+ obj.Status.Artifact = artifact.DeepCopy()
+ conditions.MarkTrue(obj, sourcev1.ArtifactAvailableCondition, sourcev1.ChartPackageSucceededReason, "Artifact revision %s", artifact.Revision)
+ r.Events.EventWithMetaf(ctx, obj, map[string]string{
+ "revision": obj.GetArtifact().Revision,
+ }, events.EventSeverityInfo, sourcev1.ChartPackageSucceededReason, conditions.Get(obj, sourcev1.ArtifactAvailableCondition).Message)
+
+ // Update symlink on a "best effort" basis
+ u, err := r.Storage.Symlink(artifact, "latest.tar.gz")
if err != nil {
- err = fmt.Errorf("storage error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ r.Events.Eventf(ctx, obj, events.EventSeverityError, sourcev1.StorageOperationFailedReason, "Failed to update status URL symlink: %s", err)
+ }
+ if u != "" {
+ obj.Status.URL = u
}
- message := fmt.Sprintf("Fetched and packaged revision: %s", newArtifact.Revision)
- return sourcev1.HelmChartReady(chart, newArtifact, cUrl, sourcev1.ChartPackageSucceededReason, message), nil
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
}
-func (r *HelmChartReconciler) reconcileDelete(ctx context.Context, chart sourcev1.HelmChart) (ctrl.Result, error) {
- // Our finalizer is still present, so lets handle garbage collection
- if err := r.gc(chart); err != nil {
- r.event(ctx, chart, events.EventSeverityError,
- fmt.Sprintf("garbage collection for deleted resource failed: %s", err.Error()))
+func (r *HelmChartReconciler) reconcileDelete(ctx context.Context, obj *sourcev1.HelmChart) (ctrl.Result, error) {
+ // Garbage collect the resource's artifacts
+ if err := r.garbageCollect(obj); err != nil {
+ r.Events.Eventf(ctx, obj, events.EventSeverityError, "GarbageCollectionFailed", "Garbage collection for deleted resource failed: %s", err)
// Return the error so we retry the failed garbage collection
return ctrl.Result{}, err
}
- // Record deleted status
- r.recordReadiness(ctx, chart)
-
- // Remove our finalizer from the list and update it
- controllerutil.RemoveFinalizer(&chart, sourcev1.SourceFinalizer)
- if err := r.Update(ctx, &chart); err != nil {
- return ctrl.Result{}, err
- }
+ // Remove our finalizer from the list
+ controllerutil.RemoveFinalizer(obj, sourcev1.SourceFinalizer)
// Stop reconciliation as the object is being deleted
return ctrl.Result{}, nil
}
-// resetStatus returns a modified v1beta1.HelmChart and a boolean indicating
-// if the status field has been reset.
-func (r *HelmChartReconciler) resetStatus(chart sourcev1.HelmChart) (sourcev1.HelmChart, bool) {
- // We do not have an artifact, or it does no longer exist
- if chart.GetArtifact() == nil || !r.Storage.ArtifactExist(*chart.GetArtifact()) {
- chart = sourcev1.HelmChartProgressing(chart)
- chart.Status.Artifact = nil
- return chart, true
- }
- // The chart specification has changed
- if chart.Generation != chart.Status.ObservedGeneration {
- return sourcev1.HelmChartProgressing(chart), true
- }
- return chart, false
-}
-
-// gc performs a garbage collection for the given v1beta1.HelmChart.
-// It removes all but the current artifact except for when the
-// deletion timestamp is set, which will result in the removal of
-// all artifacts for the resource.
-func (r *HelmChartReconciler) gc(chart sourcev1.HelmChart) error {
+// garbageCollect performs a garbage collection for the given
+// v1beta1.HelmChart. It removes all but the current artifact except
+// for when the deletion timestamp is set, which will result in the
+// removal of all artifacts for the resource.
+func (r *HelmChartReconciler) garbageCollect(chart *sourcev1.HelmChart) error {
if !chart.DeletionTimestamp.IsZero() {
return r.Storage.RemoveAll(r.Storage.NewArtifactFor(chart.Kind, chart.GetObjectMeta(), "", "*"))
}
@@ -787,59 +346,6 @@ func (r *HelmChartReconciler) gc(chart sourcev1.HelmChart) error {
return nil
}
-// event emits a Kubernetes event and forwards the event to notification
-// controller if configured.
-func (r *HelmChartReconciler) event(ctx context.Context, chart sourcev1.HelmChart, severity, msg string) {
- log := logr.FromContext(ctx)
- if r.EventRecorder != nil {
- r.EventRecorder.Eventf(&chart, "Normal", severity, msg)
- }
- if r.ExternalEventRecorder != nil {
- objRef, err := reference.GetReference(r.Scheme, &chart)
- if err != nil {
- log.Error(err, "unable to send event")
- return
- }
-
- if err := r.ExternalEventRecorder.Eventf(*objRef, nil, severity, severity, msg); err != nil {
- log.Error(err, "unable to send event")
- return
- }
- }
-}
-
-func (r *HelmChartReconciler) recordReadiness(ctx context.Context, chart sourcev1.HelmChart) {
- log := logr.FromContext(ctx)
- if r.MetricsRecorder == nil {
- return
- }
- objRef, err := reference.GetReference(r.Scheme, &chart)
- if err != nil {
- log.Error(err, "unable to record readiness metric")
- return
- }
- if rc := apimeta.FindStatusCondition(chart.Status.Conditions, meta.ReadyCondition); rc != nil {
- r.MetricsRecorder.RecordCondition(*objRef, *rc, !chart.DeletionTimestamp.IsZero())
- } else {
- r.MetricsRecorder.RecordCondition(*objRef, metav1.Condition{
- Type: meta.ReadyCondition,
- Status: metav1.ConditionUnknown,
- }, !chart.DeletionTimestamp.IsZero())
- }
-}
-
-func (r *HelmChartReconciler) updateStatus(ctx context.Context, req ctrl.Request, newStatus sourcev1.HelmChartStatus) error {
- var chart sourcev1.HelmChart
- if err := r.Get(ctx, req.NamespacedName, &chart); err != nil {
- return err
- }
-
- patch := client.MergeFrom(chart.DeepCopy())
- chart.Status = newStatus
-
- return r.Status().Patch(ctx, &chart, patch)
-}
-
func (r *HelmChartReconciler) indexHelmRepositoryByURL(o client.Object) []string {
repo, ok := o.(*sourcev1.HelmRepository)
if !ok {
@@ -860,47 +366,6 @@ func (r *HelmChartReconciler) indexHelmChartBySource(o client.Object) []string {
return []string{fmt.Sprintf("%s/%s", hc.Spec.SourceRef.Kind, hc.Spec.SourceRef.Name)}
}
-func (r *HelmChartReconciler) resolveDependencyRepository(ctx context.Context, dep *helmchart.Dependency, namespace string) (*sourcev1.HelmRepository, error) {
- u := helm.NormalizeChartRepositoryURL(dep.Repository)
- if u == "" {
- return nil, fmt.Errorf("invalid repository URL")
- }
-
- listOpts := []client.ListOption{
- client.InNamespace(namespace),
- client.MatchingFields{sourcev1.HelmRepositoryURLIndexKey: u},
- }
- var list sourcev1.HelmRepositoryList
- err := r.Client.List(ctx, &list, listOpts...)
- if err != nil {
- return nil, fmt.Errorf("unable to retrieve HelmRepositoryList: %w", err)
- }
- if len(list.Items) > 0 {
- return &list.Items[0], nil
- }
-
- return nil, fmt.Errorf("no HelmRepository found")
-}
-
-func (r *HelmChartReconciler) getHelmRepositorySecret(ctx context.Context, repository *sourcev1.HelmRepository) (*corev1.Secret, error) {
- if repository.Spec.SecretRef != nil {
- name := types.NamespacedName{
- Namespace: repository.GetNamespace(),
- Name: repository.Spec.SecretRef.Name,
- }
-
- var secret corev1.Secret
- err := r.Client.Get(ctx, name, &secret)
- if err != nil {
- err = fmt.Errorf("auth secret error: %w", err)
- return nil, err
- }
- return &secret, nil
- }
-
- return nil, nil
-}
-
func (r *HelmChartReconciler) requestsForHelmRepositoryChange(o client.Object) []reconcile.Request {
repo, ok := o.(*sourcev1.HelmRepository)
if !ok {
@@ -999,22 +464,3 @@ func validHelmChartName(s string) error {
}
return nil
}
-
-func (r *HelmChartReconciler) recordSuspension(ctx context.Context, chart sourcev1.HelmChart) {
- if r.MetricsRecorder == nil {
- return
- }
- log := logr.FromContext(ctx)
-
- objRef, err := reference.GetReference(r.Scheme, &chart)
- if err != nil {
- log.Error(err, "unable to record suspended metric")
- return
- }
-
- if !chart.DeletionTimestamp.IsZero() {
- r.MetricsRecorder.RecordSuspend(*objRef, false)
- } else {
- r.MetricsRecorder.RecordSuspend(*objRef, chart.Spec.Suspend)
- }
-}
diff --git a/controllers/helmchart_controller_chart.go b/controllers/helmchart_controller_chart.go
new file mode 100644
index 000000000..60c42d02c
--- /dev/null
+++ b/controllers/helmchart_controller_chart.go
@@ -0,0 +1,361 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ securejoin "github.com/cyphar/filepath-securejoin"
+ "github.com/fluxcd/pkg/runtime/conditions"
+ "github.com/fluxcd/pkg/runtime/events"
+ "github.com/fluxcd/pkg/runtime/transform"
+ sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+ "github.com/fluxcd/source-controller/internal/helm"
+ "github.com/go-logr/logr"
+ helmchart "helm.sh/helm/v3/pkg/chart"
+ "helm.sh/helm/v3/pkg/chart/loader"
+ "helm.sh/helm/v3/pkg/chartutil"
+ "helm.sh/helm/v3/pkg/getter"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/types"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/yaml"
+)
+
+func (r *HelmChartReconciler) reconcileChart(ctx context.Context, obj *sourcev1.HelmChart, path string, artifact *sourcev1.Artifact, result *string) (ctrl.Result, error) {
+ if path == "" {
+ logr.FromContext(ctx).Info("No chart: skipping chart reconciliation")
+ }
+
+ // Determine exact chart path
+ pathInfo, err := os.Stat(path)
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.ChartReconciled, sourcev1.StorageOperationFailedReason, "Could not stat path %s: %s", path, err.Error())
+ return ctrl.Result{}, err
+ }
+ chartPath := path
+ if pathInfo.IsDir() {
+ var err error
+ if chartPath, err = securejoin.SecureJoin(chartPath, obj.Spec.Chart); err != nil {
+ return ctrl.Result{}, nil
+ }
+ }
+
+ // Attempt to load chart from the determined path
+ chart, err := loader.Load(chartPath)
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.ChartReconciled, sourcev1.StorageOperationFailedReason, "Could not load Helm chart %s: %s", obj.Spec.Chart, err.Error())
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+ }
+
+ // The artifact is up-to-date
+ if obj.GetArtifact().HasRevision(chart.Metadata.Version) {
+ logr.FromContext(ctx).Info("Artifact up-to-date: skipping chart reconciliation")
+ return ctrl.Result{RequeueAfter: obj.GetInterval().Duration}, nil
+ }
+
+ // Ensure we have all the dependencies and merge any values files
+ if result, err := r.buildChartDependencies(ctx, obj, chart, path, chartPath); err != nil || conditions.IsFalse(obj, sourcev1.ChartReconciled) {
+ return result, err
+ }
+ if result, err := r.mergeChartValuesFiles(ctx, obj, chart, path); err != nil || conditions.IsFalse(obj, sourcev1.ChartReconciled) {
+ return result, err
+ }
+
+ // We need to (re)package the chart
+ if conditions.IsTrue(obj, sourcev1.DependenciesBuildCondition) || conditions.IsTrue(obj, sourcev1.ValuesFilesMergedCondition) {
+ tmpDir, err := os.MkdirTemp("", "helm-chart-pkg-")
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.ChartPackagedCondition, sourcev1.StorageOperationFailedReason, "Could not create temporary directory for packaging operation: %s", err.Error())
+ return ctrl.Result{}, err
+ }
+ chartPath, err = chartutil.Save(chart, tmpDir)
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.ChartPackagedCondition, sourcev1.StorageOperationFailedReason, "Could not package chart: %s", err.Error())
+ return ctrl.Result{}, err
+ }
+ } else {
+ conditions.Delete(obj, sourcev1.ChartPackagedCondition)
+ }
+
+ // Create potential new artifact
+ *artifact = r.Storage.NewArtifactFor(obj.Kind, obj, chart.Metadata.Version, fmt.Sprintf("%s-%s.tgz", chart.Name(), chart.Metadata.Version))
+ *result = chartPath
+ conditions.MarkTrue(obj, sourcev1.ChartReconciled, "Success", "Reconciled Helm chart with revision %s", chart.Metadata.Version)
+
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+}
+
+func (r *HelmChartReconciler) buildChartDependencies(ctx context.Context, obj *sourcev1.HelmChart, chart *helmchart.Chart, path, chartPath string) (ctrl.Result, error) {
+ // Gather information about the chart path
+ chartPathInfo, err := os.Stat(chartPath)
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.ChartReconciled, sourcev1.StorageOperationFailedReason, "Could not stat path %q: %s", chartPath, err.Error())
+ return ctrl.Result{}, err
+ }
+
+ // We only want to build dependencies for chart directories
+ if !chartPathInfo.IsDir() {
+ logr.FromContext(ctx).Info("Chart is already packaged: skipping dependency build")
+ conditions.Delete(obj, sourcev1.DependenciesBuildCondition)
+ return ctrl.Result{}, nil
+ }
+
+ // Collect chart dependency metadata
+ var (
+ deps = chart.Dependencies()
+ reqs = chart.Metadata.Dependencies
+ lock = chart.Lock
+ )
+ if lock != nil {
+ // Load from lockfile if exists
+ reqs = lock.Dependencies
+ }
+
+ // If the number of dependencies equals the number of requests
+ // we already do have all dependencies.
+ if len(deps) == len(reqs) {
+ logr.FromContext(ctx).Info("Chart does already have all dependencies: skipping dependency build")
+ conditions.Delete(obj, sourcev1.DependenciesBuildCondition)
+ return ctrl.Result{}, nil
+ }
+
+ dm := &helm.DependencyManager{
+ WorkingDir: path,
+ ChartPath: obj.Spec.Chart,
+ Chart: chart,
+ }
+
+ tmpDir, err := os.MkdirTemp("", "build-chart-deps-")
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.ChartReconciled, sourcev1.StorageOperationFailedReason, "Could not create temporary directory for dependency credentials: %s", err.Error())
+ return ctrl.Result{}, err
+ }
+
+ for _, dep := range reqs {
+ // Exclude existing dependencies
+ found := false
+ for _, existing := range deps {
+ if existing.Name() == dep.Name {
+ found = true
+ }
+ }
+ if found {
+ continue
+ }
+
+ dwr, err := r.getRepositoryIndex(ctx, obj, dep, tmpDir)
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.ChartReconciled, "IndexFailure", "Could not construct Helm repository index for dependency %q: %s", dep.Name, err.Error())
+ return ctrl.Result{}, err
+ }
+ dm.Dependencies = append(dm.Dependencies, dwr)
+ }
+
+ if len(dm.Dependencies) == 0 {
+ // This should theoretically never happen due to the check we did earlier
+ logr.FromContext(ctx).Info("Chart does already have all dependencies")
+ conditions.Delete(obj, sourcev1.DependenciesBuildCondition)
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+ }
+ if err := dm.Build(ctx); err != nil {
+ conditions.MarkFalse(obj, sourcev1.ChartReconciled, "BuildFailure", "Failed to build dependencies for %q: %s", chart.Name(), err.Error())
+ return ctrl.Result{}, err
+ }
+
+ conditions.MarkTrue(obj, sourcev1.DependenciesBuildCondition, "Success", "Downloaded %d dependencies", len(dm.Dependencies))
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+}
+
+func (r *HelmChartReconciler) getRepositoryIndex(ctx context.Context, obj *sourcev1.HelmChart, dep *helmchart.Dependency, dir string) (*helm.DependencyWithRepository, error) {
+ // Return early if file schema detected
+ if dep.Repository == "" || strings.HasPrefix(dep.Repository, "file://") {
+ return &helm.DependencyWithRepository{
+ Dependency: dep,
+ Repository: nil,
+ }, nil
+ }
+
+ // Discover existing HelmRepository by URL,
+ // if no repository is found a mock is created to attempt to
+ // download the index without any custom configuration.
+ repository, err := r.resolveDependencyRepository(ctx, dep, obj.Namespace)
+ if err != nil {
+ repository = &sourcev1.HelmRepository{
+ Spec: sourcev1.HelmRepositorySpec{
+ URL: dep.Repository,
+ },
+ }
+ }
+
+ // Configure Helm client getter options
+ clientOpts := []getter.Option{
+ getter.WithTimeout(obj.Spec.Interval.Duration),
+ getter.WithURL(repository.Spec.URL),
+ getter.WithPassCredentialsAll(repository.Spec.PassCredentials),
+ }
+ if repository.Spec.SecretRef != nil {
+ name := types.NamespacedName{
+ Namespace: repository.GetNamespace(),
+ Name: repository.Spec.SecretRef.Name,
+ }
+ secret := &corev1.Secret{}
+ if err := r.Client.Get(ctx, name, secret); err != nil {
+ return nil, err
+ }
+ opts, err := helm.ClientOptionsFromSecret(*secret, dir)
+ if err != nil {
+ return nil, err
+ }
+ clientOpts = append(clientOpts, opts...)
+ }
+
+ // Initialize the chart repository and load the index file
+ index, err := helm.NewChartRepository(repository.Spec.URL, r.Getters, clientOpts)
+ if err != nil {
+ return nil, err
+ }
+
+ // Load or download the repository index
+ switch repository.Status.Artifact {
+ case nil:
+ err = index.DownloadIndex()
+ default:
+ err = index.LoadIndexFile(r.Storage.LocalPath(*repository.GetArtifact()))
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ return &helm.DependencyWithRepository{
+ Dependency: dep,
+ Repository: index,
+ }, nil
+}
+
+func (r *HelmChartReconciler) mergeChartValuesFiles(ctx context.Context, obj *sourcev1.HelmChart, chart *helmchart.Chart, path string) (ctrl.Result, error) {
+ valuesFiles := obj.GetValuesFiles()
+ if len(valuesFiles) < 1 {
+ logr.FromContext(ctx).Info("No values files defined: skipping merge and overwrite of values")
+ return ctrl.Result{}, nil
+ }
+
+ var values map[string]interface{}
+ var err error
+ if path == "" {
+ values, err = mergeChartValues(chart, valuesFiles)
+ } else {
+ values, err = mergeFileValues(path, valuesFiles)
+ }
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.ChartReconciled, "MergeFailure", "Merge of %v values failed: %s", valuesFiles, err.Error())
+ return ctrl.Result{}, err
+ }
+
+ b, err := yaml.Marshal(values)
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.ChartReconciled, "MarshalFailure", "Marshalling of merged values failed: %s", err.Error())
+ return ctrl.Result{}, err
+ }
+
+ modifiedValues, err := helm.OverwriteChartDefaultValues(chart, b)
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.ChartReconciled, "OverwriteFailure", "Overwrite of chart default values with merged values failed: %s", err.Error())
+ return ctrl.Result{}, err
+ }
+ if !modifiedValues {
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+ }
+
+ conditions.MarkTrue(obj, sourcev1.ValuesFilesMergedCondition, "ModifiedValues", "Replaced chart values with merged values from: %v", valuesFiles)
+ r.Events.Eventf(ctx, obj, events.EventSeverityInfo, "ModifiedValues", "Replaced chart values with merged values from: %v", valuesFiles)
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+}
+
+func (r *HelmChartReconciler) resolveDependencyRepository(ctx context.Context, dep *helmchart.Dependency, namespace string) (*sourcev1.HelmRepository, error) {
+ u := helm.NormalizeChartRepositoryURL(dep.Repository)
+ if u == "" {
+ return nil, fmt.Errorf("empty repository dependency URL")
+ }
+ listOpts := []client.ListOption{
+ client.InNamespace(namespace),
+ client.MatchingFields{sourcev1.HelmRepositoryURLIndexKey: u},
+ }
+ var list sourcev1.HelmRepositoryList
+ if err := r.Client.List(ctx, &list, listOpts...); err != nil {
+ return nil, fmt.Errorf("unable to retrieve HelmRepositoryList: %w", err)
+ }
+ if len(list.Items) > 0 {
+ return &list.Items[0], nil
+ }
+ return nil, fmt.Errorf("no HelmRepository found")
+}
+
+func mergeChartValues(chart *helmchart.Chart, valuesFiles []string) (map[string]interface{}, error) {
+ mergedValues := make(map[string]interface{})
+ for _, p := range valuesFiles {
+ cfn := filepath.Clean(p)
+ if cfn == chartutil.ValuesfileName {
+ mergedValues = transform.MergeMaps(mergedValues, chart.Values)
+ }
+ var b []byte
+ for _, f := range chart.Files {
+ if f.Name == cfn {
+ b = f.Data
+ }
+ }
+ if b == nil {
+ return nil, fmt.Errorf("no values file found at path %q", p)
+ }
+ values := make(map[string]interface{})
+ if err := yaml.Unmarshal(b, values); err != nil {
+ return nil, fmt.Errorf("unmarshaling values from %q failed: %w", p, err)
+ }
+ mergedValues = transform.MergeMaps(mergedValues, values)
+ }
+ return mergedValues, nil
+}
+
+func mergeFileValues(dir string, valuesFiles []string) (map[string]interface{}, error) {
+ mergedValues := make(map[string]interface{})
+ for _, p := range valuesFiles {
+ secureP, err := securejoin.SecureJoin(dir, p)
+ if err != nil {
+ return nil, err
+ }
+ if f, err := os.Stat(secureP); os.IsNotExist(err) || !f.Mode().IsRegular() {
+ return nil, fmt.Errorf("invalid values file path %q", p)
+ }
+
+ b, err := os.ReadFile(secureP)
+ if err != nil {
+ return nil, fmt.Errorf("could not read values from file %q: %w", p, err)
+ }
+ values := make(map[string]interface{})
+ err = yaml.Unmarshal(b, &values)
+ if err != nil {
+ return nil, fmt.Errorf("unmarshaling values from %q failed: %w", p, err)
+ }
+ mergedValues = transform.MergeMaps(mergedValues, values)
+ }
+ return mergedValues, nil
+}
diff --git a/controllers/helmchart_controller_source.go b/controllers/helmchart_controller_source.go
new file mode 100644
index 000000000..1c4b838f2
--- /dev/null
+++ b/controllers/helmchart_controller_source.go
@@ -0,0 +1,241 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/url"
+ "os"
+
+ "github.com/go-logr/logr"
+ "helm.sh/helm/v3/pkg/getter"
+ corev1 "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/types"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+
+ "github.com/fluxcd/pkg/runtime/conditions"
+ "github.com/fluxcd/pkg/runtime/events"
+ "github.com/fluxcd/pkg/untar"
+
+ sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+ "github.com/fluxcd/source-controller/internal/helm"
+)
+
+func (r *HelmChartReconciler) reconcileSource(ctx context.Context, obj *sourcev1.HelmChart, path *string) (ctrl.Result, error) {
+ // Attempt to get the source
+ sourceObj, err := r.getSource(ctx, obj)
+ if err != nil {
+ switch {
+ case errors.Is(err, unsupportedSourceKindError{}):
+ return ctrl.Result{}, nil
+ case apierrors.IsNotFound(err):
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+ default:
+ return ctrl.Result{}, err
+ }
+ }
+
+ // Mirror source readiness
+ conditions.SetMirror(obj, sourcev1.SourceRefReadyCondition, sourceObj)
+
+ // Confirm source has an artifact
+ if sourceObj.GetArtifact() == nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, "NoArtifact", "No artifact available for %s %q", obj.Spec.SourceRef.Kind, obj.Spec.SourceRef.Name)
+ // The watcher should notice an artifact change
+ return ctrl.Result{}, nil
+ }
+
+ // Retrieve the contents from the source
+ switch typedSource := sourceObj.(type) {
+ case *sourcev1.HelmRepository:
+ return r.reconcileFromHelmRepository(ctx, obj, typedSource, path)
+ case *sourcev1.GitRepository, *sourcev1.Bucket:
+ return r.reconcileFromTarballArtifact(ctx, obj, *typedSource.GetArtifact(), path)
+ default:
+ // This should never happen
+ return ctrl.Result{}, fmt.Errorf("missing target for typed source object")
+ }
+}
+
+func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context, obj *sourcev1.HelmChart, repository *sourcev1.HelmRepository, path *string) (_ ctrl.Result, retErr error) {
+ // TODO: move this to a validation webhook once the discussion around
+ // certificates has settled: https://github.com/fluxcd/image-reflector-controller/issues/69
+ if err := validHelmChartName(obj.Spec.Chart); err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, "InvalidChartName", "Validation error: %s", err.Error())
+ return ctrl.Result{}, nil
+ }
+
+ // Configure Helm client to access repository
+ clientOpts := []getter.Option{
+ getter.WithTimeout(repository.Spec.Timeout.Duration),
+ getter.WithURL(repository.Spec.URL),
+ getter.WithPassCredentialsAll(repository.Spec.PassCredentials),
+ }
+ if repository.Spec.SecretRef != nil {
+ // Attempt to retrieve secret
+ name := types.NamespacedName{
+ Namespace: repository.GetNamespace(),
+ Name: repository.Spec.SecretRef.Name,
+ }
+ var secret corev1.Secret
+ if err := r.Client.Get(ctx, name, &secret); err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.AuthenticationFailedReason, "Failed to get secret %s: %s", name.String(), err.Error())
+ // Return transient errors but wait for next interval on not found
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, client.IgnoreNotFound(err)
+ }
+
+ // Get client options from secret
+ tmpDir, err := os.MkdirTemp("", "helm-client-")
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.AuthenticationFailedReason, "Could not create temporary directory for : %s", err.Error())
+ return ctrl.Result{}, err
+ }
+ defer os.RemoveAll(tmpDir)
+ opts, err := helm.ClientOptionsFromSecret(secret, tmpDir)
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.AuthenticationFailedReason, "Failed to configure Helm client with secret data: %s", err)
+ // Return err as the content of the secret may change
+ return ctrl.Result{}, err
+ }
+ clientOpts = append(clientOpts, opts...)
+ }
+
+ // Construct Helm chart repository with options and load the index
+ index, err := helm.NewChartRepository(repository.Spec.URL, r.Getters, clientOpts)
+ if err != nil {
+ switch err.(type) {
+ case *url.Error:
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.URLInvalidReason, "Invalid Helm repository URL: %s", err.Error())
+ return ctrl.Result{}, nil
+ default:
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.URLInvalidReason, "Helm client construction failed: %s", err.Error())
+ return ctrl.Result{}, nil
+ }
+ }
+ if err = index.LoadIndexFile(r.Storage.LocalPath(*repository.GetArtifact())); err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.ChartPullFailedReason, "Helm repository index load from artifact failed: %s", err.Error())
+ return ctrl.Result{}, err
+ }
+
+ // Lookup the chart version in the chart repository index
+ chartVer, err := index.Get(obj.Spec.Chart, obj.Spec.Version)
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.ChartPullFailedReason, "Could not find %q chart with version %q: %s", obj.Spec.Chart, obj.Spec.Version, err.Error())
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+ }
+
+ // The artifact is up-to-date
+ if obj.GetArtifact().HasRevision(chartVer.Version) {
+ logr.FromContext(ctx).Info("Artifact up-to-date: skipping chart download")
+ return ctrl.Result{RequeueAfter: obj.GetInterval().Duration}, nil
+ }
+
+ // Create a new temporary file for the chart and download it
+ f, err := os.CreateTemp("", fmt.Sprintf("%s-%s-", obj.Name, obj.Namespace))
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.ChartPullFailedReason, "Chart download for %q version %q failed: %s", obj.Spec.Chart, chartVer.Version, err.Error())
+ return ctrl.Result{}, err
+ }
+ b, err := index.DownloadChart(chartVer)
+ if err != nil {
+ f.Close()
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.ChartPullFailedReason, "Chart download for %q version %q failed: %s", obj.Spec.Chart, chartVer.Version, err.Error())
+ return ctrl.Result{}, err
+ }
+ if _, err = io.Copy(f, b); err != nil {
+ f.Close()
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.ChartPullFailedReason, "Chart download for %q version %q failed: %s", obj.Spec.Chart, chartVer.Version, err.Error())
+ return ctrl.Result{}, err
+ }
+ f.Close()
+
+ *path = f.Name()
+ conditions.MarkTrue(obj, sourcev1.SourceAvailableCondition, sourcev1.ChartPullSucceededReason, "Pulled chart version %s", chartVer.Version)
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+}
+
+func (r *HelmChartReconciler) reconcileFromTarballArtifact(ctx context.Context, obj *sourcev1.HelmChart, artifact sourcev1.Artifact, path *string) (_ ctrl.Result, retErr error) {
+ f, err := os.Open(r.Storage.LocalPath(artifact))
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.StorageOperationFailedReason, "Could not open artifact: %s", err.Error())
+ r.Events.Event(ctx, obj, events.EventSeverityError, sourcev1.StorageOperationFailedReason, conditions.Get(obj, sourcev1.SourceAvailableCondition).Message)
+ return ctrl.Result{}, err
+ }
+
+ dir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s-", obj.Name, obj.Namespace))
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.StorageOperationFailedReason, "Could not create temporary working directory: %s", err.Error())
+ return ctrl.Result{}, err
+ }
+ *path = dir
+
+ if _, err = untar.Untar(f, dir); err != nil {
+ f.Close()
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.StorageOperationFailedReason, "Decompression of artifact failed: %s", err.Error())
+ r.Events.Event(ctx, obj, events.EventSeverityError, sourcev1.StorageOperationFailedReason, conditions.Get(obj, sourcev1.SourceAvailableCondition).Message)
+ return ctrl.Result{}, err
+ }
+ f.Close()
+
+ conditions.MarkTrue(obj, sourcev1.SourceAvailableCondition, sourcev1.ChartPullSucceededReason, "Decompressed artifact %s", artifact.Revision)
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+}
+
+// getSource attempts to get the source referenced in the given object,
+// if the referenced source kind is not supported it returns an
+// unsupportedSourceKindError.
+func (r *HelmChartReconciler) getSource(ctx context.Context, obj *sourcev1.HelmChart) (sourcev1.Source, error) {
+ var s sourcev1.Source
+ namespacedName := types.NamespacedName{
+ Namespace: obj.GetNamespace(),
+ Name: obj.Spec.SourceRef.Name,
+ }
+ switch obj.Spec.SourceRef.Kind {
+ case sourcev1.HelmRepositoryKind:
+ repository := &sourcev1.HelmRepository{}
+ err := r.Client.Get(ctx, namespacedName, repository)
+ if err != nil {
+ return nil, err
+ }
+ s = repository
+ case sourcev1.GitRepositoryKind:
+ repository := &sourcev1.GitRepository{}
+ err := r.Client.Get(ctx, namespacedName, repository)
+ if err != nil {
+ return nil, err
+ }
+ s = repository
+ case sourcev1.BucketKind:
+ bucket := &sourcev1.Bucket{}
+ err := r.Client.Get(ctx, namespacedName, bucket)
+ if err != nil {
+ return nil, err
+ }
+ s = bucket
+ default:
+ return nil, unsupportedSourceKindError{
+ Kind: obj.Spec.SourceRef.Kind,
+ Supported: []string{sourcev1.HelmRepositoryKind, sourcev1.GitRepositoryKind, sourcev1.BucketKind},
+ }
+ }
+ return s, nil
+}
diff --git a/controllers/helmchart_controller_source_test.go b/controllers/helmchart_controller_source_test.go
new file mode 100644
index 000000000..25bc4c7d7
--- /dev/null
+++ b/controllers/helmchart_controller_source_test.go
@@ -0,0 +1,595 @@
+package controllers
+
+import (
+ "bytes"
+ "net/http"
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+
+ "github.com/fluxcd/pkg/apis/meta"
+ "github.com/fluxcd/pkg/runtime/conditions"
+ "github.com/go-logr/logr"
+ . "github.com/onsi/gomega"
+ "helm.sh/helm/v3/pkg/chart/loader"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ ctrl "sigs.k8s.io/controller-runtime"
+ fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
+ "sigs.k8s.io/controller-runtime/pkg/log"
+
+ "github.com/fluxcd/pkg/helmtestserver"
+ sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+)
+
+func TestHelmChartReconciler_reconcileSource(t *testing.T) {
+
+}
+
+func TestHelmChartReconciler_reconcileFromHelmRepository(t *testing.T) {
+ g := NewWithT(t)
+
+ versions := []string{"v0.1.0", "0.2.0", "v0.2.1", "v1.0.0-alpha", "v1.1.0", "v2.0.0"}
+
+ tests := []struct {
+ name string
+ beforeFunc func(obj *sourcev1.HelmChart, sourceObj *sourcev1.HelmRepository)
+ want ctrl.Result
+ wantErr bool
+ wantRevision string
+ assertConditions []metav1.Condition
+ }{
+ {
+ name: "Empty version (latest)",
+ want: ctrl.Result{RequeueAfter: interval},
+ wantRevision: "v2.0.0",
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.SourceAvailableCondition, sourcev1.ChartPullSucceededReason, "Pulled chart version v2.0.0"),
+ },
+ },
+ {
+ name: "SemVer (any)",
+ want: ctrl.Result{RequeueAfter: interval},
+ beforeFunc: func(obj *sourcev1.HelmChart, _ *sourcev1.HelmRepository) {
+ obj.Spec.Version = "*"
+ },
+ wantRevision: "v2.0.0",
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.SourceAvailableCondition, sourcev1.ChartPullSucceededReason, "Pulled chart version v2.0.0"),
+ },
+ },
+ {
+ name: "SemVer selector",
+ want: ctrl.Result{RequeueAfter: interval},
+ beforeFunc: func(obj *sourcev1.HelmChart, _ *sourcev1.HelmRepository) {
+ obj.Spec.Version = " 0 {
+ server.WithMiddleware(func(handler http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ u, p, ok := r.BasicAuth()
+ if !ok || tt.server.username != u || tt.server.password != p {
+ w.WriteHeader(401)
+ return
+ }
+ handler.ServeHTTP(w, r)
+ })
+ })
+ }
+
+ if len(tt.server.publicKey)+len(tt.server.privateKey)+len(tt.server.ca) > 0 {
+ g.Expect(server.StartTLS(tt.server.publicKey, tt.server.privateKey, tt.server.ca, "example.com")).To(Succeed())
+ } else {
+ server.Start()
+ }
+ defer server.Stop()
+
+ repository := &sourcev1.HelmRepository{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "helmrepository",
+ },
+ Spec: sourcev1.HelmRepositorySpec{
+ Timeout: &metav1.Duration{Duration: timeout},
+ URL: server.URL(),
+ },
+ Status: sourcev1.HelmRepositoryStatus{
+ Artifact: &sourcev1.Artifact{
+ Path: "/helmrepository/secret-ref/index.yaml",
+ },
+ },
+ }
+
+ g.Expect(storage.MkdirAll(*repository.GetArtifact())).To(Succeed())
+ defer storage.RemoveAll(*repository.GetArtifact())
+ g.Expect(storage.CopyFromPath(repository.GetArtifact(), filepath.Join(server.Root(), "index.yaml"))).To(Succeed())
+
+ obj := &sourcev1.HelmChart{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "helmchart",
+ },
+ Spec: sourcev1.HelmChartSpec{
+ Interval: metav1.Duration{Duration: interval},
+ Chart: "helmchart",
+ },
+ }
+
+ if tt.beforeFunc != nil {
+ tt.beforeFunc(repository)
+ }
+
+ builder := fakeclient.NewClientBuilder().WithScheme(env.GetScheme())
+ secret := tt.secret.DeepCopy()
+ if secret != nil {
+ builder.WithObjects(secret.DeepCopy())
+ }
+
+ r := &HelmChartReconciler{
+ Client: builder.Build(),
+ Storage: storage,
+ Getters: testGetters,
+ }
+
+ var path string
+ got, err := r.reconcileFromHelmRepository(logr.NewContext(ctx, log.NullLogger{}), obj, repository, &path)
+ defer os.RemoveAll(path)
+ g.Expect(err != nil).To(Equal(tt.wantErr))
+ g.Expect(got).To(Equal(tt.want))
+
+ g.Expect(obj.GetConditions()).ToNot(BeNil())
+ g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions))
+ })
+ }
+}
+
+func TestHelmChartReconciler_reconcileFromTarballArtifact(t *testing.T) {
+ tests := []struct {
+ name string
+ artifact sourcev1.Artifact
+ beforeFunc func(artifact *sourcev1.Artifact)
+ afterFunc func(artifact *sourcev1.Artifact)
+ want ctrl.Result
+ wantErr bool
+ wantPath bool
+ wantRevision string
+ assertConditions []metav1.Condition
+ }{
+ {
+ name: "Reconcile",
+ artifact: sourcev1.Artifact{
+ Revision: "checksum",
+ Path: "/fake/archive.tar.gz",
+ },
+ beforeFunc: func(artifact *sourcev1.Artifact) {
+ storage.MkdirAll(*artifact)
+ storage.Archive(artifact, "testdata/charts", nil)
+ },
+ afterFunc: func(artifact *sourcev1.Artifact) {
+ os.RemoveAll(storage.LocalPath(*artifact))
+ },
+ want: ctrl.Result{RequeueAfter: interval},
+ wantPath: true,
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.SourceAvailableCondition, sourcev1.ChartPullSucceededReason, "Decompressed artifact checksum"),
+ },
+ },
+ {
+ name: "Non-existing artifact path",
+ artifact: sourcev1.Artifact{
+ Path: "/some/invalid/path",
+ },
+ wantErr: true,
+ wantPath: false,
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceAvailableCondition, sourcev1.StorageOperationFailedReason, "Could not open artifact"),
+ },
+ },
+ {
+ name: "Invalid artifact file type",
+ artifact: sourcev1.Artifact{
+ Path: "/invalid/file.txt",
+ },
+ beforeFunc: func(artifact *sourcev1.Artifact) {
+ storage.MkdirAll(*artifact)
+ storage.AtomicWriteFile(artifact, bytes.NewReader([]byte("invalid")), 0655)
+ },
+ afterFunc: func(artifact *sourcev1.Artifact) {
+ os.RemoveAll(storage.LocalPath(*artifact))
+ },
+ wantErr: true,
+ wantPath: true,
+ assertConditions: []metav1.Condition{
+ *conditions.FalseCondition(sourcev1.SourceAvailableCondition, sourcev1.StorageOperationFailedReason, "Decompression of artifact failed: requires gzip-compressed body: unexpected EOF"),
+ },
+ },
+ }
+
+ builder := fakeclient.NewClientBuilder().WithScheme(env.GetScheme())
+
+ r := &HelmChartReconciler{
+ Client: builder.Build(),
+ Storage: storage,
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ obj := &sourcev1.HelmChart{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "from-artifact-",
+ },
+ Spec: sourcev1.HelmChartSpec{
+ Interval: metav1.Duration{Duration: interval},
+ Chart: "./helmchart",
+ },
+ }
+
+ if tt.beforeFunc != nil {
+ tt.beforeFunc(tt.artifact.DeepCopy())
+ }
+ if tt.afterFunc != nil {
+ defer tt.afterFunc(tt.artifact.DeepCopy())
+ }
+
+ var path string
+ got, err := r.reconcileFromTarballArtifact(logr.NewContext(ctx, log.NullLogger{}), obj, tt.artifact, &path)
+ defer os.RemoveAll(path)
+ g.Expect(err != nil).To(Equal(tt.wantErr))
+ g.Expect(path != "").To(Equal(tt.wantPath))
+ g.Expect(got).To(Equal(tt.want))
+
+ g.Expect(obj.GetConditions()).ToNot(BeNil())
+ g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions))
+ })
+ }
+}
+
+func TestHelmChartReconciler_getSource(t *testing.T) {
+ helmRepo := &sourcev1.HelmRepository{
+ TypeMeta: metav1.TypeMeta{
+ Kind: sourcev1.HelmRepositoryKind,
+ APIVersion: sourcev1.GroupVersion.String(),
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "helmrepository",
+ },
+ }
+ gitRepo := &sourcev1.GitRepository{
+ TypeMeta: metav1.TypeMeta{
+ Kind: sourcev1.GitRepositoryKind,
+ APIVersion: sourcev1.GroupVersion.String(),
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "gitrepository",
+ },
+ }
+ bucket := &sourcev1.Bucket{
+ TypeMeta: metav1.TypeMeta{
+ Kind: sourcev1.BucketKind,
+ APIVersion: sourcev1.GroupVersion.String(),
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "bucket",
+ },
+ }
+
+ tests := []struct {
+ name string
+ sourceRef sourcev1.LocalHelmChartSourceReference
+ want sourcev1.Source
+ wantErr error
+ }{
+ {
+ name: "HelmRepository",
+ sourceRef: sourcev1.LocalHelmChartSourceReference{
+ Kind: sourcev1.HelmRepositoryKind,
+ Name: helmRepo.Name,
+ },
+ want: helmRepo,
+ },
+ {
+ name: "GitRepository",
+ sourceRef: sourcev1.LocalHelmChartSourceReference{
+ Kind: sourcev1.GitRepositoryKind,
+ Name: gitRepo.Name,
+ },
+ want: gitRepo,
+ },
+ {
+ name: "Bucket",
+ sourceRef: sourcev1.LocalHelmChartSourceReference{
+ Kind: sourcev1.BucketKind,
+ Name: bucket.Name,
+ },
+ want: bucket,
+ },
+ {
+ name: "Unsupported",
+ sourceRef: sourcev1.LocalHelmChartSourceReference{
+ Kind: sourcev1.HelmChartKind,
+ },
+ want: nil,
+ wantErr: unsupportedSourceKindError{
+ Kind: sourcev1.HelmChartKind,
+ Supported: []string{sourcev1.HelmRepositoryKind, sourcev1.GitRepositoryKind, sourcev1.BucketKind},
+ },
+ },
+ }
+
+ builder := fakeclient.NewClientBuilder().WithScheme(env.GetScheme())
+ builder.WithObjects(helmRepo, gitRepo, bucket)
+
+ r := &HelmChartReconciler{
+ Client: builder.Build(),
+ Storage: storage,
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ obj := &sourcev1.HelmChart{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "helmchart",
+ },
+ Spec: sourcev1.HelmChartSpec{
+ SourceRef: tt.sourceRef,
+ },
+ }
+ got, err := r.getSource(logr.NewContext(ctx, log.NullLogger{}), obj)
+ if !reflect.DeepEqual(err, tt.wantErr) {
+ t.Errorf("getSource() error = %v, wantErr %v", err, tt.wantErr)
+ }
+ if !reflect.DeepEqual(got, tt.want) {
+ t.Errorf("getSource() got = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
diff --git a/controllers/helmchart_controller_test.go b/controllers/helmchart_controller_test.go
index c88fa3395..3ce476a5c 100644
--- a/controllers/helmchart_controller_test.go
+++ b/controllers/helmchart_controller_test.go
@@ -17,1301 +17,91 @@ limitations under the License.
package controllers
import (
- "context"
- "fmt"
- "io/ioutil"
- "net/http"
- "net/url"
- "os"
- "path"
- "path/filepath"
- "strings"
"testing"
"time"
- "github.com/fluxcd/pkg/apis/meta"
- "github.com/fluxcd/pkg/gittestserver"
- "github.com/fluxcd/pkg/helmtestserver"
- "github.com/go-git/go-billy/v5/memfs"
- "github.com/go-git/go-git/v5"
- "github.com/go-git/go-git/v5/config"
- "github.com/go-git/go-git/v5/plumbing/object"
- "github.com/go-git/go-git/v5/storage/memory"
- . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
- helmchart "helm.sh/helm/v3/pkg/chart"
- "helm.sh/helm/v3/pkg/chart/loader"
- "helm.sh/helm/v3/pkg/chartutil"
- corev1 "k8s.io/api/core/v1"
- apimeta "k8s.io/apimachinery/pkg/api/meta"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/types"
- "sigs.k8s.io/yaml"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+
+ "github.com/fluxcd/pkg/apis/meta"
+ "github.com/fluxcd/pkg/runtime/conditions"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
)
-var _ = Describe("HelmChartReconciler", func() {
-
- const (
- timeout = time.Second * 30
- interval = time.Second * 1
- indexInterval = time.Second * 2
- pullInterval = time.Second * 3
- )
-
- Context("HelmChart from HelmRepository", func() {
- var (
- namespace *corev1.Namespace
- helmServer *helmtestserver.HelmServer
- err error
- )
-
- BeforeEach(func() {
- namespace = &corev1.Namespace{
- ObjectMeta: metav1.ObjectMeta{Name: "helm-chart-test-" + randStringRunes(5)},
- }
- err = k8sClient.Create(context.Background(), namespace)
- Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
-
- helmServer, err = helmtestserver.NewTempHelmServer()
- Expect(err).To(Succeed())
- helmServer.Start()
- })
-
- AfterEach(func() {
- os.RemoveAll(helmServer.Root())
- helmServer.Stop()
-
- err = k8sClient.Delete(context.Background(), namespace)
- Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
- })
-
- It("Creates artifacts for", func() {
- Expect(helmServer.PackageChart(path.Join("testdata/charts/helmchart"))).Should(Succeed())
- Expect(helmServer.GenerateIndex()).Should(Succeed())
-
- repositoryKey := types.NamespacedName{
- Name: "helmrepository-sample-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- Expect(k8sClient.Create(context.Background(), &sourcev1.HelmRepository{
- ObjectMeta: metav1.ObjectMeta{
- Name: repositoryKey.Name,
- Namespace: repositoryKey.Namespace,
- },
- Spec: sourcev1.HelmRepositorySpec{
- URL: helmServer.URL(),
- Interval: metav1.Duration{Duration: indexInterval},
- },
- })).Should(Succeed())
-
- key := types.NamespacedName{
- Name: "helmchart-sample-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- created := &sourcev1.HelmChart{
- ObjectMeta: metav1.ObjectMeta{
- Name: key.Name,
- Namespace: key.Namespace,
- },
- Spec: sourcev1.HelmChartSpec{
- Chart: "helmchart",
- Version: "",
- SourceRef: sourcev1.LocalHelmChartSourceReference{
- Kind: sourcev1.HelmRepositoryKind,
- Name: repositoryKey.Name,
- },
- Interval: metav1.Duration{Duration: pullInterval},
- },
- }
- Expect(k8sClient.Create(context.Background(), created)).Should(Succeed())
-
- By("Expecting artifact")
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact != nil && storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(helmChart.Values["testDefault"]).To(BeTrue())
- Expect(helmChart.Values["testOverride"]).To(BeFalse())
-
- By("Packaging a new chart version and regenerating the index")
- Expect(helmServer.PackageChartWithVersion(path.Join("testdata/charts/helmchart"), "0.2.0")).Should(Succeed())
- Expect(helmServer.GenerateIndex()).Should(Succeed())
-
- By("Expecting new artifact revision and GC")
- Eventually(func() bool {
- now := &sourcev1.HelmChart{}
- _ = k8sClient.Get(context.Background(), key, now)
- // Test revision change and garbage collection
- return now.Status.Artifact.Revision != got.Status.Artifact.Revision &&
- !storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
-
- When("Setting valid valuesFiles attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFiles = []string{
- "values.yaml",
- "override.yaml",
- }
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact.Checksum != updated.Status.Artifact.Checksum &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(helmChart.Values["testDefault"]).To(BeTrue())
- Expect(helmChart.Values["testOverride"]).To(BeTrue())
- })
-
- When("Setting invalid valuesFiles attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFiles = []string{
- "values.yaml",
- "invalid.yaml",
- }
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.ObservedGeneration > updated.Status.ObservedGeneration &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(helmChart.Values["testDefault"]).To(BeTrue())
- Expect(helmChart.Values["testOverride"]).To(BeTrue())
- })
-
- When("Setting valid valuesFiles and valuesFile attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFile = "values.yaml"
- updated.Spec.ValuesFiles = []string{
- "override.yaml",
- }
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact.Checksum != updated.Status.Artifact.Checksum &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(helmChart.Values["testDefault"]).To(BeTrue())
- Expect(helmChart.Values["testOverride"]).To(BeTrue())
- })
-
- When("Setting valid valuesFile attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFile = "override.yaml"
- updated.Spec.ValuesFiles = []string{}
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact.Checksum != updated.Status.Artifact.Checksum &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- _, exists := helmChart.Values["testDefault"]
- Expect(exists).To(BeFalse())
- Expect(helmChart.Values["testOverride"]).To(BeTrue())
- })
-
- When("Setting identical valuesFile attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFile = "duplicate.yaml"
- updated.Spec.ValuesFiles = []string{}
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact.Checksum != updated.Status.Artifact.Checksum &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(helmChart.Values["testDefault"]).To(BeTrue())
- Expect(helmChart.Values["testOverride"]).To(BeFalse())
- })
-
- When("Setting invalid valuesFile attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFile = "invalid.yaml"
- updated.Spec.ValuesFiles = []string{}
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.ObservedGeneration > updated.Status.ObservedGeneration &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(helmChart.Values["testDefault"]).To(BeTrue())
- Expect(helmChart.Values["testOverride"]).To(BeFalse())
- })
-
- By("Expecting missing HelmRepository error")
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).Should(Succeed())
- updated.Spec.SourceRef.Name = "invalid"
- updated.Spec.ValuesFile = ""
- updated.Spec.ValuesFiles = []string{}
- Expect(k8sClient.Update(context.Background(), updated)).Should(Succeed())
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, updated)
- for _, c := range updated.Status.Conditions {
- if c.Reason == sourcev1.ChartPullFailedReason &&
- strings.Contains(c.Message, "failed to retrieve source") {
- return true
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
- Expect(updated.Status.Artifact).ToNot(BeNil())
-
- By("Expecting to delete successfully")
- got = &sourcev1.HelmChart{}
- Eventually(func() error {
- _ = k8sClient.Get(context.Background(), key, got)
- return k8sClient.Delete(context.Background(), got)
- }, timeout, interval).Should(Succeed())
-
- By("Expecting delete to finish")
- Eventually(func() error {
- c := &sourcev1.HelmChart{}
- return k8sClient.Get(context.Background(), key, c)
- }, timeout, interval).ShouldNot(Succeed())
-
- exists := func(path string) bool {
- // wait for tmp sync on macOS
- time.Sleep(time.Second)
- _, err := os.Stat(path)
- return err == nil
- }
-
- By("Expecting GC on delete")
- Eventually(exists(got.Status.Artifact.Path), timeout, interval).ShouldNot(BeTrue())
- })
-
- It("Filters versions", func() {
- versions := []string{"0.1.0", "0.1.1", "0.2.0", "0.3.0-rc.1", "1.0.0-alpha.1", "1.0.0"}
- for k := range versions {
- Expect(helmServer.PackageChartWithVersion(path.Join("testdata/charts/helmchart"), versions[k])).Should(Succeed())
- }
-
- Expect(helmServer.GenerateIndex()).Should(Succeed())
-
- repositoryKey := types.NamespacedName{
- Name: "helmrepository-sample-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- repository := &sourcev1.HelmRepository{
- ObjectMeta: metav1.ObjectMeta{
- Name: repositoryKey.Name,
- Namespace: repositoryKey.Namespace,
- },
- Spec: sourcev1.HelmRepositorySpec{
- URL: helmServer.URL(),
- Interval: metav1.Duration{Duration: 1 * time.Hour},
- },
- }
- Expect(k8sClient.Create(context.Background(), repository)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), repository)
-
- key := types.NamespacedName{
- Name: "helmchart-sample-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- chart := &sourcev1.HelmChart{
- ObjectMeta: metav1.ObjectMeta{
- Name: key.Name,
- Namespace: key.Namespace,
- },
- Spec: sourcev1.HelmChartSpec{
- Chart: "helmchart",
- Version: "*",
- SourceRef: sourcev1.LocalHelmChartSourceReference{
- Kind: sourcev1.HelmRepositoryKind,
- Name: repositoryKey.Name,
- },
- Interval: metav1.Duration{Duration: 1 * time.Hour},
- },
- }
- Expect(k8sClient.Create(context.Background(), chart)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), chart)
-
- Eventually(func() string {
- _ = k8sClient.Get(context.Background(), key, chart)
- if chart.Status.Artifact != nil {
- return chart.Status.Artifact.Revision
- }
- return ""
- }, timeout, interval).Should(Equal("1.0.0"))
-
- chart.Spec.Version = "<0.2.0"
- Expect(k8sClient.Update(context.Background(), chart)).Should(Succeed())
- Eventually(func() string {
- _ = k8sClient.Get(context.Background(), key, chart)
- if chart.Status.Artifact != nil {
- return chart.Status.Artifact.Revision
- }
- return ""
- }, timeout, interval).Should(Equal("0.1.1"))
-
- chart.Spec.Version = "invalid"
- Expect(k8sClient.Update(context.Background(), chart)).Should(Succeed())
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, chart)
- for _, c := range chart.Status.Conditions {
- if c.Reason == sourcev1.ChartPullFailedReason {
- return true
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
- Expect(chart.GetArtifact()).NotTo(BeNil())
- Expect(chart.Status.Artifact.Revision).Should(Equal("0.1.1"))
- })
-
- It("Authenticates when credentials are provided", func() {
- helmServer.Stop()
- var username, password = "john", "doe"
- helmServer.WithMiddleware(func(handler http.Handler) http.Handler {
- return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- u, p, ok := r.BasicAuth()
- if !ok || username != u || password != p {
- w.WriteHeader(401)
- return
- }
- handler.ServeHTTP(w, r)
- })
- })
- helmServer.Start()
-
- Expect(helmServer.PackageChartWithVersion(path.Join("testdata/charts/helmchart"), "0.1.0")).Should(Succeed())
- Expect(helmServer.GenerateIndex()).Should(Succeed())
-
- secretKey := types.NamespacedName{
- Name: "helmrepository-auth-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- secret := &corev1.Secret{
- ObjectMeta: metav1.ObjectMeta{
- Name: secretKey.Name,
- Namespace: secretKey.Namespace,
- },
- Data: map[string][]byte{
- "username": []byte(username),
- "password": []byte(password),
- },
- }
- Expect(k8sClient.Create(context.Background(), secret)).Should(Succeed())
-
- By("Creating repository and waiting for artifact")
- repositoryKey := types.NamespacedName{
- Name: "helmrepository-sample-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- repository := &sourcev1.HelmRepository{
- ObjectMeta: metav1.ObjectMeta{
- Name: repositoryKey.Name,
- Namespace: repositoryKey.Namespace,
- },
- Spec: sourcev1.HelmRepositorySpec{
- URL: helmServer.URL(),
- SecretRef: &meta.LocalObjectReference{
- Name: secretKey.Name,
- },
- Interval: metav1.Duration{Duration: pullInterval},
- },
- }
- Expect(k8sClient.Create(context.Background(), repository)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), repository)
-
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), repositoryKey, repository)
- return repository.Status.Artifact != nil
- }, timeout, interval).Should(BeTrue())
-
- By("Deleting secret before applying HelmChart")
- Expect(k8sClient.Delete(context.Background(), secret)).Should(Succeed())
-
- By("Applying HelmChart")
- key := types.NamespacedName{
- Name: "helmchart-sample-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- chart := &sourcev1.HelmChart{
- ObjectMeta: metav1.ObjectMeta{
- Name: key.Name,
- Namespace: key.Namespace,
- },
- Spec: sourcev1.HelmChartSpec{
- Chart: "helmchart",
- Version: "*",
- SourceRef: sourcev1.LocalHelmChartSourceReference{
- Kind: sourcev1.HelmRepositoryKind,
- Name: repositoryKey.Name,
- },
- Interval: metav1.Duration{Duration: pullInterval},
- },
- }
- Expect(k8sClient.Create(context.Background(), chart)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), chart)
-
- By("Expecting missing secret error")
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- for _, c := range got.Status.Conditions {
- if c.Reason == sourcev1.AuthenticationFailedReason &&
- strings.Contains(c.Message, "auth secret error") {
- return true
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
-
- By("Applying secret with missing keys")
- secret.ResourceVersion = ""
- secret.Data["username"] = []byte{}
- secret.Data["password"] = []byte{}
- Expect(k8sClient.Create(context.Background(), secret)).Should(Succeed())
-
- By("Expecting 401")
- Eventually(func() bool {
- got := &sourcev1.HelmChart{}
- _ = k8sClient.Get(context.Background(), key, got)
- for _, c := range got.Status.Conditions {
- if c.Reason == sourcev1.ChartPullFailedReason &&
- strings.Contains(c.Message, "401 Unauthorized") {
- return true
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
-
- By("Adding username key")
- secret.Data["username"] = []byte(username)
- Expect(k8sClient.Update(context.Background(), secret)).Should(Succeed())
-
- By("Expecting missing field error")
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- for _, c := range got.Status.Conditions {
- if c.Reason == sourcev1.AuthenticationFailedReason {
- return true
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
-
- By("Adding password key")
- secret.Data["password"] = []byte(password)
- Expect(k8sClient.Update(context.Background(), secret)).Should(Succeed())
-
- By("Expecting artifact")
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return apimeta.IsStatusConditionTrue(got.Status.Conditions, meta.ReadyCondition)
- }, timeout, interval).Should(BeTrue())
- Expect(got.Status.Artifact).ToNot(BeNil())
- })
- })
-
- Context("HelmChart from GitRepository", func() {
- var (
- namespace *corev1.Namespace
- gitServer *gittestserver.GitServer
- err error
- )
-
- BeforeEach(func() {
- namespace = &corev1.Namespace{
- ObjectMeta: metav1.ObjectMeta{Name: "test-git-repository-" + randStringRunes(5)},
- }
- err = k8sClient.Create(context.Background(), namespace)
- Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
-
- gitServer, err = gittestserver.NewTempGitServer()
- Expect(err).NotTo(HaveOccurred())
- gitServer.AutoCreate()
- Expect(gitServer.StartHTTP()).To(Succeed())
- })
-
- AfterEach(func() {
- gitServer.StopHTTP()
- os.RemoveAll(gitServer.Root())
-
- err = k8sClient.Delete(context.Background(), namespace)
- Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
- })
-
- It("Creates artifacts for", func() {
- fs := memfs.New()
- gitrepo, err := git.Init(memory.NewStorage(), fs)
- Expect(err).NotTo(HaveOccurred())
-
- wt, err := gitrepo.Worktree()
- Expect(err).NotTo(HaveOccurred())
-
- u, err := url.Parse(gitServer.HTTPAddress())
- Expect(err).NotTo(HaveOccurred())
- u.Path = path.Join(u.Path, fmt.Sprintf("repository-%s.git", randStringRunes(5)))
-
- _, err = gitrepo.CreateRemote(&config.RemoteConfig{
- Name: "origin",
- URLs: []string{u.String()},
- })
- Expect(err).NotTo(HaveOccurred())
-
- chartDir := "testdata/charts"
- Expect(filepath.Walk(chartDir, func(p string, fi os.FileInfo, err error) error {
- if err != nil {
- return err
- }
-
- switch {
- case fi.Mode().IsDir():
- return fs.MkdirAll(p, os.ModeDir)
- case !fi.Mode().IsRegular():
- return nil
- }
-
- b, err := ioutil.ReadFile(p)
- if err != nil {
- return err
- }
-
- ff, err := fs.Create(p)
- if err != nil {
- return err
- }
- if _, err := ff.Write(b); err != nil {
- return err
- }
- _ = ff.Close()
- _, err = wt.Add(p)
-
- return err
- })).To(Succeed())
-
- _, err = wt.Commit("Helm charts", &git.CommitOptions{Author: &object.Signature{
- Name: "John Doe",
- Email: "john@example.com",
- When: time.Now(),
- }})
- Expect(err).NotTo(HaveOccurred())
-
- err = gitrepo.Push(&git.PushOptions{})
- Expect(err).NotTo(HaveOccurred())
-
- repositoryKey := types.NamespacedName{
- Name: fmt.Sprintf("git-repository-sample-%s", randStringRunes(5)),
- Namespace: namespace.Name,
- }
- repository := &sourcev1.GitRepository{
- ObjectMeta: metav1.ObjectMeta{
- Name: repositoryKey.Name,
- Namespace: repositoryKey.Namespace,
- },
- Spec: sourcev1.GitRepositorySpec{
- URL: u.String(),
- Interval: metav1.Duration{Duration: indexInterval},
- },
- }
- Expect(k8sClient.Create(context.Background(), repository)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), repository)
-
- key := types.NamespacedName{
- Name: "helmchart-sample-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- chart := &sourcev1.HelmChart{
- ObjectMeta: metav1.ObjectMeta{
- Name: key.Name,
- Namespace: key.Namespace,
- },
- Spec: sourcev1.HelmChartSpec{
- Chart: "testdata/charts/helmchartwithdeps",
- Version: "*",
- SourceRef: sourcev1.LocalHelmChartSourceReference{
- Kind: sourcev1.GitRepositoryKind,
- Name: repositoryKey.Name,
- },
- Interval: metav1.Duration{Duration: pullInterval},
- },
- }
- Expect(k8sClient.Create(context.Background(), chart)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), chart)
-
- By("Expecting artifact")
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact != nil &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
-
- By("Committing a new version in the chart metadata")
- f, err := fs.OpenFile(fs.Join(chartDir, "helmchartwithdeps", chartutil.ChartfileName), os.O_RDWR, os.FileMode(0600))
- Expect(err).NotTo(HaveOccurred())
-
- b := make([]byte, 2048)
- n, err := f.Read(b)
- Expect(err).NotTo(HaveOccurred())
- b = b[0:n]
-
- y := new(helmchart.Metadata)
- err = yaml.Unmarshal(b, y)
- Expect(err).NotTo(HaveOccurred())
-
- y.Version = "0.2.0"
- b, err = yaml.Marshal(y)
- Expect(err).NotTo(HaveOccurred())
-
- _, err = f.Write(b)
- Expect(err).NotTo(HaveOccurred())
-
- err = f.Close()
- Expect(err).NotTo(HaveOccurred())
-
- _, err = wt.Commit("Chart version bump", &git.CommitOptions{
- Author: &object.Signature{
- Name: "John Doe",
- Email: "john@example.com",
- When: time.Now(),
- },
- All: true,
- })
- Expect(err).NotTo(HaveOccurred())
-
- err = gitrepo.Push(&git.PushOptions{})
- Expect(err).NotTo(HaveOccurred())
-
- By("Expecting new artifact revision and GC")
- now := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, now)
- // Test revision change and garbage collection
- return now.Status.Artifact.Revision != got.Status.Artifact.Revision &&
- !storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- helmChart, err := loader.Load(storage.LocalPath(*now.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(helmChart.Values["testDefault"]).To(BeTrue())
- Expect(helmChart.Values["testOverride"]).To(BeFalse())
-
- When("Setting valid valuesFiles attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFiles = []string{
- "./testdata/charts/helmchart/values.yaml",
- "./testdata/charts/helmchart/override.yaml",
- }
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact.Checksum != updated.Status.Artifact.Checksum &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(helmChart.Values["testDefault"]).To(BeTrue())
- Expect(helmChart.Values["testOverride"]).To(BeTrue())
- })
-
- When("Setting invalid valuesFiles attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFiles = []string{
- "./testdata/charts/helmchart/values.yaml",
- "./testdata/charts/helmchart/invalid.yaml",
- }
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.ObservedGeneration > updated.Status.ObservedGeneration &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(helmChart.Values["testDefault"]).To(BeTrue())
- Expect(helmChart.Values["testOverride"]).To(BeTrue())
- })
-
- When("Setting valid valuesFiles and valuesFile attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFile = "./testdata/charts/helmchart/values.yaml"
- updated.Spec.ValuesFiles = []string{
- "./testdata/charts/helmchart/override.yaml",
- }
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact.Checksum != updated.Status.Artifact.Checksum &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(helmChart.Values["testDefault"]).To(BeTrue())
- Expect(helmChart.Values["testOverride"]).To(BeTrue())
- })
-
- When("Setting valid valuesFile attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFile = "./testdata/charts/helmchart/override.yaml"
- updated.Spec.ValuesFiles = []string{}
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact.Checksum != updated.Status.Artifact.Checksum &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- _, exists := helmChart.Values["testDefault"]
- Expect(exists).To(BeFalse())
- Expect(helmChart.Values["testOverride"]).To(BeTrue())
- })
-
- When("Setting invalid valuesFile attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFile = "./testdata/charts/helmchart/invalid.yaml"
- updated.Spec.ValuesFiles = []string{}
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.ObservedGeneration > updated.Status.ObservedGeneration &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- _, exists := helmChart.Values["testDefault"]
- Expect(exists).To(BeFalse())
- Expect(helmChart.Values["testOverride"]).To(BeTrue())
- })
- })
-
- It("Creates artifacts with .tgz file", func() {
- fs := memfs.New()
- gitrepo, err := git.Init(memory.NewStorage(), fs)
- Expect(err).NotTo(HaveOccurred())
-
- wt, err := gitrepo.Worktree()
- Expect(err).NotTo(HaveOccurred())
-
- u, err := url.Parse(gitServer.HTTPAddress())
- Expect(err).NotTo(HaveOccurred())
- u.Path = path.Join(u.Path, fmt.Sprintf("repository-%s.git", randStringRunes(5)))
-
- _, err = gitrepo.CreateRemote(&config.RemoteConfig{
- Name: "origin",
- URLs: []string{u.String()},
- })
- Expect(err).NotTo(HaveOccurred())
-
- chartDir := "testdata/charts/helmchart"
- helmChart, err := loader.LoadDir(chartDir)
- Expect(err).NotTo(HaveOccurred())
-
- chartPackagePath, err := ioutil.TempDir("", fmt.Sprintf("chartpackage-%s-%s", helmChart.Name(), randStringRunes(5)))
- Expect(err).NotTo(HaveOccurred())
- defer os.RemoveAll(chartPackagePath)
-
- pkg, err := chartutil.Save(helmChart, chartPackagePath)
- Expect(err).NotTo(HaveOccurred())
-
- b, err := ioutil.ReadFile(pkg)
- Expect(err).NotTo(HaveOccurred())
-
- tgz := filepath.Base(pkg)
- ff, err := fs.Create(tgz)
- Expect(err).NotTo(HaveOccurred())
-
- _, err = ff.Write(b)
- Expect(err).NotTo(HaveOccurred())
-
- ff.Close()
- _, err = wt.Add(tgz)
- Expect(err).NotTo(HaveOccurred())
-
- _, err = wt.Commit("Helm chart", &git.CommitOptions{Author: &object.Signature{
- Name: "John Doe",
- Email: "john@example.com",
- When: time.Now(),
- }})
- Expect(err).NotTo(HaveOccurred())
-
- err = gitrepo.Push(&git.PushOptions{})
- Expect(err).NotTo(HaveOccurred())
-
- repositoryKey := types.NamespacedName{
- Name: fmt.Sprintf("git-repository-sample-%s", randStringRunes(5)),
- Namespace: namespace.Name,
- }
- repository := &sourcev1.GitRepository{
- ObjectMeta: metav1.ObjectMeta{
- Name: repositoryKey.Name,
- Namespace: repositoryKey.Namespace,
- },
- Spec: sourcev1.GitRepositorySpec{
- URL: u.String(),
- Interval: metav1.Duration{Duration: indexInterval},
- },
- }
- Expect(k8sClient.Create(context.Background(), repository)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), repository)
-
- key := types.NamespacedName{
- Name: "helmchart-sample-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- chart := &sourcev1.HelmChart{
- ObjectMeta: metav1.ObjectMeta{
- Name: key.Name,
- Namespace: key.Namespace,
- },
- Spec: sourcev1.HelmChartSpec{
- Chart: tgz,
- Version: "*",
- SourceRef: sourcev1.LocalHelmChartSourceReference{
- Kind: sourcev1.GitRepositoryKind,
- Name: repositoryKey.Name,
- },
- Interval: metav1.Duration{Duration: pullInterval},
- },
- }
- Expect(k8sClient.Create(context.Background(), chart)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), chart)
-
- By("Expecting artifact")
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact != nil &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- })
- })
-
- Context("HelmChart from GitRepository with HelmRepository dependency", func() {
- var (
- namespace *corev1.Namespace
- gitServer *gittestserver.GitServer
- helmServer *helmtestserver.HelmServer
- err error
- )
-
- BeforeEach(func() {
- namespace = &corev1.Namespace{
- ObjectMeta: metav1.ObjectMeta{Name: "test-git-repository-" + randStringRunes(5)},
- }
- err = k8sClient.Create(context.Background(), namespace)
- Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
-
- gitServer, err = gittestserver.NewTempGitServer()
- Expect(err).NotTo(HaveOccurred())
- gitServer.AutoCreate()
- Expect(gitServer.StartHTTP()).To(Succeed())
-
- helmServer, err = helmtestserver.NewTempHelmServer()
- Expect(err).To(Succeed())
- helmServer.Start()
- })
-
- AfterEach(func() {
- gitServer.StopHTTP()
- os.RemoveAll(gitServer.Root())
-
- os.RemoveAll(helmServer.Root())
- helmServer.Stop()
-
- err = k8sClient.Delete(context.Background(), namespace)
- Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
- })
-
- It("Creates artifacts for", func() {
- helmServer.Stop()
- var username, password = "john", "doe"
- helmServer.WithMiddleware(func(handler http.Handler) http.Handler {
- return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- u, p, ok := r.BasicAuth()
- if !ok || username != u || password != p {
- w.WriteHeader(401)
- return
- }
- handler.ServeHTTP(w, r)
- })
- })
- helmServer.Start()
-
- Expect(helmServer.PackageChart(path.Join("testdata/charts/helmchart"))).Should(Succeed())
- Expect(helmServer.GenerateIndex()).Should(Succeed())
-
- secretKey := types.NamespacedName{
- Name: "helmrepository-auth-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- secret := &corev1.Secret{
- ObjectMeta: metav1.ObjectMeta{
- Name: secretKey.Name,
- Namespace: secretKey.Namespace,
- },
- StringData: map[string]string{
- "username": username,
- "password": password,
- },
- }
- Expect(k8sClient.Create(context.Background(), secret)).Should(Succeed())
-
- By("Creating repository and waiting for artifact")
- helmRepositoryKey := types.NamespacedName{
- Name: "helmrepository-sample-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- helmRepository := &sourcev1.HelmRepository{
- ObjectMeta: metav1.ObjectMeta{
- Name: helmRepositoryKey.Name,
- Namespace: helmRepositoryKey.Namespace,
- },
- Spec: sourcev1.HelmRepositorySpec{
- URL: helmServer.URL(),
- SecretRef: &meta.LocalObjectReference{
- Name: secretKey.Name,
- },
- Interval: metav1.Duration{Duration: pullInterval},
- },
- }
- Expect(k8sClient.Create(context.Background(), helmRepository)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), helmRepository)
-
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), helmRepositoryKey, helmRepository)
- return helmRepository.Status.Artifact != nil
- }, timeout, interval).Should(BeTrue())
-
- fs := memfs.New()
- gitrepo, err := git.Init(memory.NewStorage(), fs)
- Expect(err).NotTo(HaveOccurred())
-
- wt, err := gitrepo.Worktree()
- Expect(err).NotTo(HaveOccurred())
-
- u, err := url.Parse(gitServer.HTTPAddress())
- Expect(err).NotTo(HaveOccurred())
- u.Path = path.Join(u.Path, fmt.Sprintf("repository-%s.git", randStringRunes(5)))
-
- _, err = gitrepo.CreateRemote(&config.RemoteConfig{
- Name: "origin",
- URLs: []string{u.String()},
- })
- Expect(err).NotTo(HaveOccurred())
-
- chartDir := "testdata/charts/helmchartwithdeps"
- Expect(filepath.Walk(chartDir, func(p string, fi os.FileInfo, err error) error {
- if err != nil {
- return err
- }
-
- switch {
- case fi.Mode().IsDir():
- return fs.MkdirAll(p, os.ModeDir)
- case !fi.Mode().IsRegular():
- return nil
- }
-
- b, err := ioutil.ReadFile(p)
- if err != nil {
- return err
- }
-
- ff, err := fs.Create(p)
- if err != nil {
- return err
- }
- if _, err := ff.Write(b); err != nil {
- return err
- }
- _ = ff.Close()
- _, err = wt.Add(p)
-
- return err
- })).To(Succeed())
-
- By("Configuring the chart dependency")
- filePath := fs.Join(chartDir, chartutil.ChartfileName)
- f, err := fs.OpenFile(filePath, os.O_RDWR, os.FileMode(0600))
- Expect(err).NotTo(HaveOccurred())
-
- b := make([]byte, 2048)
- n, err := f.Read(b)
- Expect(err).NotTo(HaveOccurred())
- b = b[0:n]
-
- err = f.Close()
- Expect(err).NotTo(HaveOccurred())
-
- y := new(helmchart.Metadata)
- err = yaml.Unmarshal(b, y)
- Expect(err).NotTo(HaveOccurred())
-
- y.Dependencies = []*helmchart.Dependency{
- {
- Name: "helmchart",
- Version: ">=0.1.0",
- Repository: helmRepository.Spec.URL,
- },
- }
-
- b, err = yaml.Marshal(y)
- Expect(err).NotTo(HaveOccurred())
-
- ff, err := fs.Create(filePath)
- Expect(err).NotTo(HaveOccurred())
-
- _, err = ff.Write(b)
- Expect(err).NotTo(HaveOccurred())
-
- err = ff.Close()
- Expect(err).NotTo(HaveOccurred())
-
- _, err = wt.Commit("Helm charts", &git.CommitOptions{
- Author: &object.Signature{
- Name: "John Doe",
- Email: "john@example.com",
- When: time.Now(),
- },
- All: true,
- })
- Expect(err).NotTo(HaveOccurred())
-
- err = gitrepo.Push(&git.PushOptions{})
- Expect(err).NotTo(HaveOccurred())
-
- repositoryKey := types.NamespacedName{
- Name: fmt.Sprintf("git-repository-sample-%s", randStringRunes(5)),
- Namespace: namespace.Name,
- }
- repository := &sourcev1.GitRepository{
- ObjectMeta: metav1.ObjectMeta{
- Name: repositoryKey.Name,
- Namespace: repositoryKey.Namespace,
- },
- Spec: sourcev1.GitRepositorySpec{
- URL: u.String(),
- Interval: metav1.Duration{Duration: indexInterval},
- },
- }
- Expect(k8sClient.Create(context.Background(), repository)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), repository)
-
- key := types.NamespacedName{
- Name: "helmchart-sample-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- chart := &sourcev1.HelmChart{
- ObjectMeta: metav1.ObjectMeta{
- Name: key.Name,
- Namespace: key.Namespace,
- },
- Spec: sourcev1.HelmChartSpec{
- Chart: "testdata/charts/helmchartwithdeps",
- Version: "*",
- SourceRef: sourcev1.LocalHelmChartSourceReference{
- Kind: sourcev1.GitRepositoryKind,
- Name: repositoryKey.Name,
- },
- Interval: metav1.Duration{Duration: pullInterval},
- },
- }
- Expect(k8sClient.Create(context.Background(), chart)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), chart)
-
- By("Expecting artifact")
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact != nil &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(helmChart.Values["testDefault"]).To(BeTrue())
- Expect(helmChart.Values["testOverride"]).To(BeFalse())
-
- When("Setting valid valuesFiles attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFiles = []string{
- "./testdata/charts/helmchartwithdeps/values.yaml",
- "./testdata/charts/helmchartwithdeps/override.yaml",
- }
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact.Checksum != updated.Status.Artifact.Checksum &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(helmChart.Values["testDefault"]).To(BeTrue())
- Expect(helmChart.Values["testOverride"]).To(BeTrue())
- })
-
- When("Setting invalid valuesFiles attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFiles = []string{
- "./testdata/charts/helmchartwithdeps/values.yaml",
- "./testdata/charts/helmchartwithdeps/invalid.yaml",
- }
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.ObservedGeneration > updated.Status.ObservedGeneration &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(helmChart.Values["testDefault"]).To(BeTrue())
- Expect(helmChart.Values["testOverride"]).To(BeTrue())
- })
-
- When("Setting valid valuesFiles and valuesFile attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFile = "./testdata/charts/helmchartwithdeps/values.yaml"
- updated.Spec.ValuesFiles = []string{
- "./testdata/charts/helmchartwithdeps/override.yaml",
- }
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact.Checksum != updated.Status.Artifact.Checksum &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(helmChart.Values["testDefault"]).To(BeTrue())
- Expect(helmChart.Values["testOverride"]).To(BeTrue())
- })
-
- When("Setting valid valuesFile attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFile = "./testdata/charts/helmchartwithdeps/override.yaml"
- updated.Spec.ValuesFiles = []string{}
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact.Checksum != updated.Status.Artifact.Checksum &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- _, exists := helmChart.Values["testDefault"]
- Expect(exists).To(BeFalse())
- Expect(helmChart.Values["testOverride"]).To(BeTrue())
- })
-
- When("Setting invalid valuesFile attribute", func() {
- updated := &sourcev1.HelmChart{}
- Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed())
- updated.Spec.ValuesFile = "./testdata/charts/helmchartwithdeps/invalid.yaml"
- updated.Spec.ValuesFiles = []string{}
- Expect(k8sClient.Update(context.Background(), updated)).To(Succeed())
- got := &sourcev1.HelmChart{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.ObservedGeneration > updated.Status.ObservedGeneration &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
- f, err := os.Stat(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- Expect(f.Size()).To(BeNumerically(">", 0))
- helmChart, err := loader.Load(storage.LocalPath(*got.Status.Artifact))
- Expect(err).NotTo(HaveOccurred())
- _, exists := helmChart.Values["testDefault"]
- Expect(exists).To(BeFalse())
- Expect(helmChart.Values["testOverride"]).To(BeTrue())
- })
- })
- })
-})
+func TestHelmChartReconciler_Reconcile(t *testing.T) {
+ g := NewWithT(t)
+
+ sourceObj := &sourcev1.HelmRepository{
+ ObjectMeta: metav1.ObjectMeta{
+ GenerateName: "helm-chart-reconcile",
+ Namespace: "default",
+ },
+ Spec: sourcev1.HelmRepositorySpec{
+ URL: "https://stefanprodan.github.io/podinfo",
+ },
+ }
+ g.Expect(env.Create(ctx, sourceObj)).To(Succeed())
+
+ obj := &sourcev1.HelmChart{
+ ObjectMeta: metav1.ObjectMeta{
+ GenerateName: "helmchart-reconcile-",
+ Namespace: "default",
+ },
+ Spec: sourcev1.HelmChartSpec{
+ Chart: "podinfo",
+ SourceRef: sourcev1.LocalHelmChartSourceReference{
+ Kind: sourcev1.HelmRepositoryKind,
+ Name: sourceObj.Name,
+ },
+ },
+ }
+ g.Expect(env.Create(ctx, obj)).To(Succeed())
+
+ key := client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace}
+
+ // Wait for finalizer to be set
+ g.Eventually(func() bool {
+ if err := env.Get(ctx, key, obj); err != nil {
+ return false
+ }
+ return len(obj.Finalizers) > 0
+ }, timeout).Should(BeTrue())
+
+ // Wait for HelmChart to be Ready
+ g.Eventually(func() bool {
+ if err := env.Get(ctx, key, obj); err != nil {
+ return false
+ }
+
+ if !conditions.Has(obj, sourcev1.ArtifactAvailableCondition) ||
+ !conditions.Has(obj, sourcev1.ChartReconciled) ||
+ !conditions.Has(obj, sourcev1.SourceAvailableCondition) ||
+ !conditions.Has(obj, meta.ReadyCondition) ||
+ conditions.Has(obj, meta.StalledCondition) ||
+ conditions.Has(obj, meta.ReconcilingCondition) ||
+ obj.Status.Artifact == nil {
+ return false
+ }
+
+ readyCondition := conditions.Get(obj, meta.ReadyCondition)
+
+ return readyCondition.Status == metav1.ConditionTrue &&
+ obj.Generation == readyCondition.ObservedGeneration
+ }, timeout, 1*time.Second).Should(BeTrue())
+
+ g.Expect(env.Delete(ctx, obj)).To(Succeed())
+
+ // Wait for HelmChart to be deleted
+ g.Eventually(func() bool {
+ if err := env.Get(ctx, key, obj); err != nil {
+ return apierrors.IsNotFound(err)
+ }
+ return false
+ }, timeout).Should(BeTrue())
+}
func Test_validHelmChartName(t *testing.T) {
tests := []struct {
diff --git a/controllers/helmrepository_controller.go b/controllers/helmrepository_controller.go
index b7f8cd516..c4c563fef 100644
--- a/controllers/helmrepository_controller.go
+++ b/controllers/helmrepository_controller.go
@@ -21,17 +21,17 @@ import (
"context"
"fmt"
"net/url"
+ "os"
"time"
+ "github.com/fluxcd/pkg/runtime/conditions"
+ "github.com/fluxcd/pkg/runtime/patch"
"github.com/go-logr/logr"
"helm.sh/helm/v3/pkg/getter"
corev1 "k8s.io/api/core/v1"
- apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
- kuberecorder "k8s.io/client-go/tools/record"
- "k8s.io/client-go/tools/reference"
+ kerrors "k8s.io/apimachinery/pkg/util/errors"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -40,8 +40,8 @@ import (
"sigs.k8s.io/yaml"
"github.com/fluxcd/pkg/apis/meta"
+ helper "github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/events"
- "github.com/fluxcd/pkg/runtime/metrics"
"github.com/fluxcd/pkg/runtime/predicates"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
@@ -56,12 +56,11 @@ import (
// HelmRepositoryReconciler reconciles a HelmRepository object
type HelmRepositoryReconciler struct {
client.Client
- Scheme *runtime.Scheme
- Storage *Storage
- Getters getter.Providers
- EventRecorder kuberecorder.EventRecorder
- ExternalEventRecorder *events.Recorder
- MetricsRecorder *metrics.Recorder
+ helper.Events
+ helper.Metrics
+
+ Getters getter.Providers
+ Storage *Storage
}
type HelmRepositoryReconcilerOptions struct {
@@ -80,306 +79,319 @@ func (r *HelmRepositoryReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager,
Complete(r)
}
-func (r *HelmRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+func (r *HelmRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, retErr error) {
start := time.Now()
log := logr.FromContext(ctx)
- var repository sourcev1.HelmRepository
- if err := r.Get(ctx, req.NamespacedName, &repository); err != nil {
+ // Fetch the HelmRepository
+ obj := &sourcev1.HelmRepository{}
+ if err := r.Get(ctx, req.NamespacedName, obj); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
- // Add our finalizer if it does not exist
- if !controllerutil.ContainsFinalizer(&repository, sourcev1.SourceFinalizer) {
- controllerutil.AddFinalizer(&repository, sourcev1.SourceFinalizer)
- if err := r.Update(ctx, &repository); err != nil {
- log.Error(err, "unable to register finalizer")
- return ctrl.Result{}, err
- }
- }
-
- // Examine if the object is under deletion
- if !repository.ObjectMeta.DeletionTimestamp.IsZero() {
- return r.reconcileDelete(ctx, repository)
- }
+ // Record suspended status metric
+ r.RecordSuspend(ctx, obj, obj.Spec.Suspend)
- // Return early if the object is suspended.
- if repository.Spec.Suspend {
+ // Return early if the object is suspended
+ if obj.Spec.Suspend {
log.Info("Reconciliation is suspended for this object")
return ctrl.Result{}, nil
}
- // record reconciliation duration
- if r.MetricsRecorder != nil {
- objRef, err := reference.GetReference(r.Scheme, &repository)
- if err != nil {
- return ctrl.Result{}, err
- }
- defer r.MetricsRecorder.RecordDuration(*objRef, start)
+ // Initialize the patch helper
+ patchHelper, err := patch.NewHelper(obj, r.Client)
+ if err != nil {
+ return ctrl.Result{}, err
}
- // set initial status
- if resetRepository, ok := r.resetStatus(repository); ok {
- repository = resetRepository
- if err := r.updateStatus(ctx, req, repository.Status); err != nil {
- log.Error(err, "unable to update status")
- return ctrl.Result{Requeue: true}, err
+ // Always attempt to patch the object and status after each
+ // reconciliation
+ defer func() {
+ // Record the value of the reconciliation request, if any
+ if v, ok := meta.ReconcileAnnotationValue(obj.GetAnnotations()); ok {
+ obj.Status.SetLastHandledReconcileRequest(v)
}
- r.recordReadiness(ctx, repository)
+
+ // Summarize Ready condition
+ conditions.SetSummary(obj,
+ meta.ReadyCondition,
+ conditions.WithConditions(
+ sourcev1.ArtifactAvailableCondition,
+ sourcev1.SourceAvailableCondition,
+ ),
+ )
+
+ // Patch the object, ignoring conflicts on the conditions owned by
+ // this controller
+ patchOpts := []patch.Option{
+ patch.WithOwnedConditions{
+ Conditions: []string{
+ sourcev1.ArtifactAvailableCondition,
+ sourcev1.SourceAvailableCondition,
+ meta.ReadyCondition,
+ meta.ReconcilingCondition,
+ meta.StalledCondition,
+ },
+ },
+ }
+
+ // Determine if the resource is still being reconciled, or if
+ // it has stalled, and record this observation
+ if retErr == nil && (result.IsZero() || !result.Requeue) {
+ // We are no longer reconciling
+ conditions.Delete(obj, meta.ReconcilingCondition)
+
+ // We have now observed this generation
+ patchOpts = append(patchOpts, patch.WithStatusObservedGeneration{})
+
+ readyCondition := conditions.Get(obj, meta.ReadyCondition)
+ switch readyCondition.Status {
+ case metav1.ConditionFalse:
+ // As we are no longer reconciling and the end-state
+ // is not ready, the reconciliation has stalled
+ conditions.MarkStalled(obj, readyCondition.Reason, readyCondition.Message)
+ case metav1.ConditionTrue:
+ // As we are no longer reconciling and the end-state
+ // is ready, the reconciliation is no longer stalled
+ conditions.Delete(obj, meta.StalledCondition)
+ }
+ }
+
+ // Finally, patch the resource
+ if err := patchHelper.Patch(ctx, obj, patchOpts...); err != nil {
+ retErr = kerrors.NewAggregate([]error{retErr, err})
+ }
+
+ // Always record readiness and duration metrics
+ r.Metrics.RecordReadiness(ctx, obj)
+ r.Metrics.RecordDuration(ctx, obj, start)
+ }()
+
+ // Add finalizer first if not exist to avoid the race condition
+ // between init and delete
+ if !controllerutil.ContainsFinalizer(obj, sourcev1.SourceFinalizer) {
+ controllerutil.AddFinalizer(obj, sourcev1.SourceFinalizer)
+ return ctrl.Result{Requeue: true}, nil
}
- // record the value of the reconciliation request, if any
- // TODO(hidde): would be better to defer this in combination with
- // always patching the status sub-resource after a reconciliation.
- if v, ok := meta.ReconcileAnnotationValue(repository.GetAnnotations()); ok {
- repository.Status.SetLastHandledReconcileRequest(v)
+ // Examine if the object is under deletion
+ if !obj.ObjectMeta.DeletionTimestamp.IsZero() {
+ return r.reconcileDelete(ctx, obj)
}
- // purge old artifacts from storage
- if err := r.gc(repository); err != nil {
- log.Error(err, "unable to purge old artifacts")
+ // Reconcile actual object
+ return r.reconcile(ctx, obj)
+}
+
+func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.HelmRepository) (ctrl.Result, error) {
+ // Mark the resource as under reconciliation
+ conditions.MarkReconciling(obj, "Reconciling", "")
+
+ // Reconcile the storage data
+ if result, err := r.reconcileStorage(ctx, obj); err != nil {
+ return result, err
+ }
+
+ // Reconcile the source from upstream
+ var index helm.ChartRepository
+ var artifact sourcev1.Artifact
+ if result, err := r.reconcileSource(ctx, obj, &artifact, &index); err != nil || conditions.IsFalse(obj, sourcev1.SourceAvailableCondition) {
+ return result, err
}
- // reconcile repository by downloading the index.yaml file
- reconciledRepository, reconcileErr := r.reconcile(ctx, *repository.DeepCopy())
+ // Reconcile the artifact to storage
+ if result, err := r.reconcileArtifact(ctx, obj, artifact, index); err != nil {
+ return result, err
+ }
+
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+}
- // update status with the reconciliation result
- if err := r.updateStatus(ctx, req, reconciledRepository.Status); err != nil {
- log.Error(err, "unable to update status")
- return ctrl.Result{Requeue: true}, err
+// reconcileStorage reconciles the storage data for the given object
+// by observing if the artifact in the status still exists, and
+// ensuring the URLs are up-to-date with the current hostname
+// configuration.
+func (r *HelmRepositoryReconciler) reconcileStorage(ctx context.Context, obj *sourcev1.HelmRepository) (ctrl.Result, error) {
+ // Purge old artifacts from the storage
+ if err := r.garbageCollect(obj); err != nil {
+ r.Events.Eventf(ctx, obj, events.EventSeverityError, "GarbageCollectionFailed", "Garbage collection failed: %s", err)
}
- // if reconciliation failed, record the failure and requeue immediately
- if reconcileErr != nil {
- r.event(ctx, reconciledRepository, events.EventSeverityError, reconcileErr.Error())
- r.recordReadiness(ctx, reconciledRepository)
- return ctrl.Result{Requeue: true}, reconcileErr
+ // Determine if the artifact is still in storage
+ if artifact := obj.GetArtifact(); artifact != nil && !r.Storage.ArtifactExist(*artifact) {
+ obj.Status.Artifact = nil
+ obj.Status.URL = ""
}
- // emit revision change event
- if repository.Status.Artifact == nil || reconciledRepository.Status.Artifact.Revision != repository.Status.Artifact.Revision {
- r.event(ctx, reconciledRepository, events.EventSeverityInfo, sourcev1.HelmRepositoryReadyMessage(reconciledRepository))
+ // Record that we have no artifact
+ if obj.GetArtifact() == nil {
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, "NoArtifactFound", "No artifact for resource in storage")
+ return ctrl.Result{Requeue: true}, nil
}
- r.recordReadiness(ctx, reconciledRepository)
- log.Info(fmt.Sprintf("Reconciliation finished in %s, next run in %s",
- time.Now().Sub(start).String(),
- repository.GetInterval().Duration.String(),
- ))
+ // Always update URLs to ensure hostname is up-to-date
+ r.Storage.SetArtifactURL(obj.GetArtifact())
+ obj.Status.URL = r.Storage.SetHostname(obj.Status.URL)
- return ctrl.Result{RequeueAfter: repository.GetInterval().Duration}, nil
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
}
-func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sourcev1.HelmRepository) (sourcev1.HelmRepository, error) {
+// reconcileSource reconciles the Helm repository index from upstream
+// to the
+// given directory path while using the information on the object to
+// determine authentication and checkout strategies.
+// On a successful checkout of HEAD the artifact metadata the given
+// pointer is set to a new artifact.
+func (r *HelmRepositoryReconciler) reconcileSource(ctx context.Context, obj *sourcev1.HelmRepository, artifact *sourcev1.Artifact, index *helm.ChartRepository) (ctrl.Result, error) {
+ // Configure Helm client to access repository
clientOpts := []getter.Option{
- getter.WithURL(repository.Spec.URL),
- getter.WithTimeout(repository.Spec.Timeout.Duration),
- getter.WithPassCredentialsAll(repository.Spec.PassCredentials),
+ getter.WithTimeout(obj.Spec.Timeout.Duration),
+ getter.WithURL(obj.Spec.URL),
+ getter.WithPassCredentialsAll(obj.Spec.PassCredentials),
}
- if repository.Spec.SecretRef != nil {
+ if obj.Spec.SecretRef != nil {
+ // Attempt to retrieve secret
name := types.NamespacedName{
- Namespace: repository.GetNamespace(),
- Name: repository.Spec.SecretRef.Name,
+ Namespace: obj.GetNamespace(),
+ Name: obj.Spec.SecretRef.Name,
}
-
var secret corev1.Secret
- err := r.Client.Get(ctx, name, &secret)
- if err != nil {
- err = fmt.Errorf("auth secret error: %w", err)
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
+ if err := r.Client.Get(ctx, name, &secret); err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.AuthenticationFailedReason, "Failed to get secret %s: %s", name.String(), err.Error())
+ r.Events.Event(ctx, obj, events.EventSeverityError, sourcev1.AuthenticationFailedReason, conditions.Get(obj, sourcev1.SourceAvailableCondition).Message)
+ // Return transient errors but wait for next interval on not found
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, client.IgnoreNotFound(err)
}
- opts, cleanup, err := helm.ClientOptionsFromSecret(secret)
+ // Get client options from secret
+ tmpDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s-source-", obj.Name, obj.Namespace))
+ if err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.StorageOperationFailedReason, "Could not create temporary directory for credentials: %s", err.Error())
+ r.Events.Event(ctx, obj, events.EventSeverityError, sourcev1.AuthenticationFailedReason, conditions.Get(obj, sourcev1.SourceAvailableCondition).Message)
+ return ctrl.Result{}, err
+ }
+ defer os.RemoveAll(tmpDir)
+ opts, err := helm.ClientOptionsFromSecret(secret, tmpDir)
if err != nil {
- err = fmt.Errorf("auth options error: %w", err)
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.AuthenticationFailedReason, "Failed to configure Helm client with secret data: %s", err)
+ r.Events.Event(ctx, obj, events.EventSeverityError, sourcev1.AuthenticationFailedReason, conditions.Get(obj, sourcev1.SourceAvailableCondition).Message)
+ // Return err as the content of the secret may change
+ return ctrl.Result{}, err
}
- defer cleanup()
clientOpts = append(clientOpts, opts...)
}
- chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Getters, clientOpts)
+ // Construct Helm chart repository with options and download index
+ newIndex, err := helm.NewChartRepository(obj.Spec.URL, r.Getters, clientOpts)
if err != nil {
switch err.(type) {
case *url.Error:
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.URLInvalidReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.URLInvalidReason, "Invalid Helm repository URL: %s", err.Error())
+ return ctrl.Result{}, nil
default:
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.URLInvalidReason, "Failed to construct Helm client: %s", err.Error())
+ return ctrl.Result{}, nil
}
}
- if err := chartRepo.DownloadIndex(); err != nil {
- err = fmt.Errorf("failed to download repository index: %w", err)
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err
+ if err := newIndex.DownloadIndex(); err != nil {
+ conditions.MarkFalse(obj, sourcev1.SourceAvailableCondition, sourcev1.URLInvalidReason, "Failed to download Helm repository index: %s", err.Error())
+ return ctrl.Result{}, err
}
- indexBytes, err := yaml.Marshal(&chartRepo.Index)
- if err != nil {
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- hash := r.Storage.Checksum(bytes.NewReader(indexBytes))
- artifact := r.Storage.NewArtifactFor(repository.Kind,
- repository.ObjectMeta.GetObjectMeta(),
- hash,
- fmt.Sprintf("index-%s.yaml", hash))
- // return early on unchanged index
- if apimeta.IsStatusConditionTrue(repository.Status.Conditions, meta.ReadyCondition) && repository.GetArtifact().HasRevision(artifact.Revision) {
- if artifact.URL != repository.GetArtifact().URL {
- r.Storage.SetArtifactURL(repository.GetArtifact())
- repository.Status.URL = r.Storage.SetHostname(repository.Status.URL)
- }
- return repository, nil
- }
+ *index = *newIndex
- // create artifact dir
- err = r.Storage.MkdirAll(artifact)
- if err != nil {
- err = fmt.Errorf("unable to create repository index directory: %w", err)
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
+ // Create potential new artifact
+ *artifact = r.Storage.NewArtifactFor(obj.Kind,
+ obj.ObjectMeta.GetObjectMeta(),
+ index.Checksum,
+ fmt.Sprintf("index-%s.yaml", index.Checksum))
+ conditions.MarkTrue(obj, sourcev1.SourceAvailableCondition, sourcev1.IndexationSucceededReason, "Downloaded index revision %s from %s", artifact.Revision, obj.Spec.URL)
- // acquire lock
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
+}
+
+// reconcileArtifact reconciles the Helm repository index to the artifact
+// storage.
+// On a successful archive, the artifact and includes in the status of
+// the given object are set, and the symlink in the storage is updated
+// to its path.
+func (r *HelmRepositoryReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, index helm.ChartRepository) (ctrl.Result, error) {
+ // The artifact is up-to-date
+ if obj.GetArtifact().HasRevision(artifact.Revision) {
+ logr.FromContext(ctx).Info("Artifact is up-to-date")
+ conditions.MarkTrue(obj, sourcev1.ArtifactAvailableCondition, "ArchivedArtifact", "Compressed source to artifact with revision %s", artifact.Revision)
+ return ctrl.Result{RequeueAfter: obj.GetInterval().Duration}, nil
+ }
+
+ // Ensure artifact directory exists and acquire lock
+ if err := r.Storage.MkdirAll(artifact); err != nil {
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to create directory: %s", err)
+ return ctrl.Result{}, err
+ }
unlock, err := r.Storage.Lock(artifact)
if err != nil {
- err = fmt.Errorf("unable to acquire lock: %w", err)
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to acquire lock: %s", err)
+ return ctrl.Result{}, err
}
defer unlock()
- // save artifact to storage
- if err := r.Storage.AtomicWriteFile(&artifact, bytes.NewReader(indexBytes), 0644); err != nil {
- err = fmt.Errorf("unable to write repository index file: %w", err)
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
+ // Save artifact to storage
+ b, err := yaml.Marshal(index.Index)
+ if err != nil {
+ return ctrl.Result{}, err
+ }
+ if err := r.Storage.AtomicWriteFile(&artifact, bytes.NewReader(b), 0644); err != nil {
+ conditions.MarkFalse(obj, sourcev1.ArtifactAvailableCondition, sourcev1.StorageOperationFailedReason, "Failed to write Helm repository index to storage: %s", err.Error())
+ return ctrl.Result{}, err
}
- // update index symlink
- indexURL, err := r.Storage.Symlink(artifact, "index.yaml")
+ // Record it on the object
+ obj.Status.Artifact = artifact.DeepCopy()
+ conditions.MarkTrue(obj, sourcev1.ArtifactAvailableCondition, sourcev1.IndexationSucceededReason, "Artifact revision %s", artifact.Revision)
+ r.Events.EventWithMetaf(ctx, obj, map[string]string{
+ "revision": obj.GetArtifact().Revision,
+ }, events.EventSeverityInfo, sourcev1.GitOperationSucceedReason, conditions.Get(obj, sourcev1.ArtifactAvailableCondition).Message)
+
+ // Update symlink on a "best effort" basis
+ url, err := r.Storage.Symlink(artifact, "index.yaml")
if err != nil {
- err = fmt.Errorf("storage error: %w", err)
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
+ r.Events.Eventf(ctx, obj, events.EventSeverityError, sourcev1.StorageOperationFailedReason, "Failed to update status URL symlink: %s", err)
+ }
+ if url != "" {
+ obj.Status.URL = url
}
- message := fmt.Sprintf("Fetched revision: %s", artifact.Revision)
- return sourcev1.HelmRepositoryReady(repository, artifact, indexURL, sourcev1.IndexationSucceededReason, message), nil
+ return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil
}
-func (r *HelmRepositoryReconciler) reconcileDelete(ctx context.Context, repository sourcev1.HelmRepository) (ctrl.Result, error) {
- // Our finalizer is still present, so lets handle garbage collection
- if err := r.gc(repository); err != nil {
- r.event(ctx, repository, events.EventSeverityError,
- fmt.Sprintf("garbage collection for deleted resource failed: %s", err.Error()))
+// reconcileDelete reconciles the delete of an object by garbage
+// collecting all artifacts for the object in the artifact storage,
+// if successful, the finalizer is removed from the object.
+func (r *HelmRepositoryReconciler) reconcileDelete(ctx context.Context, obj *sourcev1.HelmRepository) (ctrl.Result, error) {
+ // Garbage collect the resource's artifacts
+ if err := r.garbageCollect(obj); err != nil {
+ r.Events.Eventf(ctx, obj, events.EventSeverityError, "GarbageCollectionFailed", "Garbage collection for deleted resource failed: %s", err)
// Return the error so we retry the failed garbage collection
return ctrl.Result{}, err
}
- // Record deleted status
- r.recordReadiness(ctx, repository)
-
- // Remove our finalizer from the list and update it
- controllerutil.RemoveFinalizer(&repository, sourcev1.SourceFinalizer)
- if err := r.Update(ctx, &repository); err != nil {
- return ctrl.Result{}, err
- }
+ // Remove our finalizer from the list
+ controllerutil.RemoveFinalizer(obj, sourcev1.SourceFinalizer)
// Stop reconciliation as the object is being deleted
return ctrl.Result{}, nil
}
-// resetStatus returns a modified v1beta1.HelmRepository and a boolean indicating
-// if the status field has been reset.
-func (r *HelmRepositoryReconciler) resetStatus(repository sourcev1.HelmRepository) (sourcev1.HelmRepository, bool) {
- // We do not have an artifact, or it does no longer exist
- if repository.GetArtifact() == nil || !r.Storage.ArtifactExist(*repository.GetArtifact()) {
- repository = sourcev1.HelmRepositoryProgressing(repository)
- repository.Status.Artifact = nil
- return repository, true
- }
- if repository.Generation != repository.Status.ObservedGeneration {
- return sourcev1.HelmRepositoryProgressing(repository), true
- }
- return repository, false
-}
-
-// gc performs a garbage collection for the given v1beta1.HelmRepository.
+// garbageCollect performs a garbage collection for the given v1beta1.HelmRepository.
// It removes all but the current artifact except for when the
// deletion timestamp is set, which will result in the removal of
// all artifacts for the resource.
-func (r *HelmRepositoryReconciler) gc(repository sourcev1.HelmRepository) error {
- if !repository.DeletionTimestamp.IsZero() {
- return r.Storage.RemoveAll(r.Storage.NewArtifactFor(repository.Kind, repository.GetObjectMeta(), "", "*"))
+func (r *HelmRepositoryReconciler) garbageCollect(obj *sourcev1.HelmRepository) error {
+ if !obj.DeletionTimestamp.IsZero() {
+ return r.Storage.RemoveAll(r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), "", "*"))
}
- if repository.GetArtifact() != nil {
- return r.Storage.RemoveAllButCurrent(*repository.GetArtifact())
+ if obj.GetArtifact() != nil {
+ return r.Storage.RemoveAllButCurrent(*obj.GetArtifact())
}
return nil
}
-
-// event emits a Kubernetes event and forwards the event to notification controller if configured
-func (r *HelmRepositoryReconciler) event(ctx context.Context, repository sourcev1.HelmRepository, severity, msg string) {
- log := logr.FromContext(ctx)
- if r.EventRecorder != nil {
- r.EventRecorder.Eventf(&repository, "Normal", severity, msg)
- }
- if r.ExternalEventRecorder != nil {
- objRef, err := reference.GetReference(r.Scheme, &repository)
- if err != nil {
- log.Error(err, "unable to send event")
- return
- }
-
- if err := r.ExternalEventRecorder.Eventf(*objRef, nil, severity, severity, msg); err != nil {
- log.Error(err, "unable to send event")
- return
- }
- }
-}
-
-func (r *HelmRepositoryReconciler) recordReadiness(ctx context.Context, repository sourcev1.HelmRepository) {
- log := logr.FromContext(ctx)
- if r.MetricsRecorder == nil {
- return
- }
- objRef, err := reference.GetReference(r.Scheme, &repository)
- if err != nil {
- log.Error(err, "unable to record readiness metric")
- return
- }
- if rc := apimeta.FindStatusCondition(repository.Status.Conditions, meta.ReadyCondition); rc != nil {
- r.MetricsRecorder.RecordCondition(*objRef, *rc, !repository.DeletionTimestamp.IsZero())
- } else {
- r.MetricsRecorder.RecordCondition(*objRef, metav1.Condition{
- Type: meta.ReadyCondition,
- Status: metav1.ConditionUnknown,
- }, !repository.DeletionTimestamp.IsZero())
- }
-}
-
-func (r *HelmRepositoryReconciler) updateStatus(ctx context.Context, req ctrl.Request, newStatus sourcev1.HelmRepositoryStatus) error {
- var repository sourcev1.HelmRepository
- if err := r.Get(ctx, req.NamespacedName, &repository); err != nil {
- return err
- }
-
- patch := client.MergeFrom(repository.DeepCopy())
- repository.Status = newStatus
-
- return r.Status().Patch(ctx, &repository, patch)
-}
-
-func (r *HelmRepositoryReconciler) recordSuspension(ctx context.Context, hr sourcev1.HelmRepository) {
- if r.MetricsRecorder == nil {
- return
- }
- log := logr.FromContext(ctx)
-
- objRef, err := reference.GetReference(r.Scheme, &hr)
- if err != nil {
- log.Error(err, "unable to record suspended metric")
- return
- }
-
- if !hr.DeletionTimestamp.IsZero() {
- r.MetricsRecorder.RecordSuspend(*objRef, false)
- } else {
- r.MetricsRecorder.RecordSuspend(*objRef, hr.Spec.Suspend)
- }
-}
diff --git a/controllers/helmrepository_controller_test.go b/controllers/helmrepository_controller_test.go
index 126ed11c5..dfe8755d7 100644
--- a/controllers/helmrepository_controller_test.go
+++ b/controllers/helmrepository_controller_test.go
@@ -17,395 +17,78 @@ limitations under the License.
package controllers
import (
- "context"
- "net/http"
- "os"
- "path"
- "strings"
- "time"
+ "testing"
- . "github.com/onsi/ginkgo"
+ sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
. "github.com/onsi/gomega"
- corev1 "k8s.io/api/core/v1"
+ "helm.sh/helm/v3/pkg/getter"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/types"
+ "sigs.k8s.io/controller-runtime/pkg/client"
"github.com/fluxcd/pkg/apis/meta"
- "github.com/fluxcd/pkg/helmtestserver"
-
- sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+ "github.com/fluxcd/pkg/runtime/conditions"
)
-var _ = Describe("HelmRepositoryReconciler", func() {
-
- const (
- timeout = time.Second * 30
- interval = time.Second * 1
- indexInterval = time.Second * 2
- repositoryTimeout = time.Second * 5
- )
-
- Context("HelmRepository", func() {
- var (
- namespace *corev1.Namespace
- helmServer *helmtestserver.HelmServer
- err error
- )
-
- BeforeEach(func() {
- namespace = &corev1.Namespace{
- ObjectMeta: metav1.ObjectMeta{Name: "helm-repository-" + randStringRunes(5)},
- }
- err = k8sClient.Create(context.Background(), namespace)
- Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
-
- helmServer, err = helmtestserver.NewTempHelmServer()
- Expect(err).To(Succeed())
- })
-
- AfterEach(func() {
- os.RemoveAll(helmServer.Root())
- helmServer.Stop()
-
- Eventually(func() error {
- return k8sClient.Delete(context.Background(), namespace)
- }, timeout, interval).Should(Succeed(), "failed to delete test namespace")
- })
-
- It("Creates artifacts for", func() {
- helmServer.Start()
-
- Expect(helmServer.PackageChart(path.Join("testdata/charts/helmchart"))).Should(Succeed())
- Expect(helmServer.GenerateIndex()).Should(Succeed())
-
- key := types.NamespacedName{
- Name: "helmrepository-sample-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- created := &sourcev1.HelmRepository{
- ObjectMeta: metav1.ObjectMeta{
- Name: key.Name,
- Namespace: key.Namespace,
- },
- Spec: sourcev1.HelmRepositorySpec{
- URL: helmServer.URL(),
- Interval: metav1.Duration{Duration: indexInterval},
- Timeout: &metav1.Duration{Duration: repositoryTimeout},
- },
- }
- Expect(k8sClient.Create(context.Background(), created)).Should(Succeed())
-
- By("Expecting artifact")
- got := &sourcev1.HelmRepository{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact != nil && storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
-
- By("Updating the chart index")
- // Regenerating the index is sufficient to make the revision change
- Expect(helmServer.GenerateIndex()).Should(Succeed())
-
- By("Expecting revision change and GC")
- Eventually(func() bool {
- now := &sourcev1.HelmRepository{}
- _ = k8sClient.Get(context.Background(), key, now)
- // Test revision change and garbage collection
- return now.Status.Artifact.Revision != got.Status.Artifact.Revision &&
- !storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
-
- updated := &sourcev1.HelmRepository{}
- Expect(k8sClient.Get(context.Background(), key, updated)).Should(Succeed())
- updated.Spec.URL = "invalid#url?"
- Expect(k8sClient.Update(context.Background(), updated)).Should(Succeed())
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, updated)
- for _, c := range updated.Status.Conditions {
- if c.Reason == sourcev1.IndexationFailedReason {
- return true
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
- Expect(updated.Status.Artifact).ToNot(BeNil())
-
- By("Expecting to delete successfully")
- got = &sourcev1.HelmRepository{}
- Eventually(func() error {
- _ = k8sClient.Get(context.Background(), key, got)
- return k8sClient.Delete(context.Background(), got)
- }, timeout, interval).Should(Succeed())
-
- By("Expecting delete to finish")
- Eventually(func() error {
- r := &sourcev1.HelmRepository{}
- return k8sClient.Get(context.Background(), key, r)
- }, timeout, interval).ShouldNot(Succeed())
-
- exists := func(path string) bool {
- // wait for tmp sync on macOS
- time.Sleep(time.Second)
- _, err := os.Stat(path)
- return err == nil
- }
-
- By("Expecting GC after delete")
- Eventually(exists(got.Status.Artifact.Path), timeout, interval).ShouldNot(BeTrue())
- })
-
- It("Handles timeout", func() {
- helmServer.Start()
-
- Expect(helmServer.PackageChart(path.Join("testdata/charts/helmchart"))).Should(Succeed())
- Expect(helmServer.GenerateIndex()).Should(Succeed())
-
- key := types.NamespacedName{
- Name: "helmrepository-sample-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- created := &sourcev1.HelmRepository{
- ObjectMeta: metav1.ObjectMeta{
- Name: key.Name,
- Namespace: key.Namespace,
- },
- Spec: sourcev1.HelmRepositorySpec{
- URL: helmServer.URL(),
- Interval: metav1.Duration{Duration: indexInterval},
- },
- }
- Expect(k8sClient.Create(context.Background(), created)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), created)
-
- By("Expecting index download to succeed")
- Eventually(func() bool {
- got := &sourcev1.HelmRepository{}
- _ = k8sClient.Get(context.Background(), key, got)
- for _, condition := range got.Status.Conditions {
- if condition.Reason == sourcev1.IndexationSucceededReason {
- return true
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
-
- By("Expecting index download to timeout")
- updated := &sourcev1.HelmRepository{}
- Expect(k8sClient.Get(context.Background(), key, updated)).Should(Succeed())
- updated.Spec.Timeout = &metav1.Duration{Duration: time.Microsecond}
- Expect(k8sClient.Update(context.Background(), updated)).Should(Succeed())
- Eventually(func() string {
- got := &sourcev1.HelmRepository{}
- _ = k8sClient.Get(context.Background(), key, got)
- for _, condition := range got.Status.Conditions {
- if condition.Reason == sourcev1.IndexationFailedReason {
- return condition.Message
- }
- }
- return ""
- }, timeout, interval).Should(MatchRegexp("(?i)timeout"))
- })
-
- It("Authenticates when basic auth credentials are provided", func() {
- helmServer, err = helmtestserver.NewTempHelmServer()
- Expect(err).NotTo(HaveOccurred())
-
- var username, password = "john", "doe"
- helmServer.WithMiddleware(func(handler http.Handler) http.Handler {
- return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- u, p, ok := r.BasicAuth()
- if !ok || username != u || password != p {
- w.WriteHeader(401)
- return
- }
- handler.ServeHTTP(w, r)
- })
- })
- defer os.RemoveAll(helmServer.Root())
- defer helmServer.Stop()
- helmServer.Start()
-
- Expect(helmServer.PackageChart(path.Join("testdata/charts/helmchart"))).Should(Succeed())
- Expect(helmServer.GenerateIndex()).Should(Succeed())
-
- secretKey := types.NamespacedName{
- Name: "helmrepository-auth-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- secret := &corev1.Secret{
- ObjectMeta: metav1.ObjectMeta{
- Name: secretKey.Name,
- Namespace: secretKey.Namespace,
- },
- }
- Expect(k8sClient.Create(context.Background(), secret)).Should(Succeed())
-
- key := types.NamespacedName{
- Name: "helmrepository-sample-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- created := &sourcev1.HelmRepository{
- ObjectMeta: metav1.ObjectMeta{
- Name: key.Name,
- Namespace: key.Namespace,
- },
- Spec: sourcev1.HelmRepositorySpec{
- URL: helmServer.URL(),
- SecretRef: &meta.LocalObjectReference{
- Name: secretKey.Name,
- },
- Interval: metav1.Duration{Duration: indexInterval},
- },
- }
- Expect(k8sClient.Create(context.Background(), created)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), created)
-
- By("Expecting 401")
- Eventually(func() bool {
- got := &sourcev1.HelmRepository{}
- _ = k8sClient.Get(context.Background(), key, got)
- for _, c := range got.Status.Conditions {
- if c.Reason == sourcev1.IndexationFailedReason &&
- strings.Contains(c.Message, "401 Unauthorized") {
- return true
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
-
- By("Expecting missing field error")
- secret.Data = map[string][]byte{
- "username": []byte(username),
- }
- Expect(k8sClient.Update(context.Background(), secret)).Should(Succeed())
- Eventually(func() bool {
- got := &sourcev1.HelmRepository{}
- _ = k8sClient.Get(context.Background(), key, got)
- for _, c := range got.Status.Conditions {
- if c.Reason == sourcev1.AuthenticationFailedReason {
- return true
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
-
- By("Expecting artifact")
- secret.Data["password"] = []byte(password)
- Expect(k8sClient.Update(context.Background(), secret)).Should(Succeed())
- Eventually(func() bool {
- got := &sourcev1.HelmRepository{}
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact != nil &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
-
- By("Expecting missing secret error")
- Expect(k8sClient.Delete(context.Background(), secret)).Should(Succeed())
- got := &sourcev1.HelmRepository{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- for _, c := range got.Status.Conditions {
- if c.Reason == sourcev1.AuthenticationFailedReason {
- return true
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
- Expect(got.Status.Artifact).ShouldNot(BeNil())
- })
-
- It("Authenticates when TLS credentials are provided", func() {
- err = helmServer.StartTLS(examplePublicKey, examplePrivateKey, exampleCA, "example.com")
- Expect(err).NotTo(HaveOccurred())
-
- Expect(helmServer.PackageChart(path.Join("testdata/charts/helmchart"))).Should(Succeed())
- Expect(helmServer.GenerateIndex()).Should(Succeed())
-
- secretKey := types.NamespacedName{
- Name: "helmrepository-auth-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- secret := &corev1.Secret{
- ObjectMeta: metav1.ObjectMeta{
- Name: secretKey.Name,
- Namespace: secretKey.Namespace,
- },
- }
- Expect(k8sClient.Create(context.Background(), secret)).Should(Succeed())
-
- key := types.NamespacedName{
- Name: "helmrepository-sample-" + randStringRunes(5),
- Namespace: namespace.Name,
- }
- created := &sourcev1.HelmRepository{
- ObjectMeta: metav1.ObjectMeta{
- Name: key.Name,
- Namespace: key.Namespace,
- },
- Spec: sourcev1.HelmRepositorySpec{
- URL: helmServer.URL(),
- SecretRef: &meta.LocalObjectReference{
- Name: secretKey.Name,
- },
- Interval: metav1.Duration{Duration: indexInterval},
- },
- }
- Expect(k8sClient.Create(context.Background(), created)).Should(Succeed())
- defer k8sClient.Delete(context.Background(), created)
-
- By("Expecting unknown authority error")
- Eventually(func() bool {
- got := &sourcev1.HelmRepository{}
- _ = k8sClient.Get(context.Background(), key, got)
- for _, c := range got.Status.Conditions {
- if c.Reason == sourcev1.IndexationFailedReason &&
- strings.Contains(c.Message, "certificate signed by unknown authority") {
- return true
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
-
- By("Expecting missing field error")
- secret.Data = map[string][]byte{
- "certFile": examplePublicKey,
- }
- Expect(k8sClient.Update(context.Background(), secret)).Should(Succeed())
- Eventually(func() bool {
- got := &sourcev1.HelmRepository{}
- _ = k8sClient.Get(context.Background(), key, got)
- for _, c := range got.Status.Conditions {
- if c.Reason == sourcev1.AuthenticationFailedReason {
- return true
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
-
- By("Expecting artifact")
- secret.Data["keyFile"] = examplePrivateKey
- secret.Data["caFile"] = exampleCA
- Expect(k8sClient.Update(context.Background(), secret)).Should(Succeed())
- Eventually(func() bool {
- got := &sourcev1.HelmRepository{}
- _ = k8sClient.Get(context.Background(), key, got)
- return got.Status.Artifact != nil &&
- storage.ArtifactExist(*got.Status.Artifact)
- }, timeout, interval).Should(BeTrue())
+var (
+ testGetters = getter.Providers{
+ getter.Provider{
+ Schemes: []string{"http", "https"},
+ New: getter.NewHTTPGetter,
+ },
+ }
+)
- By("Expecting missing secret error")
- Expect(k8sClient.Delete(context.Background(), secret)).Should(Succeed())
- got := &sourcev1.HelmRepository{}
- Eventually(func() bool {
- _ = k8sClient.Get(context.Background(), key, got)
- for _, c := range got.Status.Conditions {
- if c.Reason == sourcev1.AuthenticationFailedReason {
- return true
- }
- }
- return false
- }, timeout, interval).Should(BeTrue())
- Expect(got.Status.Artifact).ShouldNot(BeNil())
- })
- })
-})
+func TestHelmRepositoryReconciler_Reconcile(t *testing.T) {
+ g := NewWithT(t)
+
+ obj := &sourcev1.HelmRepository{
+ ObjectMeta: metav1.ObjectMeta{
+ GenerateName: "helmrepository-reconcile-",
+ Namespace: "default",
+ },
+ Spec: sourcev1.HelmRepositorySpec{
+ URL: "https://stefanprodan.github.io/podinfo",
+ },
+ }
+ g.Expect(env.Create(ctx, obj)).To(Succeed())
+
+ key := client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace}
+
+ // Wait for finalizer to be set
+ g.Eventually(func() bool {
+ if err := env.Get(ctx, key, obj); err != nil {
+ return false
+ }
+ return len(obj.Finalizers) > 0
+ }, timeout).Should(BeTrue())
+
+ // Wait for HelmRepository to be Ready
+ g.Eventually(func() bool {
+ if err := env.Get(ctx, key, obj); err != nil {
+ return false
+ }
+
+ if !conditions.Has(obj, sourcev1.ArtifactAvailableCondition) ||
+ !conditions.Has(obj, sourcev1.SourceAvailableCondition) ||
+ !conditions.Has(obj, meta.ReadyCondition) ||
+ obj.Status.Artifact == nil {
+ return false
+ }
+
+ readyCondition := conditions.Get(obj, meta.ReadyCondition)
+
+ return readyCondition.Status == metav1.ConditionTrue &&
+ obj.Generation == readyCondition.ObservedGeneration
+ }, timeout).Should(BeTrue())
+
+ g.Expect(env.Delete(ctx, obj)).To(Succeed())
+
+ // Wait for HelmRepository to be deleted
+ g.Eventually(func() bool {
+ if err := env.Get(ctx, key, obj); err != nil {
+ return apierrors.IsNotFound(err)
+ }
+ return false
+ }, timeout).Should(BeTrue())
+}
diff --git a/controllers/matchers_test.go b/controllers/matchers_test.go
new file mode 100644
index 000000000..2dfa671f5
--- /dev/null
+++ b/controllers/matchers_test.go
@@ -0,0 +1,67 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+ "fmt"
+
+ sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+ . "github.com/onsi/gomega"
+ "github.com/onsi/gomega/types"
+)
+
+// MatchArtifact returns a custom matcher to check equality of a sourcev1.Artifact, the timestamp and URL are ignored.
+func MatchArtifact(expected *sourcev1.Artifact) types.GomegaMatcher {
+ return &matchArtifact{
+ expected: expected,
+ }
+}
+
+type matchArtifact struct {
+ expected *sourcev1.Artifact
+}
+
+func (m matchArtifact) Match(actual interface{}) (success bool, err error) {
+ actualArtifact, ok := actual.(*sourcev1.Artifact)
+ if !ok {
+ return false, fmt.Errorf("actual should be a pointer to an Artifact")
+ }
+
+ if ok, _ := BeNil().Match(m.expected); ok {
+ return BeNil().Match(actual)
+ }
+
+ if ok, err = Equal(m.expected.Path).Match(actualArtifact.Path); !ok {
+ return ok, err
+ }
+ if ok, err = Equal(m.expected.Revision).Match(actualArtifact.Revision); !ok {
+ return ok, err
+ }
+ if ok, err = Equal(m.expected.Checksum).Match(actualArtifact.Checksum); !ok {
+ return ok, err
+ }
+
+ return ok, err
+}
+
+func (m matchArtifact) FailureMessage(actual interface{}) (message string) {
+ return fmt.Sprintf("expected\n\t%#v\nto match\n\t%#v\n", actual, m.expected)
+}
+
+func (m matchArtifact) NegatedFailureMessage(actual interface{}) (message string) {
+ return fmt.Sprintf("expected\n\t%#v\nto not match\n\t%#v\n", actual, m.expected)
+}
diff --git a/controllers/storage.go b/controllers/storage.go
index 09b3b760a..ba1d27972 100644
--- a/controllers/storage.go
+++ b/controllers/storage.go
@@ -23,13 +23,13 @@ import (
"fmt"
"hash"
"io"
- "io/ioutil"
"net/url"
"os"
"path/filepath"
"strings"
"time"
+ securejoin "github.com/cyphar/filepath-securejoin"
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -81,7 +81,11 @@ func (s Storage) SetArtifactURL(artifact *sourcev1.Artifact) {
if artifact.Path == "" {
return
}
- artifact.URL = fmt.Sprintf("http://%s/%s", s.Hostname, artifact.Path)
+ format := "http://%s/%s"
+ if strings.HasPrefix(s.Hostname, "http://") || strings.HasPrefix(s.Hostname, "https://") {
+ format = "%s/%s"
+ }
+ artifact.URL = fmt.Sprintf(format, s.Hostname, strings.TrimLeft(artifact.Path, "/"))
}
// SetHostname sets the hostname of the given URL string to the current Storage.Hostname
@@ -174,7 +178,7 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv
}
localPath := s.LocalPath(*artifact)
- tf, err := ioutil.TempFile(filepath.Split(localPath))
+ tf, err := os.CreateTemp(filepath.Split(localPath))
if err != nil {
return err
}
@@ -272,7 +276,7 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv
// If successful, it sets the checksum and last update time on the artifact.
func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader, mode os.FileMode) (err error) {
localPath := s.LocalPath(*artifact)
- tf, err := ioutil.TempFile(filepath.Split(localPath))
+ tf, err := os.CreateTemp(filepath.Split(localPath))
if err != nil {
return err
}
@@ -311,7 +315,7 @@ func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader,
// If successful, it sets the checksum and last update time on the artifact.
func (s *Storage) Copy(artifact *sourcev1.Artifact, reader io.Reader) (err error) {
localPath := s.LocalPath(*artifact)
- tf, err := ioutil.TempFile(filepath.Split(localPath))
+ tf, err := os.CreateTemp(filepath.Split(localPath))
if err != nil {
return err
}
@@ -354,10 +358,11 @@ func (s *Storage) CopyFromPath(artifact *sourcev1.Artifact, path string) (err er
return s.Copy(artifact, f)
}
-// CopyToPath copies the contents of the given artifact to the path.
+// CopyToPath copies the contents in the (sub)path of the given artifact to the
+// given path.
func (s *Storage) CopyToPath(artifact *sourcev1.Artifact, subPath, toPath string) error {
// create a tmp directory to store artifact
- tmp, err := ioutil.TempDir("", "flux-include")
+ tmp, err := os.MkdirTemp("", "flux-include-")
if err != nil {
return err
}
@@ -372,7 +377,7 @@ func (s *Storage) CopyToPath(artifact *sourcev1.Artifact, subPath, toPath string
defer f.Close()
// untar the artifact
- untarPath := filepath.Join(tmp, "tar")
+ untarPath := filepath.Join(tmp, "unpack")
if _, err = untar.Untar(f, untarPath); err != nil {
return err
}
@@ -434,7 +439,11 @@ func (s *Storage) LocalPath(artifact sourcev1.Artifact) string {
if artifact.Path == "" {
return ""
}
- return filepath.Join(s.BasePath, artifact.Path)
+ path, err := securejoin.SecureJoin(s.BasePath, artifact.Path)
+ if err != nil {
+ return ""
+ }
+ return path
}
// newHash returns a new SHA1 hash.
diff --git a/controllers/storage_test.go b/controllers/storage_test.go
index a79df6a14..8da8d49df 100644
--- a/controllers/storage_test.go
+++ b/controllers/storage_test.go
@@ -21,7 +21,6 @@ import (
"compress/gzip"
"fmt"
"io"
- "io/ioutil"
"os"
"path"
"path/filepath"
@@ -34,7 +33,7 @@ import (
)
func createStoragePath() (string, error) {
- return ioutil.TempDir("", "")
+ return os.MkdirTemp("", "")
}
func cleanupStoragePath(dir string) func() {
@@ -52,7 +51,7 @@ func TestStorageConstructor(t *testing.T) {
t.Fatal("nonexistent path was allowable in storage constructor")
}
- f, err := ioutil.TempFile(dir, "")
+ f, err := os.CreateTemp(dir, "")
if err != nil {
t.Fatalf("while creating temporary file: %v", err)
}
@@ -124,7 +123,7 @@ func TestStorage_Archive(t *testing.T) {
os.RemoveAll(dir)
}
}()
- dir, err = ioutil.TempDir("", "archive-test-files-")
+ dir, err = os.MkdirTemp("", "archive-test-files-")
if err != nil {
return
}
@@ -244,7 +243,7 @@ func TestStorage_Archive(t *testing.T) {
func TestStorageRemoveAllButCurrent(t *testing.T) {
t.Run("bad directory in archive", func(t *testing.T) {
- dir, err := ioutil.TempDir("", "")
+ dir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
diff --git a/controllers/suite_test.go b/controllers/suite_test.go
index 0dd4351a3..1e4d72b5f 100644
--- a/controllers/suite_test.go
+++ b/controllers/suite_test.go
@@ -17,166 +17,162 @@ limitations under the License.
package controllers
import (
- "io/ioutil"
+ "fmt"
"math/rand"
- "net/http"
"os"
"path/filepath"
"testing"
"time"
- . "github.com/onsi/ginkgo"
- . "github.com/onsi/gomega"
- "helm.sh/helm/v3/pkg/getter"
+ utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes/scheme"
- "k8s.io/client-go/rest"
ctrl "sigs.k8s.io/controller-runtime"
- "sigs.k8s.io/controller-runtime/pkg/client"
- "sigs.k8s.io/controller-runtime/pkg/envtest"
- "sigs.k8s.io/controller-runtime/pkg/envtest/printer"
- logf "sigs.k8s.io/controller-runtime/pkg/log"
- "sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+ "github.com/fluxcd/pkg/runtime/controller"
+ "github.com/fluxcd/pkg/testserver"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
// +kubebuilder:scaffold:imports
+ "github.com/fluxcd/pkg/runtime/testenv"
)
-// These tests use Ginkgo (BDD-style Go testing framework). Refer to
-// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
+// These tests make use of plain Go using Gomega for assertions.
+// At the beginning of every (sub)test Gomega can be initialized
+// using gomega.NewWithT.
+// Refer to http://onsi.github.io/gomega/ to learn more about
+// Gomega.
-var cfg *rest.Config
-var k8sClient client.Client
-var k8sManager ctrl.Manager
-var testEnv *envtest.Environment
-var storage *Storage
+const (
+ timeout = 10 * time.Second
+ interval = 1 * time.Second
+)
-var examplePublicKey []byte
-var examplePrivateKey []byte
-var exampleCA []byte
+var (
+ env *testenv.Environment
+ storage *Storage
+ server *testserver.ArtifactServer
+ eventsH controller.Events
+ metricsH controller.Metrics
+ ctx = ctrl.SetupSignalHandler()
+)
-func TestAPIs(t *testing.T) {
- RegisterFailHandler(Fail)
+var (
+ tlsPublicKey []byte
+ tlsPrivateKey []byte
+ tlsCA []byte
+)
- RunSpecsWithDefaultAndCustomReporters(t,
- "Controller Suite",
- []Reporter{printer.NewlineReporter{}})
+func init() {
+ rand.Seed(time.Now().UnixNano())
}
-var _ = BeforeSuite(func(done Done) {
- logf.SetLogger(
- zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)),
- )
-
- By("bootstrapping test environment")
- t := true
- if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
- testEnv = &envtest.Environment{
- UseExistingCluster: &t,
- }
- } else {
- testEnv = &envtest.Environment{
- CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")},
- }
- }
-
- var err error
- cfg, err = testEnv.Start()
- Expect(err).ToNot(HaveOccurred())
- Expect(cfg).ToNot(BeNil())
-
- err = sourcev1.AddToScheme(scheme.Scheme)
- Expect(err).NotTo(HaveOccurred())
-
- err = sourcev1.AddToScheme(scheme.Scheme)
- Expect(err).NotTo(HaveOccurred())
+func TestMain(m *testing.M) {
+ initTestTLS()
- err = sourcev1.AddToScheme(scheme.Scheme)
- Expect(err).NotTo(HaveOccurred())
+ utilruntime.Must(sourcev1.AddToScheme(scheme.Scheme))
- // +kubebuilder:scaffold:scheme
+ env = testenv.New(testenv.WithCRDPath(filepath.Join("..", "config", "crd", "bases")))
- Expect(loadExampleKeys()).To(Succeed())
+ var err error
+ server, err = testserver.NewTempArtifactServer()
+ if err != nil {
+ panic(fmt.Sprintf("Failed to create a temporary storage server: %v", err))
+ }
+ fmt.Println("Starting the test storage server")
+ server.Start()
- tmpStoragePath, err := ioutil.TempDir("", "source-controller-storage-")
- Expect(err).NotTo(HaveOccurred(), "failed to create tmp storage dir")
+ storage, err = newTestStorage(server.HTTPServer)
+ if err != nil {
+ panic(fmt.Sprintf("Failed to create a test storage: %v", err))
+ }
- storage, err = NewStorage(tmpStoragePath, "localhost:5050", time.Second*30)
- Expect(err).NotTo(HaveOccurred(), "failed to create tmp storage")
- // serve artifacts from the filesystem, as done in main.go
- fs := http.FileServer(http.Dir(tmpStoragePath))
- http.Handle("/", fs)
- go http.ListenAndServe(":5050", nil)
+ eventsH = controller.MakeEvents(env, "test", nil)
+ metricsH = controller.MustMakeMetrics(env)
- k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{
- Scheme: scheme.Scheme,
- })
- Expect(err).ToNot(HaveOccurred())
+ if err := (&GitRepositoryReconciler{
+ Client: env,
+ Events: eventsH,
+ Metrics: metricsH,
+ Storage: storage,
+ }).SetupWithManager(env); err != nil {
+ panic(fmt.Sprintf("Failed to start GitRepositoryReconciler: %v", err))
+ }
- err = (&GitRepositoryReconciler{
- Client: k8sManager.GetClient(),
- Scheme: scheme.Scheme,
+ if err := (&HelmRepositoryReconciler{
+ Client: env,
+ Events: eventsH,
+ Metrics: metricsH,
+ Getters: testGetters,
Storage: storage,
- }).SetupWithManager(k8sManager)
- Expect(err).ToNot(HaveOccurred(), "failed to setup GtRepositoryReconciler")
+ }).SetupWithManager(env); err != nil {
+ panic(fmt.Sprintf("Failed to start HelmRepositoryReconciler: %v", err))
+ }
- err = (&HelmRepositoryReconciler{
- Client: k8sManager.GetClient(),
- Scheme: scheme.Scheme,
+ if err := (&BucketReconciler{
+ Client: env,
+ Events: eventsH,
+ Metrics: metricsH,
Storage: storage,
- Getters: getter.Providers{getter.Provider{
- Schemes: []string{"http", "https"},
- New: getter.NewHTTPGetter,
- }},
- }).SetupWithManager(k8sManager)
- Expect(err).ToNot(HaveOccurred(), "failed to setup HelmRepositoryReconciler")
-
- err = (&HelmChartReconciler{
- Client: k8sManager.GetClient(),
- Scheme: scheme.Scheme,
+ }).SetupWithManager(env); err != nil {
+ panic(fmt.Sprintf("Failed to start BucketReconciler: %v", err))
+ }
+
+ if err := (&HelmChartReconciler{
+ Client: env,
+ Events: eventsH,
+ Metrics: metricsH,
+ Getters: testGetters,
Storage: storage,
- Getters: getter.Providers{getter.Provider{
- Schemes: []string{"http", "https"},
- New: getter.NewHTTPGetter,
- }},
- }).SetupWithManager(k8sManager)
- Expect(err).ToNot(HaveOccurred(), "failed to setup HelmChartReconciler")
+ }).SetupWithManager(env); err != nil {
+ panic(fmt.Sprintf("Failed to start HelmChartReconciler: %v", err))
+ }
go func() {
- err = k8sManager.Start(ctrl.SetupSignalHandler())
- Expect(err).ToNot(HaveOccurred())
+ fmt.Println("Starting the test environment")
+ if err := env.Start(ctx); err != nil {
+ panic(fmt.Sprintf("Failed to start the test environment manager: %v", err))
+ }
}()
+ <-env.Manager.Elected()
- k8sClient = k8sManager.GetClient()
- Expect(k8sClient).ToNot(BeNil())
+ code := m.Run()
- close(done)
-}, 60)
+ fmt.Println("Stopping the test environment")
+ if err := env.Stop(); err != nil {
+ panic(fmt.Sprintf("Failed to stop the test environment: %v", err))
+ }
-var _ = AfterSuite(func() {
- By("tearing down the test environment")
- if storage != nil {
- err := os.RemoveAll(storage.BasePath)
- Expect(err).NotTo(HaveOccurred())
+ fmt.Println("Stopping the storage server")
+ server.Stop()
+ if err := os.RemoveAll(server.Root()); err != nil {
+ panic(fmt.Sprintf("Failed to remove storage server dir: %v", err))
}
- err := testEnv.Stop()
- Expect(err).ToNot(HaveOccurred())
-})
-func init() {
- rand.Seed(time.Now().UnixNano())
+ os.Exit(code)
}
-func loadExampleKeys() (err error) {
- examplePublicKey, err = ioutil.ReadFile("testdata/certs/server.pem")
+func initTestTLS() {
+ var err error
+ tlsPublicKey, err = os.ReadFile("testdata/certs/server.pem")
+ if err != nil {
+ panic(err)
+ }
+ tlsPrivateKey, err = os.ReadFile("testdata/certs/server-key.pem")
if err != nil {
- return err
+ panic(err)
}
- examplePrivateKey, err = ioutil.ReadFile("testdata/certs/server-key.pem")
+ tlsCA, err = os.ReadFile("testdata/certs/ca.pem")
+ if err != nil {
+ panic(err)
+ }
+}
+
+func newTestStorage(s *testserver.HTTPServer) (*Storage, error) {
+ storage, err := NewStorage(s.Root(), s.URL(), timeout)
if err != nil {
- return err
+ return nil, err
}
- exampleCA, err = ioutil.ReadFile("testdata/certs/ca.pem")
- return err
+ return storage, nil
}
var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz1234567890")
diff --git a/controllers/testdata/git/repository/.sourceignore b/controllers/testdata/git/repository/.sourceignore
new file mode 100644
index 000000000..989478d13
--- /dev/null
+++ b/controllers/testdata/git/repository/.sourceignore
@@ -0,0 +1 @@
+**.txt
diff --git a/controllers/testdata/git/repository/foo.txt b/controllers/testdata/git/repository/foo.txt
new file mode 100644
index 000000000..e69de29bb
diff --git a/controllers/testdata/git/repository/manifest.yaml b/controllers/testdata/git/repository/manifest.yaml
new file mode 100644
index 000000000..220e1b33e
--- /dev/null
+++ b/controllers/testdata/git/repository/manifest.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: dummy
diff --git a/go.mod b/go.mod
index ddb83ca4d..fdfd259b2 100644
--- a/go.mod
+++ b/go.mod
@@ -6,15 +6,15 @@ replace github.com/fluxcd/source-controller/api => ./api
require (
github.com/Masterminds/semver/v3 v3.1.1
- github.com/blang/semver/v4 v4.0.0
github.com/cyphar/filepath-securejoin v0.2.2
- github.com/fluxcd/pkg/apis/meta v0.10.0
+ github.com/fluxcd/pkg/apis/meta v0.11.0-rc.1
github.com/fluxcd/pkg/gittestserver v0.3.0
github.com/fluxcd/pkg/gitutil v0.1.0
github.com/fluxcd/pkg/helmtestserver v0.2.0
github.com/fluxcd/pkg/lockedfile v0.1.0
- github.com/fluxcd/pkg/runtime v0.12.0
+ github.com/fluxcd/pkg/runtime v0.13.0-rc.2
github.com/fluxcd/pkg/ssh v0.1.0
+ github.com/fluxcd/pkg/testserver v0.1.0
github.com/fluxcd/pkg/untar v0.1.0
github.com/fluxcd/pkg/version v0.1.0
github.com/fluxcd/source-controller/api v0.15.3
@@ -23,17 +23,17 @@ require (
github.com/go-logr/logr v0.4.0
github.com/libgit2/git2go/v31 v31.4.14
github.com/minio/minio-go/v7 v7.0.10
- github.com/onsi/ginkgo v1.16.4
github.com/onsi/gomega v1.13.0
github.com/spf13/pflag v1.0.5
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b
- golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
+ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
gotest.tools v2.2.0+incompatible
helm.sh/helm/v3 v3.6.3
- k8s.io/api v0.21.1
- k8s.io/apimachinery v0.21.1
- k8s.io/client-go v0.21.1
- sigs.k8s.io/controller-runtime v0.9.0
+ k8s.io/api v0.21.2
+ k8s.io/apimachinery v0.21.2
+ k8s.io/client-go v0.21.2
+ k8s.io/utils v0.0.0-20210527160623-6fdb442a123b
+ sigs.k8s.io/controller-runtime v0.9.2
sigs.k8s.io/yaml v1.2.0
)
diff --git a/go.sum b/go.sum
index 5ccca6197..3119b0a1c 100644
--- a/go.sum
+++ b/go.sum
@@ -111,10 +111,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
-github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
-github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/bshuster-repo/logrus-logstash-hook v0.4.1 h1:pgAtgj+A31JBVtEHu2uHuEx0n+2ukqUJnS2vVe5pQNA=
github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
@@ -229,8 +226,9 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZM
github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fluxcd/pkg/apis/meta v0.10.0 h1:N7wVGHC1cyPdT87hrDC7UwCwRwnZdQM46PBSLjG2rlE=
github.com/fluxcd/pkg/apis/meta v0.10.0/go.mod h1:CW9X9ijMTpNe7BwnokiUOrLl/h13miwVr/3abEQLbKE=
+github.com/fluxcd/pkg/apis/meta v0.11.0-rc.1 h1:RHHrztAFv9wmjM+Pk7Svt1UdD+1SdnQSp76MWFiM7Hg=
+github.com/fluxcd/pkg/apis/meta v0.11.0-rc.1/go.mod h1:yUblM2vg+X8TE3A2VvJfdhkGmg+uqBlSPkLk7dxi0UM=
github.com/fluxcd/pkg/gittestserver v0.3.0 h1:6aa30mybecBwBWaJ2IEk7pQzefWnjWjxkTSrHMHawvg=
github.com/fluxcd/pkg/gittestserver v0.3.0/go.mod h1:8j36Z6B0BuKNZZ6exAWoyDEpyQoFcjz1IX3WBT7PZNg=
github.com/fluxcd/pkg/gitutil v0.1.0 h1:VO3kJY/CKOCO4ysDNqfdpTg04icAKBOSb3lbR5uE/IE=
@@ -239,8 +237,8 @@ github.com/fluxcd/pkg/helmtestserver v0.2.0 h1:cE7YHDmrWI0hr9QpaaeQ0vQ16Z0IiqZKi
github.com/fluxcd/pkg/helmtestserver v0.2.0/go.mod h1:Yie8n7xuu5Nvf1Q7302LKsubJhWpwzCaK0rLJvmF7aI=
github.com/fluxcd/pkg/lockedfile v0.1.0 h1:YsYFAkd6wawMCcD74ikadAKXA4s2sukdxrn7w8RB5eo=
github.com/fluxcd/pkg/lockedfile v0.1.0/go.mod h1:EJLan8t9MiOcgTs8+puDjbE6I/KAfHbdvIy9VUgIjm8=
-github.com/fluxcd/pkg/runtime v0.12.0 h1:BPZZ8bBkimpqGAPXqOf3LTaw+tcw6HgbWyCuzbbsJGs=
-github.com/fluxcd/pkg/runtime v0.12.0/go.mod h1:EyaTR2TOYcjL5U//C4yH3bt2tvTgIOSXpVRbWxUn/C4=
+github.com/fluxcd/pkg/runtime v0.13.0-rc.2 h1:+4uTEg+CU++hlr7NpOP4KYp60MtHDOgYvpz/74tbATg=
+github.com/fluxcd/pkg/runtime v0.13.0-rc.2/go.mod h1:TmvE2cJl1QkgZNmmlr7XUKoWDQwUiM5/wTUxXsQVoc8=
github.com/fluxcd/pkg/ssh v0.1.0 h1:cym2bqiT4IINOdLV0J6GYxer16Ii/7b2+RlK3CG+CnA=
github.com/fluxcd/pkg/ssh v0.1.0/go.mod h1:KUuVhaB6AX3IHTGCd3Ti/nesn5t1Nz4zCThFkkjHctM=
github.com/fluxcd/pkg/testserver v0.1.0 h1:nOYgM1HYFZNNSUFykuWDmrsxj4jQxUCvmLHWOQeqmyA=
@@ -841,6 +839,7 @@ github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6Ut
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI=
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE=
@@ -939,6 +938,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -979,8 +979,10 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -994,8 +996,9 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1052,10 +1055,14 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
@@ -1072,8 +1079,9 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs=
+golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1122,8 +1130,9 @@ golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.4 h1:cVngSRcfgyZCzys3KYOpCFa+4dqX/Oub9tAq00ttGVs=
+golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1258,27 +1267,34 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU=
-k8s.io/api v0.21.1 h1:94bbZ5NTjdINJEdzOkpS4vdPhkb1VFpTYC9zh43f75c=
k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s=
+k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y=
+k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU=
k8s.io/apiextensions-apiserver v0.21.0/go.mod h1:gsQGNtGkc/YoDG9loKI0V+oLZM4ljRPjc/sql5tmvzc=
-k8s.io/apiextensions-apiserver v0.21.1 h1:AA+cnsb6w7SZ1vD32Z+zdgfXdXY8X9uGX5bN6EoPEIo=
k8s.io/apiextensions-apiserver v0.21.1/go.mod h1:KESQFCGjqVcVsZ9g0xX5bacMjyX5emuWcS2arzdEouA=
+k8s.io/apiextensions-apiserver v0.21.2 h1:+exKMRep4pDrphEafRvpEi79wTnCFMqKf8LBtlA3yrE=
+k8s.io/apiextensions-apiserver v0.21.2/go.mod h1:+Axoz5/l3AYpGLlhJDfcVQzCerVYq3K3CvDMvw6X1RA=
k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY=
-k8s.io/apimachinery v0.21.1 h1:Q6XuHGlj2xc+hlMCvqyYfbv3H7SRGn2c8NycxJquDVs=
k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY=
+k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc=
+k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM=
k8s.io/apiserver v0.21.0/go.mod h1:w2YSn4/WIwYuxG5zJmcqtRdtqgW/J2JRgFAqps3bBpg=
-k8s.io/apiserver v0.21.1 h1:wTRcid53IhxhbFt4KTrFSw8tAncfr01EP91lzfcygVg=
k8s.io/apiserver v0.21.1/go.mod h1:nLLYZvMWn35glJ4/FZRhzLG/3MPxAaZTgV4FJZdr+tY=
+k8s.io/apiserver v0.21.2 h1:vfGLD8biFXHzbcIEXyW3652lDwkV8tZEFJAaS2iuJlw=
+k8s.io/apiserver v0.21.2/go.mod h1:lN4yBoGyiNT7SC1dmNk0ue6a5Wi6O3SWOIw91TsucQw=
k8s.io/cli-runtime v0.21.0 h1:/V2Kkxtf6x5NI2z+Sd/mIrq4FQyQ8jzZAUD6N5RnN7Y=
k8s.io/cli-runtime v0.21.0/go.mod h1:XoaHP93mGPF37MkLbjGVYqg3S1MnsFdKtiA/RZzzxOo=
k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA=
-k8s.io/client-go v0.21.1 h1:bhblWYLZKUu+pm50plvQF8WpY6TXdRRtcS/K9WauOj4=
k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs=
+k8s.io/client-go v0.21.2 h1:Q1j4L/iMN4pTw6Y4DWppBoUxgKO8LbffEMVEV00MUp0=
+k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA=
k8s.io/code-generator v0.21.0/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q=
k8s.io/code-generator v0.21.1/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q=
+k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U=
k8s.io/component-base v0.21.0/go.mod h1:qvtjz6X0USWXbgmbfXR+Agik4RZ3jv2Bgr5QnZzdPYw=
-k8s.io/component-base v0.21.1 h1:iLpj2btXbR326s/xNQWmPNGu0gaYSjzn7IN/5i28nQw=
k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkIIyKA=
+k8s.io/component-base v0.21.2 h1:EsnmFFoJ86cEywC0DoIkAUiEV6fjgauNugiw1lmIjs4=
+k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc=
k8s.io/component-helpers v0.21.0/go.mod h1:tezqefP7lxfvJyR+0a+6QtVrkZ/wIkyMLK4WcQ3Cj8U=
k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
@@ -1299,8 +1315,10 @@ rsc.io/letsencrypt v0.0.3/go.mod h1:buyQKZ6IXrRnB7TdkHP0RyEybLx18HHyOSoTyoOLqNY=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/controller-runtime v0.9.0 h1:ZIZ/dtpboPSbZYY7uUz2OzrkaBTOThx2yekLtpGB+zY=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
sigs.k8s.io/controller-runtime v0.9.0/go.mod h1:TgkfvrhhEw3PlI0BRL/5xM+89y3/yc0ZDfdbTl84si8=
+sigs.k8s.io/controller-runtime v0.9.2 h1:MnCAsopQno6+hI9SgJHKddzXpmv2wtouZz6931Eax+Q=
+sigs.k8s.io/controller-runtime v0.9.2/go.mod h1:TxzMCHyEUpaeuOiZx/bIdc2T81vfs/aKdvJt9wuu0zk=
sigs.k8s.io/kustomize/api v0.8.5 h1:bfCXGXDAbFbb/Jv5AhMj2BB8a5VAJuuQ5/KU69WtDjQ=
sigs.k8s.io/kustomize/api v0.8.5/go.mod h1:M377apnKT5ZHJS++6H4rQoCHmWtt6qTpp3mbe7p6OLY=
sigs.k8s.io/kustomize/cmd/config v0.9.7/go.mod h1:MvXCpHs77cfyxRmCNUQjIqCmZyYsbn5PyQpWiq44nW0=
diff --git a/internal/fs/fs.go b/internal/fs/fs.go
index c8ece049d..21cf96e69 100644
--- a/internal/fs/fs.go
+++ b/internal/fs/fs.go
@@ -8,7 +8,6 @@ import (
"errors"
"fmt"
"io"
- "io/ioutil"
"os"
"path/filepath"
"runtime"
@@ -92,7 +91,7 @@ func CopyDir(src, dst string) error {
return fmt.Errorf("cannot mkdir %s: %w", dst, err)
}
- entries, err := ioutil.ReadDir(src)
+ entries, err := os.ReadDir(src)
if err != nil {
return fmt.Errorf("cannot read directory %s: %w", dst, err)
}
diff --git a/internal/fs/fs_test.go b/internal/fs/fs_test.go
index eba87eba0..250556bc2 100644
--- a/internal/fs/fs_test.go
+++ b/internal/fs/fs_test.go
@@ -6,7 +6,6 @@ package fs
import (
"fmt"
- "io/ioutil"
"os"
"os/exec"
"path/filepath"
@@ -20,7 +19,7 @@ var (
)
func TestRenameWithFallback(t *testing.T) {
- dir, err := ioutil.TempDir("", "dep")
+ dir, err := os.MkdirTemp("", "dep")
if err != nil {
t.Fatal(err)
}
@@ -58,7 +57,7 @@ func TestRenameWithFallback(t *testing.T) {
}
func TestCopyDir(t *testing.T) {
- dir, err := ioutil.TempDir("", "dep")
+ dir, err := os.MkdirTemp("", "dep")
if err != nil {
t.Fatal(err)
}
@@ -119,7 +118,7 @@ func TestCopyDir(t *testing.T) {
t.Fatalf("expected %s to be a directory", dn)
}
- got, err := ioutil.ReadFile(fn)
+ got, err := os.ReadFile(fn)
if err != nil {
t.Fatal(err)
}
@@ -156,7 +155,7 @@ func TestCopyDirFail_SrcInaccessible(t *testing.T) {
})
defer cleanup()
- dir, err := ioutil.TempDir("", "dep")
+ dir, err := os.MkdirTemp("", "dep")
if err != nil {
t.Fatal(err)
}
@@ -178,7 +177,7 @@ func TestCopyDirFail_DstInaccessible(t *testing.T) {
var srcdir, dstdir string
- dir, err := ioutil.TempDir("", "dep")
+ dir, err := os.MkdirTemp("", "dep")
if err != nil {
t.Fatal(err)
}
@@ -203,7 +202,7 @@ func TestCopyDirFail_DstInaccessible(t *testing.T) {
func TestCopyDirFail_SrcIsNotDir(t *testing.T) {
var srcdir, dstdir string
- dir, err := ioutil.TempDir("", "dep")
+ dir, err := os.MkdirTemp("", "dep")
if err != nil {
t.Fatal(err)
}
@@ -229,7 +228,7 @@ func TestCopyDirFail_SrcIsNotDir(t *testing.T) {
func TestCopyDirFail_DstExists(t *testing.T) {
var srcdir, dstdir string
- dir, err := ioutil.TempDir("", "dep")
+ dir, err := os.MkdirTemp("", "dep")
if err != nil {
t.Fatal(err)
}
@@ -267,7 +266,7 @@ func TestCopyDirFailOpen(t *testing.T) {
var srcdir, dstdir string
- dir, err := ioutil.TempDir("", "dep")
+ dir, err := os.MkdirTemp("", "dep")
if err != nil {
t.Fatal(err)
}
@@ -298,7 +297,7 @@ func TestCopyDirFailOpen(t *testing.T) {
}
func TestCopyFile(t *testing.T) {
- dir, err := ioutil.TempDir("", "dep")
+ dir, err := os.MkdirTemp("", "dep")
if err != nil {
t.Fatal(err)
}
@@ -320,7 +319,7 @@ func TestCopyFile(t *testing.T) {
t.Fatal(err)
}
- got, err := ioutil.ReadFile(destf)
+ got, err := os.ReadFile(destf)
if err != nil {
t.Fatal(err)
}
@@ -345,7 +344,7 @@ func TestCopyFile(t *testing.T) {
}
func TestCopyFileSymlink(t *testing.T) {
- dir, err := ioutil.TempDir("", "dep")
+ dir, err := os.MkdirTemp("", "dep")
if err != nil {
t.Fatal(err)
}
@@ -370,11 +369,11 @@ func TestCopyFileSymlink(t *testing.T) {
// Creating symlinks on Windows require an additional permission
// regular users aren't granted usually. So we copy the file
// content as a fall back instead of creating a real symlink.
- srcb, err := ioutil.ReadFile(symlink)
+ srcb, err := os.ReadFile(symlink)
if err != nil {
t.Fatalf("%+v", err)
}
- dstb, err := ioutil.ReadFile(dst)
+ dstb, err := os.ReadFile(dst)
if err != nil {
t.Fatalf("%+v", err)
}
@@ -407,7 +406,7 @@ func TestCopyFileLongFilePath(t *testing.T) {
t.Skip("skipping on non-windows")
}
- dir, err := ioutil.TempDir("", "dep")
+ dir, err := os.MkdirTemp("", "dep")
if err != nil {
t.Fatal(err)
}
@@ -424,7 +423,7 @@ func TestCopyFileLongFilePath(t *testing.T) {
t.Fatalf("%+v", fmt.Errorf("unable to create temp directory: %s", fullPath))
}
- err = ioutil.WriteFile(fullPath+"src", []byte(nil), 0644)
+ err = os.WriteFile(fullPath+"src", []byte(nil), 0644)
if err != nil {
t.Fatalf("%+v", err)
}
@@ -445,7 +444,7 @@ func TestCopyFileFail(t *testing.T) {
t.Skip("skipping on windows")
}
- dir, err := ioutil.TempDir("", "dep")
+ dir, err := os.MkdirTemp("", "dep")
if err != nil {
t.Fatal(err)
}
@@ -485,7 +484,7 @@ func TestCopyFileFail(t *testing.T) {
// files this function creates. It is the caller's responsibility to call
// this function before the test is done running, whether there's an error or not.
func setupInaccessibleDir(t *testing.T, op func(dir string) error) func() {
- dir, err := ioutil.TempDir("", "dep")
+ dir, err := os.MkdirTemp("", "dep")
if err != nil {
t.Fatal(err)
return nil // keep compiler happy
@@ -569,7 +568,7 @@ func TestIsDir(t *testing.T) {
}
func TestIsSymlink(t *testing.T) {
- dir, err := ioutil.TempDir("", "dep")
+ dir, err := os.MkdirTemp("", "dep")
if err != nil {
t.Fatal(err)
}
diff --git a/internal/helm/dependency_manager_test.go b/internal/helm/dependency_manager_test.go
index a66977751..5a5def3c2 100644
--- a/internal/helm/dependency_manager_test.go
+++ b/internal/helm/dependency_manager_test.go
@@ -19,7 +19,7 @@ package helm
import (
"context"
"fmt"
- "io/ioutil"
+ "os"
"strings"
"testing"
@@ -170,7 +170,7 @@ func TestBuild_WithLocalChart(t *testing.T) {
func TestBuild_WithRemoteChart(t *testing.T) {
chart := chartFixture
- b, err := ioutil.ReadFile(helmPackageFile)
+ b, err := os.ReadFile(helmPackageFile)
if err != nil {
t.Fatal(err)
}
diff --git a/internal/helm/getter.go b/internal/helm/getter.go
index bc7435c4f..c92351184 100644
--- a/internal/helm/getter.go
+++ b/internal/helm/getter.go
@@ -17,34 +17,35 @@ limitations under the License.
package helm
import (
+ "bytes"
"fmt"
- "io/ioutil"
+ "io"
"os"
- "path/filepath"
"helm.sh/helm/v3/pkg/getter"
corev1 "k8s.io/api/core/v1"
)
-// ClientOptionsFromSecret constructs a getter.Option slice for the given secret.
+// ClientOptionsFromSecret constructs a getter.Option slice for the given secret,
+// storing any temporary credentials .
// It returns the slice, and a callback to remove temporary files.
-func ClientOptionsFromSecret(secret corev1.Secret) ([]getter.Option, func(), error) {
+func ClientOptionsFromSecret(secret corev1.Secret, dir string) ([]getter.Option, error) {
var opts []getter.Option
basicAuth, err := BasicAuthFromSecret(secret)
if err != nil {
- return opts, nil, err
+ return opts, err
}
if basicAuth != nil {
opts = append(opts, basicAuth)
}
- tlsClientConfig, cleanup, err := TLSClientConfigFromSecret(secret)
+ tlsClientConfig, err := TLSClientConfigFromSecret(secret, dir)
if err != nil {
- return opts, nil, err
+ return opts, err
}
if tlsClientConfig != nil {
opts = append(opts, tlsClientConfig)
}
- return opts, cleanup, nil
+ return opts, nil
}
// BasicAuthFromSecret attempts to construct a basic auth getter.Option for the
@@ -69,45 +70,54 @@ func BasicAuthFromSecret(secret corev1.Secret) (getter.Option, error) {
//
// Secrets with no certFile, keyFile, AND caFile are ignored, if only a
// certBytes OR keyBytes is defined it returns an error.
-func TLSClientConfigFromSecret(secret corev1.Secret) (getter.Option, func(), error) {
+func TLSClientConfigFromSecret(secret corev1.Secret, dir string) (getter.Option, error) {
certBytes, keyBytes, caBytes := secret.Data["certFile"], secret.Data["keyFile"], secret.Data["caFile"]
switch {
case len(certBytes)+len(keyBytes)+len(caBytes) == 0:
- return nil, func() {}, nil
+ return nil, nil
case (len(certBytes) > 0 && len(keyBytes) == 0) || (len(keyBytes) > 0 && len(certBytes) == 0):
- return nil, nil, fmt.Errorf("invalid '%s' secret data: fields 'certFile' and 'keyFile' require each other's presence",
+ return nil, fmt.Errorf("invalid '%s' secret data: fields 'certFile' and 'keyFile' require each other's presence",
secret.Name)
}
- // create tmp dir for TLS files
- tmp, err := ioutil.TempDir("", "helm-tls-"+secret.Name)
- if err != nil {
- return nil, nil, err
- }
- cleanup := func() { os.RemoveAll(tmp) }
-
var certFile, keyFile, caFile string
-
if len(certBytes) > 0 && len(keyBytes) > 0 {
- certFile = filepath.Join(tmp, "cert.crt")
- if err := ioutil.WriteFile(certFile, certBytes, 0644); err != nil {
- cleanup()
- return nil, nil, err
+ f, err := os.CreateTemp(dir, "cert-")
+ if err != nil {
+
+ }
+ if _, err = io.Copy(f, bytes.NewReader(certBytes)); err != nil {
+ f.Close()
+ return nil, err
}
- keyFile = filepath.Join(tmp, "key.crt")
- if err := ioutil.WriteFile(keyFile, keyBytes, 0644); err != nil {
- cleanup()
- return nil, nil, err
+ f.Close()
+ certFile = f.Name()
+
+ f, err = os.CreateTemp(dir, "key-")
+ if err != nil {
+ f.Close()
+ return nil, err
}
+ if _, err = io.Copy(f, bytes.NewReader(keyBytes)); err != nil {
+ f.Close()
+ return nil, err
+ }
+ f.Close()
+ keyFile = f.Name()
}
if len(caBytes) > 0 {
- caFile = filepath.Join(tmp, "ca.pem")
- if err := ioutil.WriteFile(caFile, caBytes, 0644); err != nil {
- cleanup()
- return nil, nil, err
+ f, err := os.CreateTemp(dir, "ca-")
+ if err != nil {
+ f.Close()
+ return nil, err
+ }
+ if _, err = io.Copy(f, bytes.NewReader(caBytes)); err != nil {
+ f.Close()
}
+ f.Close()
+ caFile = f.Name()
}
- return getter.WithTLSClientConfig(certFile, keyFile, caFile), cleanup, nil
+ return getter.WithTLSClientConfig(certFile, keyFile, caFile), nil
}
diff --git a/internal/helm/getter_test.go b/internal/helm/getter_test.go
index bd4e1058c..baf5c423a 100644
--- a/internal/helm/getter_test.go
+++ b/internal/helm/getter_test.go
@@ -17,6 +17,7 @@ limitations under the License.
package helm
import (
+ "os"
"testing"
corev1 "k8s.io/api/core/v1"
@@ -56,10 +57,12 @@ func TestClientOptionsFromSecret(t *testing.T) {
secret.Data[k] = v
}
}
- got, cleanup, err := ClientOptionsFromSecret(secret)
- if cleanup != nil {
- defer cleanup()
+ tmpDir, err := os.MkdirTemp("", "")
+ if err != nil {
+ t.Fatalf("Failed to create temporary directory: %v", err)
}
+ defer os.RemoveAll(tmpDir)
+ got, err := ClientOptionsFromSecret(secret, tmpDir)
if err != nil {
t.Errorf("ClientOptionsFromSecret() error = %v", err)
return
@@ -123,10 +126,12 @@ func TestTLSClientConfigFromSecret(t *testing.T) {
if tt.modify != nil {
tt.modify(secret)
}
- got, cleanup, err := TLSClientConfigFromSecret(*secret)
- if cleanup != nil {
- defer cleanup()
+ tmpDir, err := os.MkdirTemp("", "")
+ if err != nil {
+ t.Fatalf("Failed to create temporary directory: %v", err)
}
+ defer os.RemoveAll(tmpDir)
+ got, err := TLSClientConfigFromSecret(*secret, tmpDir)
if (err != nil) != tt.wantErr {
t.Errorf("TLSClientConfigFromSecret() error = %v, wantErr %v", err, tt.wantErr)
return
diff --git a/internal/helm/repository.go b/internal/helm/repository.go
index ee9453791..b964e5ac0 100644
--- a/internal/helm/repository.go
+++ b/internal/helm/repository.go
@@ -18,9 +18,11 @@ package helm
import (
"bytes"
+ "crypto/sha1"
"fmt"
- "io/ioutil"
+ "io"
"net/url"
+ "os"
"path"
"sort"
"strings"
@@ -36,10 +38,11 @@ import (
// ChartRepository represents a Helm chart repository, and the configuration
// required to download the chart index, and charts from the repository.
type ChartRepository struct {
- URL string
- Index *repo.IndexFile
- Client getter.Getter
- Options []getter.Option
+ URL string
+ Index *repo.IndexFile
+ Checksum string
+ Client getter.Getter
+ Options []getter.Option
}
// NewChartRepository constructs and returns a new ChartRepository with
@@ -175,6 +178,15 @@ func (r *ChartRepository) DownloadChart(chart *repo.ChartVersion) (*bytes.Buffer
return r.Client.Get(u.String(), r.Options...)
}
+// LoadIndexFile takes a file at the given path and loads it using LoadIndex.
+func (r *ChartRepository) LoadIndexFile(path string) error {
+ b, err := os.ReadFile(path)
+ if err != nil {
+ return err
+ }
+ return r.LoadIndex(b)
+}
+
// LoadIndex loads the given bytes into the Index while performing
// minimal validity checks. It fails if the API version is not set
// (repo.ErrNoAPIVersion), or if the unmarshal fails.
@@ -191,6 +203,7 @@ func (r *ChartRepository) LoadIndex(b []byte) error {
}
i.SortEntries()
r.Index = i
+ r.Checksum = fmt.Sprintf("%x", sha1.Sum(b))
return nil
}
@@ -209,7 +222,7 @@ func (r *ChartRepository) DownloadIndex() error {
if err != nil {
return err
}
- b, err := ioutil.ReadAll(res)
+ b, err := io.ReadAll(res)
if err != nil {
return err
}
diff --git a/internal/helm/repository_test.go b/internal/helm/repository_test.go
index 468866674..c51a19d40 100644
--- a/internal/helm/repository_test.go
+++ b/internal/helm/repository_test.go
@@ -18,8 +18,8 @@ package helm
import (
"bytes"
- "io/ioutil"
"net/url"
+ "os"
"reflect"
"strings"
"testing"
@@ -231,7 +231,7 @@ func TestChartRepository_DownloadChart(t *testing.T) {
}
func TestChartRepository_DownloadIndex(t *testing.T) {
- b, err := ioutil.ReadFile(chartmuseumtestfile)
+ b, err := os.ReadFile(chartmuseumtestfile)
if err != nil {
t.Fatal(err)
}
@@ -270,7 +270,7 @@ func TestChartRepository_LoadIndex(t *testing.T) {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
- b, err := ioutil.ReadFile(tt.filename)
+ b, err := os.ReadFile(tt.filename)
if err != nil {
t.Fatal(err)
}
@@ -292,7 +292,7 @@ func TestChartRepository_LoadIndex_Duplicates(t *testing.T) {
}
func TestChartRepository_LoadIndex_Unordered(t *testing.T) {
- b, err := ioutil.ReadFile(unorderedtestfile)
+ b, err := os.ReadFile(unorderedtestfile)
if err != nil {
t.Fatal(err)
}
diff --git a/main.go b/main.go
index 55a2d2f97..7446896d2 100644
--- a/main.go
+++ b/main.go
@@ -25,6 +25,13 @@ import (
"strings"
"time"
+ "github.com/fluxcd/pkg/runtime/client"
+ helper "github.com/fluxcd/pkg/runtime/controller"
+ "github.com/fluxcd/pkg/runtime/events"
+ "github.com/fluxcd/pkg/runtime/leaderelection"
+ "github.com/fluxcd/pkg/runtime/logger"
+ "github.com/fluxcd/pkg/runtime/pprof"
+ "github.com/fluxcd/pkg/runtime/probes"
"github.com/go-logr/logr"
flag "github.com/spf13/pflag"
"helm.sh/helm/v3/pkg/getter"
@@ -33,15 +40,6 @@ import (
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
ctrl "sigs.k8s.io/controller-runtime"
- crtlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
-
- "github.com/fluxcd/pkg/runtime/client"
- "github.com/fluxcd/pkg/runtime/events"
- "github.com/fluxcd/pkg/runtime/leaderelection"
- "github.com/fluxcd/pkg/runtime/logger"
- "github.com/fluxcd/pkg/runtime/metrics"
- "github.com/fluxcd/pkg/runtime/pprof"
- "github.com/fluxcd/pkg/runtime/probes"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
"github.com/fluxcd/source-controller/controllers"
@@ -99,26 +97,25 @@ func main() {
flag.BoolVar(&watchAllNamespaces, "watch-all-namespaces", true,
"Watch for custom resources in all namespaces, if set to false it will only watch the runtime namespace.")
flag.DurationVar(&requeueDependency, "requeue-dependency", 30*time.Second, "The interval at which failing dependencies are reevaluated.")
+
clientOptions.BindFlags(flag.CommandLine)
logOptions.BindFlags(flag.CommandLine)
leaderElectionOptions.BindFlags(flag.CommandLine)
+
flag.Parse()
ctrl.SetLogger(logger.NewLogger(logOptions))
var eventRecorder *events.Recorder
if eventsAddr != "" {
- if er, err := events.NewRecorder(eventsAddr, controllerName); err != nil {
+ er, err := events.NewRecorder(eventsAddr, controllerName)
+ if err != nil {
setupLog.Error(err, "unable to create event recorder")
os.Exit(1)
- } else {
- eventRecorder = er
}
+ eventRecorder = er
}
- metricsRecorder := metrics.NewRecorder()
- crtlmetrics.Registry.MustRegister(metricsRecorder.Collectors()...)
-
watchNamespace := ""
if !watchAllNamespaces {
watchNamespace = os.Getenv("RUNTIME_NAMESPACE")
@@ -147,18 +144,19 @@ func main() {
probes.SetupChecks(mgr, setupLog)
pprof.SetupHandlers(mgr, setupLog)
+ events := helper.MakeEvents(mgr, controllerName, eventRecorder)
+ metrics := helper.MustMakeMetrics(mgr)
+
if storageAdvAddr == "" {
storageAdvAddr = determineAdvStorageAddr(storageAddr, setupLog)
}
storage := mustInitStorage(storagePath, storageAdvAddr, setupLog)
if err = (&controllers.GitRepositoryReconciler{
- Client: mgr.GetClient(),
- Scheme: mgr.GetScheme(),
- Storage: storage,
- EventRecorder: mgr.GetEventRecorderFor(controllerName),
- ExternalEventRecorder: eventRecorder,
- MetricsRecorder: metricsRecorder,
+ Client: mgr.GetClient(),
+ Events: events,
+ Metrics: metrics,
+ Storage: storage,
}).SetupWithManagerAndOptions(mgr, controllers.GitRepositoryReconcilerOptions{
MaxConcurrentReconciles: concurrent,
DependencyRequeueInterval: requeueDependency,
@@ -167,13 +165,11 @@ func main() {
os.Exit(1)
}
if err = (&controllers.HelmRepositoryReconciler{
- Client: mgr.GetClient(),
- Scheme: mgr.GetScheme(),
- Storage: storage,
- Getters: getters,
- EventRecorder: mgr.GetEventRecorderFor(controllerName),
- ExternalEventRecorder: eventRecorder,
- MetricsRecorder: metricsRecorder,
+ Client: mgr.GetClient(),
+ Events: events,
+ Metrics: metrics,
+ Getters: getters,
+ Storage: storage,
}).SetupWithManagerAndOptions(mgr, controllers.HelmRepositoryReconcilerOptions{
MaxConcurrentReconciles: concurrent,
}); err != nil {
@@ -181,13 +177,11 @@ func main() {
os.Exit(1)
}
if err = (&controllers.HelmChartReconciler{
- Client: mgr.GetClient(),
- Scheme: mgr.GetScheme(),
- Storage: storage,
- Getters: getters,
- EventRecorder: mgr.GetEventRecorderFor(controllerName),
- ExternalEventRecorder: eventRecorder,
- MetricsRecorder: metricsRecorder,
+ Client: mgr.GetClient(),
+ Events: events,
+ Metrics: metrics,
+ Storage: storage,
+ Getters: getters,
}).SetupWithManagerAndOptions(mgr, controllers.HelmChartReconcilerOptions{
MaxConcurrentReconciles: concurrent,
}); err != nil {
@@ -195,12 +189,10 @@ func main() {
os.Exit(1)
}
if err = (&controllers.BucketReconciler{
- Client: mgr.GetClient(),
- Scheme: mgr.GetScheme(),
- Storage: storage,
- EventRecorder: mgr.GetEventRecorderFor(controllerName),
- ExternalEventRecorder: eventRecorder,
- MetricsRecorder: metricsRecorder,
+ Client: mgr.GetClient(),
+ Events: events,
+ Metrics: metrics,
+ Storage: storage,
}).SetupWithManagerAndOptions(mgr, controllers.BucketReconcilerOptions{
MaxConcurrentReconciles: concurrent,
}); err != nil {
diff --git a/pkg/git/fake/commit.go b/pkg/git/fake/commit.go
new file mode 100644
index 000000000..60a9b5c05
--- /dev/null
+++ b/pkg/git/fake/commit.go
@@ -0,0 +1,46 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fake
+
+import (
+ "errors"
+
+ corev1 "k8s.io/api/core/v1"
+)
+
+type Commit struct {
+ valid bool
+ hash string
+}
+
+func NewCommit(valid bool, hash string) Commit {
+ return Commit{
+ valid: valid,
+ hash: hash,
+ }
+}
+
+func (c Commit) Verify(secret corev1.Secret) error {
+ if !c.valid {
+ return errors.New("invalid signature")
+ }
+ return nil
+}
+
+func (c Commit) Hash() string {
+ return c.hash
+}
diff --git a/pkg/git/gogit/checkout.go b/pkg/git/gogit/checkout.go
index dfcde8498..fdf910271 100644
--- a/pkg/git/gogit/checkout.go
+++ b/pkg/git/gogit/checkout.go
@@ -193,7 +193,7 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, auth *g
tags := make(map[string]string)
tagTimestamps := make(map[string]time.Time)
- _ = repoTags.ForEach(func(t *plumbing.Reference) error {
+ if err = repoTags.ForEach(func(t *plumbing.Reference) error {
revision := plumbing.Revision(t.Name().String())
hash, err := repo.ResolveRevision(revision)
if err != nil {
@@ -207,7 +207,9 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, auth *g
tags[t.Name().Short()] = t.Strings()[1]
return nil
- })
+ }); err != nil {
+ return nil, "", err
+ }
var matchedVersions semver.Collection
for tag, _ := range tags {
diff --git a/pkg/git/gogit/checkout_test.go b/pkg/git/gogit/checkout_test.go
index aa1c3ca71..eaa12c556 100644
--- a/pkg/git/gogit/checkout_test.go
+++ b/pkg/git/gogit/checkout_test.go
@@ -18,7 +18,6 @@ package gogit
import (
"context"
- "io/ioutil"
"os"
"testing"
@@ -30,7 +29,7 @@ func TestCheckoutTagSemVer_Checkout(t *testing.T) {
tag := CheckoutTag{
tag: "v1.7.0",
}
- tmpDir, _ := ioutil.TempDir("", "test")
+ tmpDir, _ := os.MkdirTemp("", "test")
defer os.RemoveAll(tmpDir)
cTag, _, err := tag.Checkout(context.TODO(), tmpDir, "https://github.com/projectcontour/contour", auth)
@@ -41,7 +40,7 @@ func TestCheckoutTagSemVer_Checkout(t *testing.T) {
semVer := CheckoutSemVer{
semVer: ">=1.0.0 <=1.7.0",
}
- tmpDir2, _ := ioutil.TempDir("", "test")
+ tmpDir2, _ := os.MkdirTemp("", "test")
defer os.RemoveAll(tmpDir2)
cSemVer, _, err := semVer.Checkout(context.TODO(), tmpDir2, "https://github.com/projectcontour/contour", auth)
diff --git a/pkg/git/libgit2/checkout.go b/pkg/git/libgit2/checkout.go
index 5aee26a1c..01363f8fa 100644
--- a/pkg/git/libgit2/checkout.go
+++ b/pkg/git/libgit2/checkout.go
@@ -19,8 +19,11 @@ package libgit2
import (
"context"
"fmt"
+ "sort"
+ "time"
- "github.com/blang/semver/v4"
+ "github.com/Masterminds/semver/v3"
+ "github.com/fluxcd/pkg/version"
git2go "github.com/libgit2/git2go/v31"
"github.com/fluxcd/pkg/gitutil"
@@ -168,7 +171,7 @@ type CheckoutSemVer struct {
}
func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) {
- rng, err := semver.ParseRange(c.semVer)
+ verConstraint, err := semver.NewConstraint(c.semVer)
if err != nil {
return nil, "", fmt.Errorf("semver parse range error: %w", err)
}
@@ -186,28 +189,61 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, auth *g
return nil, "", fmt.Errorf("unable to clone '%s', error: %w", url, err)
}
- repoTags, err := repo.Tags.List()
- if err != nil {
- return nil, "", fmt.Errorf("git list tags error: %w", err)
- }
+ tags := make(map[string]string)
+ tagTimestamps := make(map[string]time.Time)
+ if err := repo.Tags.Foreach(func(name string, id *git2go.Oid) error {
+ tag, err := repo.LookupTag(id)
+ if err != nil {
+ return nil
+ }
- svTags := make(map[string]string)
- var svers []semver.Version
- for _, tag := range repoTags {
- v, _ := semver.ParseTolerant(tag)
- if rng(v) {
- svers = append(svers, v)
- svTags[v.String()] = tag
+ commit, err := tag.Peel(git2go.ObjectCommit)
+ if err != nil {
+ return fmt.Errorf("can't get commit for tag %s: %w", name, err)
}
+ c, err := commit.AsCommit()
+ if err != nil {
+ return err
+ }
+ tagTimestamps[tag.Name()] = c.Committer().When
+ tags[tag.Name()] = name
+ return nil
+ }); err != nil {
+ return nil, "", err
}
- if len(svers) == 0 {
+ var matchedVersions semver.Collection
+ for tag, _ := range tags {
+ v, err := version.ParseVersion(tag)
+ if err != nil {
+ continue
+ }
+ if !verConstraint.Check(v) {
+ continue
+ }
+ matchedVersions = append(matchedVersions, v)
+ }
+ if len(matchedVersions) == 0 {
return nil, "", fmt.Errorf("no match found for semver: %s", c.semVer)
}
- semver.Sort(svers)
- v := svers[len(svers)-1]
- t := svTags[v.String()]
+ // Sort versions
+ sort.SliceStable(matchedVersions, func(i, j int) bool {
+ left := matchedVersions[i]
+ right := matchedVersions[j]
+
+ if !left.Equal(right) {
+ return left.LessThan(right)
+ }
+
+ // Having tag target timestamps at our disposal, we further try to sort
+ // versions into a chronological order. This is especially important for
+ // versions that differ only by build metadata, because it is not considered
+ // a part of the comparable version in Semver
+ return tagTimestamps[left.String()].Before(tagTimestamps[right.String()])
+ })
+ v := matchedVersions[len(matchedVersions)-1]
+ t := v.Original()
ref, err := repo.References.Dwim(t)
if err != nil {
diff --git a/pkg/git/libgit2/checkout_test.go b/pkg/git/libgit2/checkout_test.go
index 8c9d94839..6de5484d8 100644
--- a/pkg/git/libgit2/checkout_test.go
+++ b/pkg/git/libgit2/checkout_test.go
@@ -21,7 +21,6 @@ import (
"crypto/sha256"
"encoding/hex"
"io"
- "io/ioutil"
"os"
"path"
"testing"
@@ -40,7 +39,7 @@ func TestCheckoutTagSemVer_Checkout(t *testing.T) {
tag := CheckoutTag{
tag: "v1.7.0",
}
- tmpDir, _ := ioutil.TempDir("", "test")
+ tmpDir, _ := os.MkdirTemp("", "test")
defer os.RemoveAll(tmpDir)
cTag, _, err := tag.Checkout(context.TODO(), tmpDir, "https://github.com/projectcontour/contour", auth)
@@ -66,7 +65,7 @@ func TestCheckoutTagSemVer_Checkout(t *testing.T) {
semVer := CheckoutSemVer{
semVer: ">=1.0.0 <=1.7.0",
}
- tmpDir2, _ := ioutil.TempDir("", "test")
+ tmpDir2, _ := os.MkdirTemp("", "test")
defer os.RemoveAll(tmpDir2)
cSemVer, _, err := semVer.Checkout(context.TODO(), tmpDir2, "https://github.com/projectcontour/contour", auth)
diff --git a/pkg/sourceignore/sourceignore.go b/pkg/sourceignore/sourceignore.go
index b4e0bf50f..f4d98e471 100644
--- a/pkg/sourceignore/sourceignore.go
+++ b/pkg/sourceignore/sourceignore.go
@@ -19,7 +19,6 @@ package sourceignore
import (
"bufio"
"io"
- "io/ioutil"
"os"
"path/filepath"
"strings"
@@ -108,7 +107,7 @@ func LoadIgnorePatterns(dir string, domain []string) ([]gitignore.Pattern, error
if err != nil {
return nil, err
}
- fis, err := ioutil.ReadDir(dir)
+ fis, err := os.ReadDir(dir)
if err != nil {
return nil, err
}
diff --git a/pkg/sourceignore/sourceignore_test.go b/pkg/sourceignore/sourceignore_test.go
index 98a88d7e0..786868ba1 100644
--- a/pkg/sourceignore/sourceignore_test.go
+++ b/pkg/sourceignore/sourceignore_test.go
@@ -17,7 +17,6 @@ limitations under the License.
package sourceignore
import (
- "io/ioutil"
"os"
"path/filepath"
"reflect"
@@ -74,7 +73,7 @@ func TestReadPatterns(t *testing.T) {
}
func TestReadIgnoreFile(t *testing.T) {
- f, err := ioutil.TempFile("", IgnoreFile)
+ f, err := os.CreateTemp("", IgnoreFile)
if err != nil {
t.Fatal(err)
}
@@ -198,7 +197,7 @@ func TestDefaultPatterns(t *testing.T) {
}
func TestLoadExcludePatterns(t *testing.T) {
- tmpDir, err := ioutil.TempDir("", "sourceignore-load-")
+ tmpDir, err := os.MkdirTemp("", "sourceignore-load-")
if err != nil {
t.Fatal(err)
}