From d3c27c37f825c6cca82f86f5b236e7cbcc90669b Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 10 Sep 2021 12:39:40 +0200 Subject: [PATCH 01/10] Update github.com/libgit2/git2go to v31.6.1 This commit updates `github.com/libgit2/git2go` to `v31.6.1` (with `libgit2` `1.1.1`), and changes the container image build process so that it makes use of `ghcr.io/hiddeco/golang-with-libgit2`. This image provides a pre-build dynamic `libgit2` dependency linked against OpenSSL and LibSSH2 (without gcrypt), and a set of cross-compile build tools (see [rationale](https://github.com/hiddeco/golang-with-libgit2#rationale) and [usage](https://github.co/hiddeco/golang-with-libgit2#usage) for more detailed information). The linked set of dependency should solve most known issues around unsupport private key types, but does not resolve the issues with ECDSA* and ED25519 hostkeys yet. Solving this requires a newer version of `libgit2` (`>=1.2.0`), which currently does not seem to work properly with `git2go/v32`. Some small changes have been made to the `libgit2` package to address (future) deprecations. Signed-off-by: Hidde Beydals --- .dockerignore | 1 + .github/actions/run-tests/Dockerfile | 15 +-- .gitignore | 3 + Dockerfile | 78 ++++++-------- Makefile | 153 ++++++++++++++++----------- go.mod | 2 +- go.sum | 4 +- internal/fs/rename.go | 1 + internal/fs/rename_windows.go | 1 + pkg/git/libgit2/checkout.go | 2 +- pkg/git/libgit2/checkout_test.go | 7 +- pkg/git/libgit2/transport.go | 22 ++-- 12 files changed, 151 insertions(+), 138 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..1e2f1e162 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +hack/libgit2/ diff --git a/.github/actions/run-tests/Dockerfile b/.github/actions/run-tests/Dockerfile index 1e8919567..93c9f549e 100644 --- a/.github/actions/run-tests/Dockerfile +++ b/.github/actions/run-tests/Dockerfile @@ -1,17 +1,4 @@ -FROM golang:1.16-buster as builder - -# Up-to-date libgit2 dependencies are only available in -# unstable, as libssh2 in testing/bullseye has been linked -# against gcrypt which causes issues with PKCS* formats. -# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271 -RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \ - && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list -RUN set -eux; \ - apt-get update \ - && apt-get install -y libgit2-dev/unstable \ - && apt-get clean \ - && apt-get autoremove --purge -y \ - && rm -rf /var/lib/apt/lists/* +FROM ghcr.io/hiddeco/golang-with-libgit2:dev as builder # Use the GitHub Actions uid:gid combination for proper fs permissions RUN groupadd -g 116 test && \ diff --git a/.gitignore b/.gitignore index 8f19ec807..b93753252 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ # vendor/ bin/ config/release/ + +# Exclude all libgit2 related files +hack/libgit2/ diff --git a/Dockerfile b/Dockerfile index 059a25bb3..083fb1e63 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,69 +1,57 @@ -FROM golang:1.16-buster as builder - -# Up-to-date libgit2 dependencies are only available in -# unstable, as libssh2 in testing/bullseye has been linked -# against gcrypt which causes issues with PKCS* formats. -# Explicitly listing all build dependencies is required because -# they can only be automagically found for AMD64 builds. -# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271 -RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \ - && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list -RUN set -eux; \ - apt-get update \ - && apt-get install -y \ - libgit2-dev/unstable \ - zlib1g-dev/unstable \ - libssh2-1-dev/unstable \ - libpcre3-dev/unstable \ - && apt-get clean \ - && apt-get autoremove --purge -y \ - && rm -rf /var/lib/apt/lists/* +ARG BASE_IMG=ghcr.io/hiddeco/golang-with-libgit2 +ARG BASE_TAG=dev +FROM ${BASE_IMG}:${BASE_TAG} AS build +# Configure workspace WORKDIR /workspace -# copy api submodule +# Copy api submodule COPY api/ api/ -# copy modules manifests +# Copy modules manifests COPY go.mod go.mod COPY go.sum go.sum -# cache modules +# Cache modules RUN go mod download -# copy source code +# Copy source code COPY main.go main.go COPY controllers/ controllers/ COPY pkg/ pkg/ COPY internal/ internal/ -# build without specifing the arch -RUN CGO_ENABLED=1 go build -o source-controller main.go +# Build the binary +ENV CGO_ENABLED=1 +ARG TARGETPLATFORM +RUN xx-go build -o source-controller -trimpath \ + main.go -FROM debian:buster-slim as controller +FROM debian:bullseye-slim as controller -# link repo to the GitHub Container Registry image +# Link repo to the GitHub Container Registry image LABEL org.opencontainers.image.source="https://github.com/fluxcd/source-controller" -# Up-to-date libgit2 dependencies are only available in -# unstable, as libssh2 in testing/bullseye has been linked -# against gcrypt which causes issues with PKCS* formats. -# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271 -RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \ - && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list -RUN set -eux; \ - apt-get update \ - && apt-get install -y \ - ca-certificates \ - libgit2-1.1 \ - && apt-get clean \ - && apt-get autoremove --purge -y \ - && rm -rf /var/lib/apt/lists/* - -COPY --from=builder /workspace/source-controller /usr/local/bin/ - +# Configure user RUN groupadd controller && \ useradd --gid controller --shell /bin/sh --create-home controller +# Copy libgit2 +COPY --from=build /libgit2/lib/* /usr/local/lib/ +RUN ldconfig + +# Upgrade packages and install runtime dependencies +RUN echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list \ + && echo "deb-src http://deb.debian.org/debian sid main" >> /etc/apt/sources.list \ + && apt update \ + && apt install --no-install-recommends -y zlib1g/sid libssl1.1/sid libssh2-1/sid \ + && apt install --no-install-recommends -y ca-certificates \ + && apt clean \ + && apt autoremove --purge -y \ + && rm -rf /var/lib/apt/lists/* + +# Copy over binary from build +COPY --from=build /workspace/source-controller /usr/local/bin/ + USER controller ENTRYPOINT [ "source-controller" ] diff --git a/Makefile b/Makefile index 8f1ecc800..167c4804c 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,32 @@ # Image URL to use all building/pushing image targets IMG ?= fluxcd/source-controller:latest + +# Base image used to build the Go binary +BASE_IMG ?= ghcr.io/hiddeco/golang-with-libgit2 +BASE_TAG ?= dev + # Produce CRDs that work back to Kubernetes 1.16 CRD_OPTIONS ?= crd:crdVersions=v1 -ENVTEST_BIN_VERSION?=1.19.2 -KUBEBUILDER_ASSETS?=$(shell $(SETUP_ENVTEST) use -i $(ENVTEST_BIN_VERSION) -p path) +# Repository root based on Git metadata +REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel) + +# Dependency versions +LIBGIT2_VERSION ?= 1.1.1 +ENVTEST_BIN_VERSION ?= 1.19.2 +KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use -i $(ENVTEST_BIN_VERSION) -p path) + +# libgit2 related magical paths +# These are used to determine if the target libgit2 version is already available on +# the system, or where they should be installed to +SYSTEM_LIBGIT2_VERSION := $(shell pkg-config --modversion libgit2 2>/dev/null) +LIBGIT2_PATH := $(REPOSITORY_ROOT)/hack/libgit2 +LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib +LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.so.$(LIBGIT2_VERSION) + +# API (doc) generation utilities +CONTROLLER_GEN_VERSION ?= v0.5.0 +GEN_API_REF_DOCS_VERSION ?= 0.3.0 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -13,121 +35,130 @@ else GOBIN=$(shell go env GOBIN) endif -all: manager +all: build -# Run tests -test: generate fmt vet manifests api-docs setup-envtest - KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) go test ./... -coverprofile cover.out - cd api; go test ./... -coverprofile cover.out - -# Build manager binary -manager: generate fmt vet +build: $(LIBGIT2) ## Build manager binary + PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig/ \ go build -o bin/manager main.go -# Run against the configured Kubernetes cluster in ~/.kube/config -run: generate fmt vet manifests +test: $(LIBGIT2) test-api ## Run tests + LD_LIBRARY_PATH=$(LIBGIT2_LIB_PATH) \ + PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig/ \ + go test ./... -coverprofile cover.out + +test-api: ## Run api tests + cd api; go test ./... -coverprofile cover.out + +run: $(LIBGIT2) generate fmt vet manifests ## Run against the configured Kubernetes cluster in ~/.kube/config + LD_LIBRARY_PATH=$(LIBGIT2_LIB_PATH) \ go run ./main.go -# Install CRDs into a cluster -install: manifests +install: manifests ## Install CRDs into a cluster kustomize build config/crd | kubectl apply -f - -# Uninstall CRDs from a cluster -uninstall: manifests +uninstall: manifests ## Uninstall CRDs from a cluster kustomize build config/crd | kubectl delete -f - -# Deploy controller in the configured Kubernetes cluster in ~/.kube/config -deploy: manifests +deploy: manifests ## Deploy controller in the configured Kubernetes cluster in ~/.kube/config cd config/manager && kustomize edit set image fluxcd/source-controller=${IMG} kustomize build config/default | kubectl apply -f - -# Deploy controller dev image in the configured Kubernetes cluster in ~/.kube/config -dev-deploy: +dev-deploy: ## Deploy controller dev image in the configured Kubernetes cluster in ~/.kube/config mkdir -p config/dev && cp config/default/* config/dev cd config/dev && kustomize edit set image fluxcd/source-controller=${IMG} kustomize build config/dev | kubectl apply -f - rm -rf config/dev -# Generate manifests e.g. CRD, RBAC etc. -manifests: controller-gen +manifests: controller-gen ## Generate manifests, e.g. CRD, RBAC, etc. $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role paths="./..." output:crd:artifacts:config="config/crd/bases" cd api; $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role paths="./..." output:crd:artifacts:config="../config/crd/bases" -# Generate API reference documentation -api-docs: gen-crd-api-reference-docs +api-docs: gen-crd-api-reference-docs ## Generate API reference documentation $(API_REF_GEN) -api-dir=./api/v1beta1 -config=./hack/api-docs/config.json -template-dir=./hack/api-docs/template -out-file=./docs/api/source.md -# Run go mod tidy -tidy: +tidy: ## Run go mod tidy go mod tidy cd api; go mod tidy -# Run go fmt against code -fmt: +fmt: ## Run go fmt against code go fmt ./... cd api; go fmt ./... -# Run go vet against code -vet: +vet: ## Run go vet against code + PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig \ go vet ./... cd api; go vet ./... -# Generate code -generate: controller-gen +generate: controller-gen ## Generate API code cd api; $(CONTROLLER_GEN) object:headerFile="../hack/boilerplate.go.txt" paths="./..." -# Build the docker image -docker-build: - docker build . -t ${IMG} +docker-build: ## Build the docker image + docker build \ + --build-arg BASE_IMG=$(BASE_IMG) \ + --build-arg BASE_TAG=$(BASE_TAG) \ + -t ${IMG} . -# Push the docker image -docker-push: +docker-push: ## Push docker image docker push ${IMG} -# Find or download controller-gen -controller-gen: +controller-gen: ## Find or download controller-gen ifeq (, $(shell which controller-gen)) @{ \ - set -e ;\ - CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ - cd $$CONTROLLER_GEN_TMP_DIR ;\ - go mod init tmp ;\ - go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.5.0 ;\ - rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ + set -e; \ + CONTROLLER_GEN_TMP_DIR=$$(mktemp -d); \ + cd $$CONTROLLER_GEN_TMP_DIR; \ + go mod init tmp; \ + go get sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_GEN_VERSION); \ + rm -rf $$CONTROLLER_GEN_TMP_DIR; \ } CONTROLLER_GEN=$(GOBIN)/controller-gen else CONTROLLER_GEN=$(shell which controller-gen) endif -# Find or download gen-crd-api-reference-docs -gen-crd-api-reference-docs: +gen-crd-api-reference-docs: ## Find or download gen-crd-api-reference-docs ifeq (, $(shell which gen-crd-api-reference-docs)) @{ \ - set -e ;\ - API_REF_GEN_TMP_DIR=$$(mktemp -d) ;\ - cd $$API_REF_GEN_TMP_DIR ;\ - go mod init tmp ;\ - go get github.com/ahmetb/gen-crd-api-reference-docs@v0.3.0 ;\ - rm -rf $$API_REF_GEN_TMP_DIR ;\ + set -e; \ + API_REF_GEN_TMP_DIR=$$(mktemp -d); \ + cd $$API_REF_GEN_TMP_DIR; \ + go mod init tmp; \ + go get github.com/ahmetb/gen-crd-api-reference-docs@$(GEN_API_REF_DOCS_VERSION); \ + rm -rf $$API_REF_GEN_TMP_DIR; \ } API_REF_GEN=$(GOBIN)/gen-crd-api-reference-docs else API_REF_GEN=$(shell which gen-crd-api-reference-docs) endif -# Find or download setup-envtest -setup-envtest: +setup-envtest: ## Find or download setup-envtest ifeq (, $(shell which setup-envtest)) @{ \ - set -e ;\ - SETUP_ENVTEST_TMP_DIR=$$(mktemp -d) ;\ - cd $$SETUP_ENVTEST_TMP_DIR ;\ - go mod init tmp ;\ - go get sigs.k8s.io/controller-runtime/tools/setup-envtest@latest ;\ - rm -rf $$SETUP_ENVTEST_TMP_DIR ;\ + set -e; \ + SETUP_ENVTEST_TMP_DIR=$$(mktemp -d); \ + cd $$SETUP_ENVTEST_TMP_DIR; \ + go mod init tmp; \ + go get sigs.k8s.io/controller-runtime/tools/setup-envtest@latest; \ + rm -rf $$SETUP_ENVTEST_TMP_DIR; \ } SETUP_ENVTEST=$(GOBIN)/setup-envtest else SETUP_ENVTEST=$(shell which setup-envtest) endif + +libgit2: $(LIBGIT2) ## Detect or download libgit2 library + +$(LIBGIT2): +ifeq ($(LIBGIT2_VERSION),$(SYSTEM_LIBGIT2_VERSION)) +else + @{ \ + set -e; \ + mkdir -p $(LIBGIT2_PATH); \ + docker cp $(shell docker create --rm $(BASE_IMG):$(BASE_TAG)):/libgit2/Makefile $(LIBGIT2_PATH); \ + INSTALL_PREFIX=$(LIBGIT2_PATH) make -C $(LIBGIT2_PATH); \ + } +endif + +.PHONY: help +help: ## Display this help menu + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/go.mod b/go.mod index 2b0337332..d68674193 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/go-git/go-billy/v5 v5.3.1 github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v0.4.0 - github.com/libgit2/git2go/v31 v31.4.14 + github.com/libgit2/git2go/v31 v31.6.1 github.com/minio/minio-go/v7 v7.0.10 github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.14.0 diff --git a/go.sum b/go.sum index 477e3ddbc..ab66ad2fa 100644 --- a/go.sum +++ b/go.sum @@ -545,8 +545,8 @@ github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6Fm github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/libgit2/git2go/v31 v31.4.14 h1:6GOd3965D9e/+gjxCwZF4eQ+vB9kKB4yKFqdQr6XZ2E= -github.com/libgit2/git2go/v31 v31.4.14/go.mod h1:c/rkJcBcUFx6wHaT++UwNpKvIsmPNqCeQ/vzO4DrEec= +github.com/libgit2/git2go/v31 v31.6.1 h1:FnKHHDDBgltSsu9RpKuL4rSR8dQ1JTf9dfvFhZ1y7Aw= +github.com/libgit2/git2go/v31 v31.6.1/go.mod h1:c/rkJcBcUFx6wHaT++UwNpKvIsmPNqCeQ/vzO4DrEec= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= diff --git a/internal/fs/rename.go b/internal/fs/rename.go index a1b4a411d..bad1f4778 100644 --- a/internal/fs/rename.go +++ b/internal/fs/rename.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !windows // +build !windows package fs diff --git a/internal/fs/rename_windows.go b/internal/fs/rename_windows.go index 3b5650573..fa9a0b4d9 100644 --- a/internal/fs/rename_windows.go +++ b/internal/fs/rename_windows.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build windows // +build windows package fs diff --git a/pkg/git/libgit2/checkout.go b/pkg/git/libgit2/checkout.go index 01363f8fa..74c976faf 100644 --- a/pkg/git/libgit2/checkout.go +++ b/pkg/git/libgit2/checkout.go @@ -156,7 +156,7 @@ func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, auth *g if err != nil { return nil, "", fmt.Errorf("git worktree error: %w", err) } - err = repo.CheckoutTree(tree, &git2go.CheckoutOpts{ + err = repo.CheckoutTree(tree, &git2go.CheckoutOptions{ Strategy: git2go.CheckoutForce, }) if err != nil { diff --git a/pkg/git/libgit2/checkout_test.go b/pkg/git/libgit2/checkout_test.go index 6de5484d8..4b06f5841 100644 --- a/pkg/git/libgit2/checkout_test.go +++ b/pkg/git/libgit2/checkout_test.go @@ -32,7 +32,7 @@ import ( func TestCheckoutTagSemVer_Checkout(t *testing.T) { certCallback := func(cert *git2go.Certificate, valid bool, hostname string) git2go.ErrorCode { - return 0 + return git2go.ErrorCodeOK } auth := &git.Auth{CertCallback: certCallback} @@ -57,9 +57,10 @@ func TestCheckoutTagSemVer_Checkout(t *testing.T) { if _, err := io.Copy(h, f); err != nil { t.Error(err) } + const expectedHash = "2bd1707542a11f987ee24698dcc095a9f57639f401133ef6a29da97bf8f3f302" fileHash := hex.EncodeToString(h.Sum(nil)) - if fileHash != "2bd1707542a11f987ee24698dcc095a9f57639f401133ef6a29da97bf8f3f302" { - t.Errorf("expected files not checked out. Expected hash %s, got %s", "2bd1707542a11f987ee24698dcc095a9f57639f401133ef6a29da97bf8f3f302", fileHash) + if fileHash != expectedHash { + t.Errorf("expected files not checked out. Expected hash %s, got %s", expectedHash, fileHash) } semVer := CheckoutSemVer{ diff --git a/pkg/git/libgit2/transport.go b/pkg/git/libgit2/transport.go index da3d04e92..6329a490d 100644 --- a/pkg/git/libgit2/transport.go +++ b/pkg/git/libgit2/transport.go @@ -66,8 +66,8 @@ func (s *BasicAuth) Method(secret corev1.Secret) (*git.Auth, error) { password = string(d) } if username != "" && password != "" { - credCallback = func(url string, usernameFromURL string, allowedTypes git2go.CredType) (*git2go.Cred, error) { - cred, err := git2go.NewCredUserpassPlaintext(username, password) + credCallback = func(url string, usernameFromURL string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) { + cred, err := git2go.NewCredentialUserpassPlaintext(username, password) if err != nil { return nil, err } @@ -81,7 +81,7 @@ func (s *BasicAuth) Method(secret corev1.Secret) (*git.Auth, error) { roots := x509.NewCertPool() ok := roots.AppendCertsFromPEM(caFile) if !ok { - return git2go.ErrCertificate + return git2go.ErrorCodeCertificate } opts := x509.VerifyOptions{ @@ -90,9 +90,9 @@ func (s *BasicAuth) Method(secret corev1.Secret) (*git.Auth, error) { } _, err := cert.X509.Verify(opts) if err != nil { - return git2go.ErrCertificate + return git2go.ErrorCodeCertificate } - return git2go.ErrOk + return git2go.ErrorCodeOK } } @@ -137,8 +137,8 @@ func (s *PublicKeyAuth) Method(secret corev1.Secret) (*git.Auth, error) { user = git.DefaultPublicKeyAuthUser } - credCallback := func(url string, usernameFromURL string, allowedTypes git2go.CredType) (*git2go.Cred, error) { - cred, err := git2go.NewCredSshKeyFromMemory(user, "", string(identity), string(password)) + credCallback := func(url string, usernameFromURL string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) { + cred, err := git2go.NewCredentialSSHKeyFromMemory(user, "", string(identity), string(password)) if err != nil { return nil, err } @@ -157,20 +157,20 @@ func (s *PublicKeyAuth) Method(secret corev1.Secret) (*git.Auth, error) { // Check if the configured host matches the hostname given to // the callback. if host != hostname { - return git2go.ErrUser + return git2go.ErrorCodeUser } // We are now certain that the configured host and the hostname // given to the callback match. Use the configured host (that - // includes the port), and normalize it so we can check if there + // includes the port), and normalize it, so we can check if there // is an entry for the hostname _and_ port. host = knownhosts.Normalize(s.host) for _, k := range kk { if k.matches(host, cert.Hostkey) { - return git2go.ErrOk + return git2go.ErrorCodeOK } } - return git2go.ErrCertificate + return git2go.ErrorCodeCertificate } return &git.Auth{CredCallback: credCallback, CertCallback: certCallback}, nil From 7a212b6570c5aa5eea43e8bc6f60c55b9c53caa4 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Tue, 28 Sep 2021 00:12:39 +0200 Subject: [PATCH 02/10] Add `docker-buildx` target to `Makefile` To allow building a multi-platform container image using `buildx`. Various configuration flags allow for fine(r)-grain control over the build process: - `BASE_IMG`: FQDN of the base image that should be used, without a tag. - `BASE_TAG: tag of the base image that should be used. Allows checksum sum to be included. - `BUILDX_PLATFORMS`: platforms to target for the final container image. - `BUILDX_ARGS`: additional `docker buildx build` arguments, e.g. `--push` to push the result to a (local) image registry. Signed-off-by: Hidde Beydals --- .github/actions/run-tests/Dockerfile | 2 +- .github/workflows/e2e.yaml | 4 ++-- Dockerfile | 2 +- Makefile | 31 +++++++++++++++++++++------- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/.github/actions/run-tests/Dockerfile b/.github/actions/run-tests/Dockerfile index 93c9f549e..f07783aca 100644 --- a/.github/actions/run-tests/Dockerfile +++ b/.github/actions/run-tests/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/hiddeco/golang-with-libgit2:dev as builder +FROM ghcr.io/hiddeco/golang-with-libgit2:dev as build # Use the GitHub Actions uid:gid combination for proper fs permissions RUN groupadd -g 116 test && \ diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 085724d56..b75e70599 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -44,11 +44,11 @@ jobs: exit 1 fi - name: Build container image - run: make docker-build IMG=test/source-controller:latest + run: make docker-build IMG=test/source-controller TAG=latest - name: Load test image run: kind load docker-image test/source-controller:latest - name: Deploy controller - run: make dev-deploy IMG=test/source-controller:latest + run: make dev-deploy IMG=test/source-controller TAG=latest - name: Run smoke tests run: | kubectl -n source-system apply -f ./config/samples diff --git a/Dockerfile b/Dockerfile index 083fb1e63..08661246a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,7 @@ RUN groupadd controller && \ useradd --gid controller --shell /bin/sh --create-home controller # Copy libgit2 -COPY --from=build /libgit2/lib/* /usr/local/lib/ +COPY --from=build /libgit2/lib/ /usr/local/lib/ RUN ldconfig # Upgrade packages and install runtime dependencies diff --git a/Makefile b/Makefile index 167c4804c..a1a6ee9c7 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,17 @@ # Image URL to use all building/pushing image targets -IMG ?= fluxcd/source-controller:latest +IMG ?= fluxcd/source-controller +TAG ?= latest # Base image used to build the Go binary BASE_IMG ?= ghcr.io/hiddeco/golang-with-libgit2 BASE_TAG ?= dev +# Allows for defining additional Docker buildx arguments, +# e.g. '--push'. +BUILDX_ARGS ?= +# Architectures to build images for +BUILDX_PLATFORMS ?= linux/amd64,linux/arm64,linux/arm/v7 + # Produce CRDs that work back to Kubernetes 1.16 CRD_OPTIONS ?= crd:crdVersions=v1 @@ -60,12 +67,12 @@ uninstall: manifests ## Uninstall CRDs from a cluster kustomize build config/crd | kubectl delete -f - deploy: manifests ## Deploy controller in the configured Kubernetes cluster in ~/.kube/config - cd config/manager && kustomize edit set image fluxcd/source-controller=${IMG} + cd config/manager && kustomize edit set image fluxcd/source-controller=$(IMG):$(TAG) kustomize build config/default | kubectl apply -f - dev-deploy: ## Deploy controller dev image in the configured Kubernetes cluster in ~/.kube/config mkdir -p config/dev && cp config/default/* config/dev - cd config/dev && kustomize edit set image fluxcd/source-controller=${IMG} + cd config/dev && kustomize edit set image fluxcd/source-controller=$(IMG):$(TAG) kustomize build config/dev | kubectl apply -f - rm -rf config/dev @@ -84,7 +91,7 @@ fmt: ## Run go fmt against code go fmt ./... cd api; go fmt ./... -vet: ## Run go vet against code +vet: $(LIBGIT2) ## Run go vet against code PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig \ go vet ./... cd api; go vet ./... @@ -92,14 +99,22 @@ vet: ## Run go vet against code generate: controller-gen ## Generate API code cd api; $(CONTROLLER_GEN) object:headerFile="../hack/boilerplate.go.txt" paths="./..." -docker-build: ## Build the docker image +docker-build: ## Build the Docker image docker build \ --build-arg BASE_IMG=$(BASE_IMG) \ --build-arg BASE_TAG=$(BASE_TAG) \ - -t ${IMG} . + -t $(IMG):$(TAG) . + +docker-buildx: ## Build the cross-platform Docker image + docker buildx build \ + --build-arg BASE_IMG=$(BASE_IMG) \ + --build-arg BASE_TAG=$(BASE_TAG) \ + --platform=$(BUILDX_PLATFORMS) \ + -t $(IMG):$(TAG) \ + $(BUILDX_ARGS) . -docker-push: ## Push docker image - docker push ${IMG} +docker-push: ## Push Docker image + docker push $(IMG):$(TAG) controller-gen: ## Find or download controller-gen ifeq (, $(shell which controller-gen)) From 0358ebc43d1e2f4c7709d5c57819fafa0b2ef5f3 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 30 Sep 2021 16:24:32 +0200 Subject: [PATCH 03/10] Change image to image under Flux organization Signed-off-by: Hidde Beydals --- .github/actions/run-tests/Dockerfile | 2 +- Dockerfile | 4 ++-- Makefile | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/run-tests/Dockerfile b/.github/actions/run-tests/Dockerfile index f07783aca..25e4f2324 100644 --- a/.github/actions/run-tests/Dockerfile +++ b/.github/actions/run-tests/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/hiddeco/golang-with-libgit2:dev as build +FROM ghcr.io/fluxcd/golang-with-libgit2:1.16.8-bullseye-libgit2-1.1.1 as build # Use the GitHub Actions uid:gid combination for proper fs permissions RUN groupadd -g 116 test && \ diff --git a/Dockerfile b/Dockerfile index 08661246a..a97bac5dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -ARG BASE_IMG=ghcr.io/hiddeco/golang-with-libgit2 -ARG BASE_TAG=dev +ARG BASE_IMG=ghcr.io/fluxcd/golang-with-libgit2 +ARG BASE_TAG=1.16.8-bullseye-libgit2-1.1.1 FROM ${BASE_IMG}:${BASE_TAG} AS build # Configure workspace diff --git a/Makefile b/Makefile index a1a6ee9c7..73fcd5082 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ IMG ?= fluxcd/source-controller TAG ?= latest # Base image used to build the Go binary -BASE_IMG ?= ghcr.io/hiddeco/golang-with-libgit2 -BASE_TAG ?= dev +BASE_IMG ?= ghcr.io/fluxcd/golang-with-libgit2 +BASE_TAG ?= 1.16.8-bullseye-libgit2-1.1.1 # Allows for defining additional Docker buildx arguments, # e.g. '--push'. From 981d91e363399fb353ba98985d13a2415bec445e Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 1 Oct 2021 19:08:59 +0200 Subject: [PATCH 04/10] Update base image to version with Darwin detection To provide a better (contributing) experience to those with Apple machines, as determining the correct paths there is a bit harder. Signed-off-by: Hidde Beydals --- .github/actions/run-tests/Dockerfile | 2 +- Dockerfile | 2 +- Makefile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/run-tests/Dockerfile b/.github/actions/run-tests/Dockerfile index 25e4f2324..afc7d4b5a 100644 --- a/.github/actions/run-tests/Dockerfile +++ b/.github/actions/run-tests/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/fluxcd/golang-with-libgit2:1.16.8-bullseye-libgit2-1.1.1 as build +FROM ghcr.io/fluxcd/golang-with-libgit2:1.16.8-bullseye-libgit2-1.1.1-1 as build # Use the GitHub Actions uid:gid combination for proper fs permissions RUN groupadd -g 116 test && \ diff --git a/Dockerfile b/Dockerfile index a97bac5dc..68d65124a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG BASE_IMG=ghcr.io/fluxcd/golang-with-libgit2 -ARG BASE_TAG=1.16.8-bullseye-libgit2-1.1.1 +ARG BASE_TAG=1.16.8-bullseye-libgit2-1.1.1-1 FROM ${BASE_IMG}:${BASE_TAG} AS build # Configure workspace diff --git a/Makefile b/Makefile index 73fcd5082..c8769a6f2 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ TAG ?= latest # Base image used to build the Go binary BASE_IMG ?= ghcr.io/fluxcd/golang-with-libgit2 -BASE_TAG ?= 1.16.8-bullseye-libgit2-1.1.1 +BASE_TAG ?= 1.16.8-bullseye-libgit2-1.1.1-1 # Allows for defining additional Docker buildx arguments, # e.g. '--push'. From a5a90fd4f6d84e052114c155c298e18ec18e8101 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Mon, 4 Oct 2021 13:51:04 +0200 Subject: [PATCH 05/10] Allow libgit2 build to be enforced This can be useful on machines where libgit2 is installed due to other applications depending on it, but where the composition of this installation does not properly work with the controller. Reason the system version is still preferred, is because this lowers the barrier for drive-by contributors, as a working set of (Git) dependencies should only really be required if you are going to perform work in that domain. Signed-off-by: Hidde Beydals --- Makefile | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c8769a6f2..bdfa840f9 100644 --- a/Makefile +++ b/Makefile @@ -18,8 +18,10 @@ CRD_OPTIONS ?= crd:crdVersions=v1 # Repository root based on Git metadata REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel) -# Dependency versions +# Libgit2 version LIBGIT2_VERSION ?= 1.1.1 + +# Other dependency versions ENVTEST_BIN_VERSION ?= 1.19.2 KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use -i $(ENVTEST_BIN_VERSION) -p path) @@ -31,6 +33,10 @@ LIBGIT2_PATH := $(REPOSITORY_ROOT)/hack/libgit2 LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.so.$(LIBGIT2_VERSION) +ifneq ($(LIBGIT2_VERSION),$(SYSTEM_LIBGIT2_VERSION)) + LIBGIT2_FORCE ?= 1 +endif + # API (doc) generation utilities CONTROLLER_GEN_VERSION ?= v0.5.0 GEN_API_REF_DOCS_VERSION ?= 0.3.0 @@ -164,8 +170,7 @@ endif libgit2: $(LIBGIT2) ## Detect or download libgit2 library $(LIBGIT2): -ifeq ($(LIBGIT2_VERSION),$(SYSTEM_LIBGIT2_VERSION)) -else +ifeq (1, $(LIBGIT2_FORCE)) @{ \ set -e; \ mkdir -p $(LIBGIT2_PATH); \ From 029617cb491004343c875b3de48436710640c587 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Mon, 4 Oct 2021 13:56:59 +0200 Subject: [PATCH 06/10] Detect macOS produced libgit2.dylib on Darwin Signed-off-by: Hidde Beydals --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index bdfa840f9..63e65106e 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,10 @@ ifneq ($(LIBGIT2_VERSION),$(SYSTEM_LIBGIT2_VERSION)) LIBGIT2_FORCE ?= 1 endif +ifeq ($(shell uname -s),Darwin) + LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.$(LIBGIT2_VERSION).dylib +endif + # API (doc) generation utilities CONTROLLER_GEN_VERSION ?= v0.5.0 GEN_API_REF_DOCS_VERSION ?= 0.3.0 From b87448864ab946c12dca863884f85fe1a80e62fd Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Mon, 4 Oct 2021 14:19:34 +0200 Subject: [PATCH 07/10] Document libgit2 build behavior in CONTRIBUTING.md Signed-off-by: Hidde Beydals --- CONTRIBUTING.md | 50 ++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 29ce578b4..76c626873 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,37 +30,37 @@ meeting](https://docs.google.com/document/d/1l_M0om0qUEN_NNiGgpqJ2tvsF2iioHkaARD ### Installing required dependencies -The dependency [libgit2](https://libgit2.org/) needs to be installed to be able to run -Source Controller or its test-suite locally (not in a container). +The dependency [libgit2](https://libgit2.org/) needs to be installed to be able +to run source-controller or its test-suite locally (not in a container). -**macOS** -``` -brew install libgit2 -``` +In case this dependency is not present on your system (at the expected +version), the first invocation of a `make` target that requires the +dependency will attempt to compile it locally to `hack/libgit2`. For this build +to succeed; CMake, Docker, OpenSSL 1.1 and LibSSH2 must be present on the system. -**Arch Linux** -``` -pacman -S libgit2 -``` +Triggering a manual build of the dependency is possible as well by running +`make libgit2`. To enforce the build, for example if your system dependencies +match but are not linked in a compatible way, append `LIBGIT2_FORCE=1` to the +`make` command. -**Building from source** +#### macOS -1. Ensure [`cmake`](https://cmake.org) is available on your system. -1. Download and unarchive [the right `libgit2` version](https://github.com/libgit2/git2go#which-go-version-to-use) - for our current `git2go` dependency: +```console +$ # Ensure libgit2 dependencies are available +$ brew install cmake openssl@1.1 libssh2 +$ LIBGIT2_FORCE=1 make libgit2 +``` + +#### Linux - ```console - $ LIBGIT2_VER=1.1.0 - $ curl -L https://github.com/libgit2/libgit2/releases/download/v$LIBGIT2_VER/libgit2-$LIBGIT2_VER.tar.gz -o /tmp/libgit2.tar.gz - $ tar -xvf /tmp/libgit2.tar.gz -C /tmp/libgit2-$LIBGIT2_VER - ``` -1. Build and install the library on your system: +```console +$ # Ensure libgit2 dependencies are available +$ pacman -S cmake openssl libssh2 +$ LIBGIT2_FORCE=1 make libgit2 +``` - ```console - $ mkdir /tmp/libgit2-$LIBGIT2_VER/build && cd /tmp/libgit2-$LIBGIT2_VER/build - $ cmake .. -DCMAKE_INSTALL_PREFIX=/usr - $ sudo cmake --build . --target install - ``` +**Note:** Example shown is for Arch Linux, but likewise procedure can be +followed using any other package manager, e.g. `apt`. ### How to run the test suite From 9fd16d8e45427174bf527e82cc022bbaece78def Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 7 Oct 2021 17:46:54 +0200 Subject: [PATCH 08/10] Switch to scratch based libgit2 container image This moves the `libgit2` compilation to the image, to ensure it can be build on builders that aren't backed by AMD64. The image is structured in such a way that e.g. running nightly builds targeting a different Go version, or targeting a different OS vendor would be possible in the future via build arguments. Signed-off-by: Hidde Beydals --- .github/workflows/e2e.yaml | 2 +- Dockerfile | 48 ++++++++++++++++++++++++++++++++++---- Makefile | 26 ++++++++------------- 3 files changed, 55 insertions(+), 21 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index b75e70599..12472092b 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -44,7 +44,7 @@ jobs: exit 1 fi - name: Build container image - run: make docker-build IMG=test/source-controller TAG=latest + run: make docker-build IMG=test/source-controller TAG=latest BUILD_PLATFORMS=linux/amd64 BUILD_ARGS=--load - name: Load test image run: kind load docker-image test/source-controller:latest - name: Deploy controller diff --git a/Dockerfile b/Dockerfile index 68d65124a..2f03fd841 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,37 @@ -ARG BASE_IMG=ghcr.io/fluxcd/golang-with-libgit2 -ARG BASE_TAG=1.16.8-bullseye-libgit2-1.1.1-1 -FROM ${BASE_IMG}:${BASE_TAG} AS build +ARG BASE_VARIANT=bullseye +ARG GO_VERSION=1.16.8 +ARG XX_VERSION=1.0.0-rc.2 + +ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2 +ARG LIBGIT2_TAG=libgit2-1.1.1 + +FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx +FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} as libgit2 + +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable +FROM --platform=$BUILDPLATFORM golang:1.17rc1-${BASE_VARIANT} AS golatest + +FROM gostable AS go-linux + +FROM go-${TARGETOS} AS build-base-bullseye + +# Copy the build utiltiies +COPY --from=xx / / +COPY --from=libgit2 /Makefile /libgit2/ + +# Install the libgit2 build dependencies +RUN make -C /libgit2 cmake + +ARG TARGETPLATFORM +RUN make -C /libgit2 dependencies + +FROM build-base-${BASE_VARIANT} as libgit2-bullseye + +# Compile and install libgit2 +ARG TARGETPLATFORM +RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2 libgit2 + +FROM libgit2-${BASE_VARIANT} as build-bullseye # Configure workspace WORKDIR /workspace @@ -27,7 +58,16 @@ ARG TARGETPLATFORM RUN xx-go build -o source-controller -trimpath \ main.go -FROM debian:bullseye-slim as controller +FROM build-${BASE_VARIANT} as prepare-bullseye + +# Move libgit2 lib to generic and predictable location +ARG TARGETPLATFORM +RUN mkdir -p /libgit2/lib/ \ + && cp -d /usr/lib/$(xx-info triple)/libgit2.so* /libgit2/lib/ + +FROM prepare-${BASE_VARIANT} as build + +FROM debian:${BASE_VARIANT}-slim as controller # Link repo to the GitHub Container Registry image LABEL org.opencontainers.image.source="https://github.com/fluxcd/source-controller" diff --git a/Makefile b/Makefile index 63e65106e..47243bd22 100644 --- a/Makefile +++ b/Makefile @@ -3,14 +3,14 @@ IMG ?= fluxcd/source-controller TAG ?= latest # Base image used to build the Go binary -BASE_IMG ?= ghcr.io/fluxcd/golang-with-libgit2 -BASE_TAG ?= 1.16.8-bullseye-libgit2-1.1.1-1 +LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2 +LIBGIT2_TAG ?= libgit2-1.1.1 # Allows for defining additional Docker buildx arguments, # e.g. '--push'. -BUILDX_ARGS ?= +BUILD_ARGS ?= # Architectures to build images for -BUILDX_PLATFORMS ?= linux/amd64,linux/arm64,linux/arm/v7 +BUILD_PLATFORMS ?= linux/amd64,linux/arm64,linux/arm/v7 # Produce CRDs that work back to Kubernetes 1.16 CRD_OPTIONS ?= crd:crdVersions=v1 @@ -110,18 +110,12 @@ generate: controller-gen ## Generate API code cd api; $(CONTROLLER_GEN) object:headerFile="../hack/boilerplate.go.txt" paths="./..." docker-build: ## Build the Docker image - docker build \ - --build-arg BASE_IMG=$(BASE_IMG) \ - --build-arg BASE_TAG=$(BASE_TAG) \ - -t $(IMG):$(TAG) . - -docker-buildx: ## Build the cross-platform Docker image docker buildx build \ - --build-arg BASE_IMG=$(BASE_IMG) \ - --build-arg BASE_TAG=$(BASE_TAG) \ - --platform=$(BUILDX_PLATFORMS) \ + --build-arg LIBGIT2_IMG=$(LIBGIT2_IMG) \ + --build-arg LIBGIT2_TAG=$(LIBGIT2_TAG) \ + --platform=$(BUILD_PLATFORMS) \ -t $(IMG):$(TAG) \ - $(BUILDX_ARGS) . + $(BUILD_ARGS) . docker-push: ## Push Docker image docker push $(IMG):$(TAG) @@ -178,8 +172,8 @@ ifeq (1, $(LIBGIT2_FORCE)) @{ \ set -e; \ mkdir -p $(LIBGIT2_PATH); \ - docker cp $(shell docker create --rm $(BASE_IMG):$(BASE_TAG)):/libgit2/Makefile $(LIBGIT2_PATH); \ - INSTALL_PREFIX=$(LIBGIT2_PATH) make -C $(LIBGIT2_PATH); \ + curl -sL https://raw.githubusercontent.com/fluxcd/golang-with-libgit2/$(LIBGIT2_TAG)/hack/Makefile -o $(LIBGIT2_PATH)/Makefile; \ + INSTALL_PREFIX=$(LIBGIT2_PATH) make -C $(LIBGIT2_PATH) libgit2; \ } endif From a871e0b527e03e7e9a0fed0056f8b0499e2914cb Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 7 Oct 2021 17:53:03 +0200 Subject: [PATCH 09/10] Update Dockerfile used in tests as well This ensures the Dockerfile used for testing is making use of the same scratch image to compile `libgit2` as the actual application image. In a future iteration we should restructure our GitHub Action workflows to re-use the application image, saving us an additional Dockerfile and a duplicate build. Inspiration for this (which makes use of a local registry for the duration of the build) can be found at: https://github.com/fluxcd/golang-with-libgit2/blob/main/.github/workflows/build.yaml Signed-off-by: Hidde Beydals --- .github/actions/run-tests/Dockerfile | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/actions/run-tests/Dockerfile b/.github/actions/run-tests/Dockerfile index afc7d4b5a..6024ca21e 100644 --- a/.github/actions/run-tests/Dockerfile +++ b/.github/actions/run-tests/Dockerfile @@ -1,4 +1,26 @@ -FROM ghcr.io/fluxcd/golang-with-libgit2:1.16.8-bullseye-libgit2-1.1.1-1 as build +ARG BASE_VARIANT=bullseye +ARG GO_VERSION=1.16.8 +ARG XX_VERSION=1.0.0-rc.2 + +ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2 +ARG LIBGIT2_TAG=libgit2-1.1.1 + +FROM tonistiigi/xx:${XX_VERSION} AS xx +FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} as libgit2 + +FROM golang:${GO_VERSION}-${BASE_VARIANT} as gostable + +# Copy the build utiltiies +COPY --from=xx / / +COPY --from=libgit2 /Makefile /libgit2/ + +# Install the libgit2 build dependencies +RUN make -C /libgit2 cmake + +RUN make -C /libgit2 dependencies + +# Compile and install libgit2 +RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2 libgit2 # Use the GitHub Actions uid:gid combination for proper fs permissions RUN groupadd -g 116 test && \ From fa9a2d85126fabe60bb5967f8abfa470dd2797d7 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 8 Oct 2021 09:40:02 +0200 Subject: [PATCH 10/10] CONTRIBUTING: include pkg-config as macOS dep As this isn't available on Darwin by default, unlike on most Linux distributions. Signed-off-by: Hidde Beydals --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 76c626873..c933bbaba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,7 +47,7 @@ match but are not linked in a compatible way, append `LIBGIT2_FORCE=1` to the ```console $ # Ensure libgit2 dependencies are available -$ brew install cmake openssl@1.1 libssh2 +$ brew install cmake openssl@1.1 libssh2 pkg-config $ LIBGIT2_FORCE=1 make libgit2 ```