Merge pull request #437 from fluxcd/update-libgit2

Update github.com/libgit2/git2go to v31.6.1

Author: hiddeco

.dockerignore

@@ -0,0 +1 @@
+hack/libgit2/

.github/actions/run-tests/Dockerfile

@@ -1,17 +1,26 @@
-FROM golang:1.16-buster as builder
-
-# Up-to-date libgit2 dependencies are only available in
-# unstable, as libssh2 in testing/bullseye has been linked
-# against gcrypt which causes issues with PKCS* formats.
-# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271
-RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \
-    && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list
-RUN set -eux; \
-    apt-get update \
-    && apt-get install -y libgit2-dev/unstable \
-    && apt-get clean \
-    && apt-get autoremove --purge -y \
-    && rm -rf /var/lib/apt/lists/*
+ARG BASE_VARIANT=bullseye
+ARG GO_VERSION=1.16.8
+ARG XX_VERSION=1.0.0-rc.2
+
+ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2
+ARG LIBGIT2_TAG=libgit2-1.1.1
+
+FROM tonistiigi/xx:${XX_VERSION} AS xx
+FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} as libgit2
+
+FROM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
+
+# Copy the build utiltiies
+COPY --from=xx / /
+COPY --from=libgit2 /Makefile /libgit2/
+
+# Install the libgit2 build dependencies
+RUN make -C /libgit2 cmake
+
+RUN make -C /libgit2 dependencies
+
+# Compile and install libgit2
+RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2 libgit2
 
 # Use the GitHub Actions uid:gid combination for proper fs permissions
 RUN groupadd -g 116 test && \

.github/workflows/e2e.yaml

@@ -44,11 +44,11 @@ jobs:
             exit 1
           fi
       - name: Build container image
-        run: make docker-build IMG=test/source-controller:latest
+        run: make docker-build IMG=test/source-controller TAG=latest BUILD_PLATFORMS=linux/amd64 BUILD_ARGS=--load
       - name: Load test image
         run: kind load docker-image test/source-controller:latest
       - name: Deploy controller
-        run: make dev-deploy IMG=test/source-controller:latest
+        run: make dev-deploy IMG=test/source-controller TAG=latest
       - name: Run smoke tests
         run: |
           kubectl -n source-system apply -f ./config/samples

.gitignore

@@ -15,3 +15,6 @@
 # vendor/
 bin/
 config/release/
+
+# Exclude all libgit2 related files
+hack/libgit2/

CONTRIBUTING.md

@@ -30,37 +30,37 @@ meeting](https://docs.google.com/document/d/1l_M0om0qUEN_NNiGgpqJ2tvsF2iioHkaARD
 
 ### Installing required dependencies
 
-The dependency [libgit2](https://libgit2.org/) needs to be installed to be able to run
-Source Controller or its test-suite locally (not in a container).
+The dependency [libgit2](https://libgit2.org/) needs to be installed to be able
+to run source-controller or its test-suite locally (not in a container).
 
-**macOS**
-```
-brew install libgit2
-```
+In case this dependency is not present on your system (at the expected
+version), the first invocation of a `make` target that requires the
+dependency will attempt to compile it locally to `hack/libgit2`. For this build
+to succeed; CMake, Docker, OpenSSL 1.1 and LibSSH2 must be present on the system.
 
-**Arch Linux**
-```
-pacman -S libgit2
-```
+Triggering a manual build of the dependency is possible as well by running
+`make libgit2`. To enforce the build, for example if your system dependencies
+match but are not linked in a compatible way, append `LIBGIT2_FORCE=1` to the
+`make` command.
 
-**Building from source**
+#### macOS
 
-1. Ensure [`cmake`](https://cmake.org) is available on your system.
-1. Download and unarchive [the right `libgit2` version](https://github.com/libgit2/git2go#which-go-version-to-use)
-   for our current `git2go` dependency:
+```console
+$ # Ensure libgit2 dependencies are available
+$ brew install cmake [email protected] libssh2 pkg-config
+$ LIBGIT2_FORCE=1 make libgit2
+```
+
+#### Linux
 
-   ```console
-   $ LIBGIT2_VER=1.1.0
-   $ curl -L https://github.com/libgit2/libgit2/releases/download/v$LIBGIT2_VER/libgit2-$LIBGIT2_VER.tar.gz -o /tmp/libgit2.tar.gz
-   $ tar -xvf /tmp/libgit2.tar.gz -C /tmp/libgit2-$LIBGIT2_VER
-   ```
-1. Build and install the library on your system:
+```console
+$ # Ensure libgit2 dependencies are available
+$ pacman -S cmake openssl libssh2
+$ LIBGIT2_FORCE=1 make libgit2
+```
 
-   ```console
-   $ mkdir /tmp/libgit2-$LIBGIT2_VER/build && cd /tmp/libgit2-$LIBGIT2_VER/build
-   $ cmake .. -DCMAKE_INSTALL_PREFIX=/usr
-   $ sudo cmake --build . --target install
-   ```
+**Note:** Example shown is for Arch Linux, but likewise procedure can be
+followed using any other package manager, e.g. `apt`.
 
 ### How to run the test suite
 

Dockerfile

@@ -1,69 +1,97 @@
-FROM golang:1.16-buster as builder
-
-# Up-to-date libgit2 dependencies are only available in
-# unstable, as libssh2 in testing/bullseye has been linked
-# against gcrypt which causes issues with PKCS* formats.
-# Explicitly listing all build dependencies is required because
-# they can only be automagically found for AMD64 builds.
-# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271
-RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \
-    && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list
-RUN set -eux; \
-    apt-get update \
-    && apt-get install -y \
-        libgit2-dev/unstable \
-        zlib1g-dev/unstable \
-        libssh2-1-dev/unstable \
-        libpcre3-dev/unstable \
-    && apt-get clean \
-    && apt-get autoremove --purge -y \
-    && rm -rf /var/lib/apt/lists/*
+ARG BASE_VARIANT=bullseye
+ARG GO_VERSION=1.16.8
+ARG XX_VERSION=1.0.0-rc.2
+
+ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2
+ARG LIBGIT2_TAG=libgit2-1.1.1
+
+FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
+FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} as libgit2
+
+FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
+FROM --platform=$BUILDPLATFORM golang:1.17rc1-${BASE_VARIANT} AS golatest
+
+FROM gostable AS go-linux
+
+FROM go-${TARGETOS} AS build-base-bullseye
+
+# Copy the build utiltiies
+COPY --from=xx / /
+COPY --from=libgit2 /Makefile /libgit2/
+
+# Install the libgit2 build dependencies
+RUN make -C /libgit2 cmake
+
+ARG TARGETPLATFORM
+RUN make -C /libgit2 dependencies
+
+FROM build-base-${BASE_VARIANT} as libgit2-bullseye
 
+# Compile and install libgit2
+ARG TARGETPLATFORM
+RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2 libgit2
+
+FROM libgit2-${BASE_VARIANT} as build-bullseye
+
+# Configure workspace
 WORKDIR /workspace
 
-# copy api submodule
+# Copy api submodule
 COPY api/ api/
 
-# copy modules manifests
+# Copy modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
 
-# cache modules
+# Cache modules
 RUN go mod download
 
-# copy source code
+# Copy source code
 COPY main.go main.go
 COPY controllers/ controllers/
 COPY pkg/ pkg/
 COPY internal/ internal/
 
-# build without specifing the arch
-RUN CGO_ENABLED=1 go build -o source-controller main.go
+# Build the binary
+ENV CGO_ENABLED=1
+ARG TARGETPLATFORM
+RUN xx-go build -o source-controller -trimpath \
+    main.go
 
-FROM debian:buster-slim as controller
+FROM build-${BASE_VARIANT} as prepare-bullseye
 
-# link repo to the GitHub Container Registry image
-LABEL org.opencontainers.image.source="https://github.com/fluxcd/source-controller"
+# Move libgit2 lib to generic and predictable location
+ARG TARGETPLATFORM
+RUN mkdir -p /libgit2/lib/ \
+    && cp -d /usr/lib/$(xx-info triple)/libgit2.so* /libgit2/lib/
 
-# Up-to-date libgit2 dependencies are only available in
-# unstable, as libssh2 in testing/bullseye has been linked
-# against gcrypt which causes issues with PKCS* formats.
-# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271
-RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \
-    && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list
-RUN set -eux; \
-    apt-get update \
-    && apt-get install -y \
-        ca-certificates \
-        libgit2-1.1 \
-    && apt-get clean \
-    && apt-get autoremove --purge -y \
-    && rm -rf /var/lib/apt/lists/*
+FROM prepare-${BASE_VARIANT} as build
+
+FROM debian:${BASE_VARIANT}-slim as controller
 
-COPY --from=builder /workspace/source-controller /usr/local/bin/
+# Link repo to the GitHub Container Registry image
+LABEL org.opencontainers.image.source="https://github.com/fluxcd/source-controller"
 
+# Configure user
 RUN groupadd controller && \
     useradd --gid controller --shell /bin/sh --create-home controller
 
+# Copy libgit2
+COPY --from=build /libgit2/lib/ /usr/local/lib/
+RUN ldconfig
+
+# Upgrade packages and install runtime dependencies
+RUN echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list \
+    && echo "deb-src http://deb.debian.org/debian sid main" >> /etc/apt/sources.list \
+    && apt update \
+    && apt install --no-install-recommends -y zlib1g/sid libssl1.1/sid libssh2-1/sid \
+    && apt install --no-install-recommends -y ca-certificates \
+    && apt clean \
+    && apt autoremove --purge -y \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy over binary from build
+COPY --from=build /workspace/source-controller /usr/local/bin/
+
 USER controller
 ENTRYPOINT [ "source-controller" ]