Sync libgit2-1.1.1 and main branches

.github/workflows/build.yaml

@@ -11,7 +11,34 @@ on:
   pull_request:
     paths-ignore:
       - README.md
+
+permissions:
+  packages: write # needed for ghcr access
+
 jobs:
+
+  mac-build:
+    runs-on: macos-11
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Test build script for darwin-amd64
+        run: |
+          TARGET_DIR=${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64 \
+          BUILD_ROOT_DIR=${GITHUB_WORKSPACE}/libgit2/build/amd \
+          ./hack/static.sh all
+        env:
+          MACOSX_DEPLOYMENT_TARGET: 10.15
+      - name: Test cross compile script for darwin-arm64
+        run: |
+          TARGET_DIR=${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64 \
+          BUILD_ROOT_DIR=${GITHUB_WORKSPACE}/libgit2/build/arm \
+          TARGET_ARCH=arm64 \
+          CMAKE_APPLE_SILICON_PROCESSOR=arm64 \
+          ./hack/static.sh all
+        env:
+          MACOSX_DEPLOYMENT_TARGET: 10.15
+
   build:
     runs-on: ubuntu-latest
     env:
@@ -55,7 +82,7 @@ jobs:
           key: ${{ runner.os }}-buildx-ghcache-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}-buildx-ghcache-
-      - run: cat ./hack/Makefile
+      - run: cat ./hack/static.sh
       - name: Build candidate image
         id: build_candidate
         uses: docker/build-push-action@v2

.github/workflows/release.yaml

@@ -0,0 +1,102 @@
+---
+name: 'Release MacOS static libraries'
+on:
+  push:
+    tags:
+      - '**'
+
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'release prefix'
+        default: 'rc'
+        required: true
+
+permissions:
+  contents: write # needed to write releases
+
+jobs:
+
+  mac-build:
+    # This job builds and releases "universal libraries" that are
+    # supported by both darwin-amd64 and darwin-arm64.
+    #
+    # First builds in amd64, then cross-compile in arm64. Later combining
+    # both outcomes onto a single binary for each static library.
+    #
+    # `macos-11` has been picked as support for arm64 was only added on Xcode 12.
+    # Although some minor versions of Catalina 10.15 can support it, at the time 
+    # of testing, GitHub's macos-10.15 did not seem to. 
+    # Cross-compiling to arm64 on that runner consistently failed.
+    runs-on: macos-11
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Build universal static libraries for Darwin
+        run: |
+          TARGET_DIR=${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64 \
+          BUILD_ROOT_DIR=${GITHUB_WORKSPACE}/libgit2/build/amd \
+          ./hack/static.sh all
+
+          TARGET_DIR=${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64 \
+          BUILD_ROOT_DIR=${GITHUB_WORKSPACE}/libgit2/build/arm \
+          TARGET_ARCH=arm64 \
+          CMAKE_APPLE_SILICON_PROCESSOR=arm64 \
+          ./hack/static.sh all
+
+          mkdir -p ./libgit2-darwin/lib
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/include ./libgit2-darwin/
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/share ./libgit2-darwin/
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/cmake ./libgit2-darwin/lib/
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/engines-3 ./libgit2-darwin/lib/
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/ossl-modules ./libgit2-darwin/lib/
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/pkgconfig ./libgit2-darwin/lib/
+
+          libtool -static -o ./libgit2-darwin/lib/libcrypto.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libcrypto.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libcrypto.a 
+          libtool -static -o ./libgit2-darwin/lib/libgit2.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libgit2.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libgit2.a 
+          libtool -static -o ./libgit2-darwin/lib/libssh2.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libssh2.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libssh2.a 
+          libtool -static -o ./libgit2-darwin/lib/libssl.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libssl.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libssl.a  
+          libtool -static -o ./libgit2-darwin/lib/libz.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libz.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libz.a 
+
+          tar -zcvf darwin-libs.tar.gz ./libgit2-darwin
+        env:
+          MACOSX_DEPLOYMENT_TARGET: 10.15
+
+      - name: Prepare
+        id: prep
+        run: |
+          VERSION="${{ github.event.inputs.tag }}-${GITHUB_SHA::8}"
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION=${GITHUB_REF/refs\/tags\//}
+          fi
+          echo ::set-output name=VERSION::${VERSION}
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          release_name: ${{ steps.prep.outputs.VERSION }}
+          tag_name: ${{ steps.prep.outputs.VERSION }}
+          draft: false
+          prerelease: true
+      - name: Upload Release Asset
+        id: upload-release-asset 
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./darwin-libs.tar.gz
+          asset_name: darwin-libs.tar.gz
+          asset_content_type: application/gzip

Dockerfile

@@ -1,15 +1,11 @@
 # This Dockerfile tests the hack/Makefile output against git2go.
 ARG BASE_VARIANT=alpine
-ARG GO_VERSION=1.17.6
+ARG GO_VERSION=1.17
 ARG XX_VERSION=1.1.0
 
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 
-FROM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
-
-FROM gostable AS go-linux
-
-FROM --platform=$BUILDPLATFORM ${BASE_VARIANT} AS build-deps
+FROM --platform=$BUILDPLATFORM ${BASE_VARIANT} AS build-base
 
 RUN apk add --no-cache \
         bash \
@@ -26,15 +22,10 @@ RUN apk add --no-cache \
 
 COPY --from=xx / /
 
-ARG TARGETPLATFORM
-
-RUN xx-apk add --no-cache \
-        xx-c-essentials 
-
-RUN xx-apk add --no-cache \
-        xx-cxx-essentials 
+FROM build-base AS build-cross
 
 ARG TARGETPLATFORM
+
 RUN xx-apk add --no-cache \
         build-base \
         pkgconfig \
@@ -48,7 +39,6 @@ RUN xx-apk add --no-cache \
 WORKDIR /build
 COPY hack/static.sh .
 
-ARG TARGETPLATFORM
 ENV CC=xx-clang
 ENV CXX=xx-clang++
 
@@ -68,12 +58,25 @@ RUN ./static.sh build_libssh2
 RUN ./static.sh build_libgit2
 
 
-FROM go-${TARGETOS} AS build
+# trimmed removes all non necessary files (i.e. openssl binary).
+FROM build-cross AS trimmed
 
-# Copy cross-compilation tools
-COPY --from=xx / /
-# Copy compiled libraries
-COPY --from=build-deps /usr/local/ /usr/local/
+ARG TARGETPLATFORM
+RUN mkdir -p /trimmed/usr/local/$(xx-info triple)/ && \
+        mkdir -p /trimmed/usr/local/$(xx-info triple)/share
+
+RUN cp -r /usr/local/$(xx-info triple)/lib/ /trimmed/usr/local/$(xx-info triple)/ && \
+        cp -r /usr/local/$(xx-info triple)/lib64/ /trimmed/usr/local/$(xx-info triple)/ | true && \
+        cp -r /usr/local/$(xx-info triple)/include/ /trimmed/usr/local/$(xx-info triple)/ && \
+        cp -r /usr/local/$(xx-info triple)/share/doc/ /trimmed/usr/local/$(xx-info triple)/share/
+
+FROM scratch as libs-arm64
+COPY --from=trimmed /trimmed/ /
+
+FROM scratch as libs-amd64
+COPY --from=trimmed /trimmed/ /
+
+FROM scratch as libs-armv7
+COPY --from=trimmed /trimmed/ /
 
-COPY ./hack/Makefile /Makefile
-COPY ./hack/static.sh /static.sh
+FROM libs-$TARGETARCH$TARGETVARIANT as libs

Dockerfile.test

@@ -1,15 +1,11 @@
 # This Dockerfile tests the hack/Makefile output against git2go.
 ARG BASE_VARIANT=alpine
-ARG GO_VERSION=1.17.6
+ARG GO_VERSION=1.17
 ARG XX_VERSION=1.1.0
 
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 
-FROM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
-
-FROM gostable AS go-linux
-
-FROM --platform=$BUILDPLATFORM ${BASE_VARIANT} AS build-deps
+FROM --platform=$BUILDPLATFORM ${BASE_VARIANT} AS build-base
 
 RUN apk add --no-cache \
         bash \
@@ -26,15 +22,10 @@ RUN apk add --no-cache \
 
 COPY --from=xx / /
 
-ARG TARGETPLATFORM
-
-RUN xx-apk add --no-cache \
-        xx-c-essentials 
-
-RUN xx-apk add --no-cache \
-        xx-cxx-essentials 
+FROM build-base AS build-cross
 
 ARG TARGETPLATFORM
+
 RUN xx-apk add --no-cache \
         build-base \
         pkgconfig \
@@ -48,7 +39,6 @@ RUN xx-apk add --no-cache \
 WORKDIR /build
 COPY hack/static.sh .
 
-ARG TARGETPLATFORM
 ENV CC=xx-clang
 ENV CXX=xx-clang++
 
@@ -68,40 +58,76 @@ RUN ./static.sh build_libssh2
 RUN ./static.sh build_libgit2
 
 
-FROM go-${TARGETOS} AS build
+# trimmed removes all non necessary files (i.e. openssl binary).
+FROM build-cross AS trimmed
 
-# Copy cross-compilation tools
-COPY --from=xx / /
-# Copy compiled libraries
-COPY --from=build-deps /usr/local/ /usr/local/
+ARG TARGETPLATFORM
+RUN mkdir -p /trimmed/usr/local/$(xx-info triple)/ && \
+        mkdir -p /trimmed/usr/local/$(xx-info triple)/share
+
+RUN cp -r /usr/local/$(xx-info triple)/lib/ /trimmed/usr/local/$(xx-info triple)/ && \
+        cp -r /usr/local/$(xx-info triple)/lib64/ /trimmed/usr/local/$(xx-info triple)/ | true && \
+        cp -r /usr/local/$(xx-info triple)/include/ /trimmed/usr/local/$(xx-info triple)/ && \
+        cp -r /usr/local/$(xx-info triple)/share/doc/ /trimmed/usr/local/$(xx-info triple)/share/
+
+FROM scratch as libs-arm64
+COPY --from=trimmed /trimmed/ /
+
+FROM scratch as libs-amd64
+COPY --from=trimmed /trimmed/ /
+
+FROM scratch as libs-armv7
+COPY --from=trimmed /trimmed/ /
+
+FROM libs-$TARGETARCH$TARGETVARIANT as libs
+
+# Everything above this line is a copy from Dockefile.
+
+FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
+
+FROM gostable AS go-linux
+
+# Build-base consists of build platform dependencies and xx.
+# These will be used at current arch to yield execute the cross compilations.
+FROM go-${TARGETOS} AS go-base
 
 RUN apk add clang lld pkgconfig
 
+COPY --from=xx / /
+
+# build-go-mod can still be cached at build platform architecture.
+FROM go-base as build-go-mod
+
 WORKDIR /root/smoketest
 COPY tests/smoketest/go.mod .
 COPY tests/smoketest/go.sum .
 RUN go mod download
 
+# Build stage install per target platform
+# dependency and effectively cross compile the application.
+FROM build-go-mod as build
+
 ARG TARGETPLATFORM
 
 # Some dependencies have to installed 
 # for the target platform: https://github.com/tonistiigi/xx#go--cgo
-RUN xx-apk add --no-cache \
-        musl-dev \
-        gcc
+RUN xx-apk add musl-dev gcc clang lld
 
+WORKDIR /root/smoketest
 
 COPY tests/smoketest/main.go .
+COPY --from=libs /usr/local/ /usr/local/
 
 ENV CGO_ENABLED=1
-RUN export LIBRARY_PATH="/usr/local/$(xx-info triple)/lib:/usr/local/$(xx-info triple)/lib64:${LIBRARY_PATH}" && \
+RUN export LIBRARY_PATH="/usr/local/$(xx-info triple):/usr/local/$(xx-info triple)/lib64" && \
     export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig:/usr/local/$(xx-info triple)/lib64/pkgconfig" && \
-	export FLAGS="$(pkg-config --static --libs --cflags libssh2 openssl libgit2)" && \
-    CGO_LDFLAGS="${FLAGS} -static" \
-	xx-go build \
+    export FLAGS="$(pkg-config --static --libs --cflags libssh2 openssl libgit2)" && \
+    export CGO_LDFLAGS="${FLAGS} -static" && \
+    xx-go build  \
         -ldflags "-s -w" \
         -tags 'netgo,osusergo,static_build' \
-        -o static-test-runner -trimpath main.go;
+        -o static-test-runner -trimpath main.go
+
 
 # Ensure that the generated binary is valid for the target platform
 RUN xx-verify --static static-test-runner
@@ -119,11 +145,4 @@ COPY --from=build \
 
 ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
 
-# To do docker run instead, replace the RUN statement with:
-# ENTRYPOINT [ "/root/smoketest/static-test-runner" ]
-
-# The approach below was preferred as it provides a way to 
-# assert the functionality across the supported architectures 
-# without any extra steps.
-
 RUN /root/smoketest/static-test-runner