Release darwin universal static binaries

The changes on static.sh highlight the nuances of compiling for linux and mac, as well as cross compiling on the latter.

Signed-off-by: Paulo Gomes <[email protected]>

Author: Paulo Gomes

.github/workflows/build.yaml

@@ -11,7 +11,89 @@ on:
   pull_request:
     paths-ignore:
       - README.md
+
+permissions:
+  contents: write # needed to write releases
+  packages: write # needed for ghcr access
+
 jobs:
+
+  mac-build:
+    # This job builds and releases "universal libraries" that are
+    # supported by both darwin-amd64 and darwin-arm64.
+    #
+    # First builds in amd64, then cross-compile in arm64. Later combining
+    # both outcomes onto a single binary for each static library.
+    #
+    # `macos-11` has been picked as support for arm64 was only added on Xcode 12.
+    # Although some minor versions of Catalina 10.15 can support it, at the time 
+    # of testing, GitHub's macos-10.15 did not seem to. 
+    # Cross-compiling to arm64 on that runner consistently failed.
+    runs-on: macos-11
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Build universal static libraries for Darwin
+        run: |
+          TARGET_DIR=${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64 \
+          BUILD_ROOT_DIR=${GITHUB_WORKSPACE}/libgit2/build/amd \
+          ./hack/static.sh all
+          
+          TARGET_DIR=${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64 \
+          BUILD_ROOT_DIR=${GITHUB_WORKSPACE}/libgit2/build/arm \
+          TARGET_ARCH=arm64 \
+          CMAKE_APPLE_SILICON_PROCESSOR=arm64 \
+          ./hack/static.sh all
+
+          mkdir -p ./libgit2-darwin/lib
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/include ./libgit2-darwin/
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/share ./libgit2-darwin/
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/cmake ./libgit2-darwin/lib/
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/engines-3 ./libgit2-darwin/lib/
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/ossl-modules ./libgit2-darwin/lib/
+          mv ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/pkgconfig ./libgit2-darwin/lib/
+
+          libtool -static -o ./libgit2-darwin/lib/libcrypto.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libcrypto.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libcrypto.a 
+          libtool -static -o ./libgit2-darwin/lib/libgit2.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libgit2.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libgit2.a 
+          libtool -static -o ./libgit2-darwin/lib/libssh2.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libssh2.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libssh2.a 
+          libtool -static -o ./libgit2-darwin/lib/libssl.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libssl.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libssl.a  
+          libtool -static -o ./libgit2-darwin/lib/libz.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/libz.a \
+            ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libz.a 
+
+          tar -zcvf darwin-libs.tar.gz ./libgit2-darwin
+
+      - name: Create Release
+        if: github.event_name != 'pull_request'
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          release_name: ${{ github.ref }}
+          tag_name: ${{ github.ref }}
+          draft: false
+          prerelease: true
+      - name: Upload Release Asset
+        if: github.event_name != 'pull_request'
+        id: upload-release-asset 
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./darwin-libs.tar.gz
+          asset_name: darwin-libs.tar.gz
+          asset_content_type: application/gzip
+
   build:
     runs-on: ubuntu-latest
     env:
@@ -55,7 +137,7 @@ jobs:
           key: ${{ runner.os }}-buildx-ghcache-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}-buildx-ghcache-
-      - run: cat ./hack/Makefile
+      - run: cat ./hack/static.sh
       - name: Build candidate image
         id: build_candidate
         uses: docker/build-push-action@v2

README.md

@@ -1,13 +1,17 @@
 # golang-with-libgit2
 
-This repository contains a `Dockerfile` with two files: `Makefile` and `static.sh`. 
+This repository contains a `Dockerfile` with the statically built libgit2 and its dependency chain.
+
+The `hack` directory contains two main files: `Makefile` and `static.sh`.
 Both of which can be used to build the [libgit2][] dependency chain for **AMD64, ARM64 and ARMv7** binaries 
 of Go projects that depend on [git2go][]. 
 
 The `Makefile` is useful for development environments and will leverage OS specific packages to build `libgit2`.
 The `static.sh` will build all `libgit2` dependencies from source using `musl` toolchain. This enables for a full
 static binary with the freedom of configuring each of the dependencies in chain.
 
+Alternatively, the statically built libraries can be pulling from the produced images for Linux or from the github release artifacts for MacOS.
+
 ### :warning: **Public usage discouraged**
 
 The set of dependencies was handpicked for the Flux project, based on the issue list documented below. While this setup
@@ -47,17 +51,14 @@ while testing these against the git2go code before releasing the image.
   - [ ] [libgit2/git2go#836](https://github.com/libgit2/git2go/issues/836)
   - [ ] [libgit2/git2go#837](https://github.com/libgit2/git2go/issues/837)
 
----
-**NOTE**
 
-The issues above do not affect libgit2 built with `static.sh` as all its
+> **NOTE:** The issues above do not affect libgit2 built with `static.sh` as all its
 dependencies have been configured to be optimal for its use, as the first supported version of libgit2 is `1.3.0`.
 
----
 
 ## Usage
 
-The [Dockerfile.test](./Dockerfile.test) file provides a working example on how to statically build a golang application that has a dependency to libgit2 and git2go.
+The [Dockerfile.test](./Dockerfile.test) file provides a working example on how to statically build a golang application that has a dependency on libgit2 and git2go.
 
 The example will statically build all dependencies based on the versions specified on `static.sh`.
 Then statically build the golang application and deploy it into an image based off `gcr.io/distroless/static`.

hack/static.sh

@@ -14,7 +14,7 @@ TARGET_DIR="${TARGET_DIR:-/usr/local/$(xx-info triple)}"
 BUILD_ROOT_DIR="${BUILD_ROOT_DIR:-/build}"
 SRC_DIR="${BUILD_ROOT_DIR}/src"
 
-TARGET_ARCH="$(uname -m)"
+TARGET_ARCH="${TARGET_ARCH:-$(uname -m)}"
 if command -v xx-info; then 
     TARGET_ARCH="$(xx-info march)"
 fi
@@ -59,21 +59,34 @@ function build_openssl(){
     export OPENSSL_ROOT_DIR="${TARGET_DIR}"
     export OPENSSL_LIBRARIES="${TARGET_DIR}/lib"
 
+    export KERNEL_BITS=64
     target_arch=""
-    if [ "${TARGET_ARCH}" = "armv7l" ]; then
-        # openssl does not have a specific armv7l
-        # using generic32 instead.
-        target_arch="linux-generic32"
-    elif [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then
-        target_arch="linux-aarch64"  
-    elif [ "${TARGET_ARCH}" = "x86_64" ]; then
-        target_arch="linux-x86_64"
+    if [[ ! $OSTYPE == darwin* ]]; then
+        if [ "${TARGET_ARCH}" = "armv7l" ]; then
+            # openssl does not have a specific armv7l
+            # using generic32 instead.
+            target_arch="linux-generic32"
+            export KERNEL_BITS=32
+        elif [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then
+            target_arch="linux-aarch64"
+        elif [ "${TARGET_ARCH}" = "x86_64" ]; then
+            target_arch="linux-x86_64"
+        fi
     else
-        echo "Architecture currently not supported: ${TARGET_ARCH}"
-        exit 1
+        SUFFIX=""
+        if [ ! "${TARGET_ARCH}" = "$(uname -m)" ]; then
+            SUFFIX="-cc"
+        fi
+
+        if [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then
+            target_arch="darwin64-arm64${SUFFIX}"
+        elif [ "${TARGET_ARCH}" = "x86_64" ]; then
+            target_arch="darwin64-x86_64${SUFFIX}"
+        fi
+        # if none of the above, let openssl figure it out.
     fi
 
-    ./Configure "${target_arch}" threads no-shared zlib -fPIC -DOPENSSL_PIC \
+    ./Configure "${target_arch}" threads no-shared no-stdio no-tests zlib -fPIC -DOPENSSL_PIC \
         --prefix="${TARGET_DIR}" \
         --with-zlib-include="${TARGET_DIR}/include" \
         --with-zlib-lib="${TARGET_DIR}/lib" \
@@ -94,20 +107,28 @@ function build_libssh2(){
     pushd build
 
     OPENSSL_LIBRARIES="${TARGET_DIR}/lib"
-    if [ "${TARGET_ARCH}" = "x86_64" ]; then
+    if [ "${TARGET_ARCH}" = "x86_64" ] && [[ ! $OSTYPE == darwin* ]]; then
         OPENSSL_LIBRARIES="${TARGET_DIR}/lib64"
     fi
 
+    # Set osx arch only when cross compiling on darwin
+    if [[ $OSTYPE == darwin* ]] && [ ! "${TARGET_ARCH}" = "$(uname -m)" ]; then
+        CMAKE_PARAMS=-DCMAKE_OSX_ARCHITECTURES="${TARGET_ARCH}"
+    fi
 
+    # Building examples allow for validating against missing symbols at compilation time.
     cmake "${CMAKE_PARAMS}" \
         -DCMAKE_C_COMPILER="${C_COMPILER}" \
         -DCMAKE_INSTALL_PREFIX="${TARGET_DIR}" \
-        -DBUILD_SHARED_LIBS=OFF \
-        -DLINT=OFF \
+        -DBUILD_SHARED_LIBS:BOOL=OFF \
+        -DLINT:BOOL=OFF \
+        -DBUILD_EXAMPLES:BOOL=ON \
+        -DBUILD_TESTING:BOOL=OFF \
         -DCMAKE_C_FLAGS=-fPIC \
         -DCRYPTO_BACKEND=OpenSSL \
-        -DENABLE_ZLIB_COMPRESSION=ON \
+        -DENABLE_ZLIB_COMPRESSION:BOOL=ON \
         -DCMAKE_BUILD_TYPE="RelWithDebInfo" \
+        -DZLIB_LIBRARY="${TARGET_DIR}/lib/libz.a" \
         -DOPENSSL_CRYPTO_LIBRARY="${OPENSSL_LIBRARIES}/libcrypto.a" \
         -DOPENSSL_SSL_LIBRARY="${OPENSSL_LIBRARIES}/libssl.a" \
         ..
@@ -129,17 +150,21 @@ function build_libgit2(){
 
     SSL_LIBRARY="${TARGET_DIR}/lib/libssl.a"
     CRYPTO_LIBRARY="${TARGET_DIR}/lib/libcrypto.a"
-    if [ "${TARGET_ARCH}" = "x86_64" ]; then
+    if [[ ! $OSTYPE == darwin* ]] && [ "${TARGET_ARCH}" = "x86_64" ]; then
         SSL_LIBRARY="${TARGET_DIR}/lib64/libssl.a"
         CRYPTO_LIBRARY="${TARGET_DIR}/lib64/libcrypto.a"
     fi
 
+    # Set osx arch only when cross compiling on darwin
+    if [[ $OSTYPE == darwin* ]] && [ ! "${TARGET_ARCH}" = "$(uname -m)" ]; then
+        CMAKE_PARAMS=-DCMAKE_OSX_ARCHITECTURES="${TARGET_ARCH}"
+    fi
+
     cmake "${CMAKE_PARAMS}" \
         -DCMAKE_C_COMPILER="${C_COMPILER}" \
         -DCMAKE_INSTALL_PREFIX="${TARGET_DIR}" \
         -DTHREADSAFE:BOOL=ON \
         -DBUILD_CLAR:BOOL=OFF \
-        -DBUILD_TESTS:BOOL=OFF \
         -DBUILD_SHARED_LIBS=OFF \
         -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON \
         -DCMAKE_C_FLAGS=-fPIC \
@@ -151,9 +176,9 @@ function build_libgit2(){
         -DREGEX_BACKEND:STRING=builtin \
         -DOPENSSL_SSL_LIBRARY="${SSL_LIBRARY}" \
         -DOPENSSL_CRYPTO_LIBRARY="${CRYPTO_LIBRARY}" \
-        -DZLIB_LIBRARY="${TARGET_DIR}/lib/libz.a" \
         -DCMAKE_INCLUDE_PATH="${TARGET_DIR}/include" \
         -DCMAKE_LIBRARY_PATH="${TARGET_DIR}/lib" \
+        -DCMAKE_PREFIX_PATH="${TARGET_DIR}" \
         -DCMAKE_BUILD_TYPE="RelWithDebInfo" \
         ..