diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml new file mode 100644 index 000000000..aadd87af8 --- /dev/null +++ b/.github/workflows/publish-image.yml @@ -0,0 +1,313 @@ +name: Release Docker Image + +on: + pull_request: + push: + paths-ignore: + - '.gitlab-ci.yml' + - '.gitignore' + - 'renovate.json' + - 'debian/**' + - 'docker/**' + branches: + - main + tags: + - '*' + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: +env: + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} + GHCR_ENABLED: "${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) && 'true' || 'false' }}" +jobs: + + define-variables: + runs-on: ubuntu-latest + + outputs: + images: ${{ steps.var.outputs.images }} + images_list: ${{ steps.var.outputs.images_list }} + build_matrix: ${{ steps.var.outputs.build_matrix }} + merge_matrix: ${{ steps.var.outputs.merge_matrix }} + ghcr_image: ${{ steps.var.outputs.ghcr_image }} + docker_image: ${{ steps.var.outputs.docker_image }} + glhr_image: ${{ steps.var.outputs.glhr_image }} + + steps: + - name: Setting variables + uses: actions/github-script@v7 + id: var + with: + script: | + const githubRepo = '${{ github.repository }}'.toLowerCase() + const repoId = githubRepo.split('/')[1] + + core.setOutput('github_repository', githubRepo) + const ghcrImage = 'ghcr.io/' + githubRepo + const glhrImage = 'registry.gitlab.com/' + '${{ vars.GITLAB_USERNAME }}'.toLowerCase() + '/' + repoId + const dockerImage = 'docker.io/' + '${{ vars.DOCKER_USERNAME }}'.toLowerCase() + '/' + repoId + core.setOutput('ghcr_image', ghcrImage) + core.setOutput('docker_image', dockerImage) + core.setOutput('glhr_image', glhrImage) + let images = [] + if (process.env.GHCR_ENABLED === "true") { + images.push(ghcrImage) + } + if (process.env.DOCKERHUB_TOKEN) { + images.push(dockerImage) + } + if (process.env.GITLAB_TOKEN) { + images.push(glhrImage) + } + core.setOutput('images', images.join("\n")) + core.setOutput('images_list', images.join(",")) + const platforms = ['linux/amd64', 'linux/arm64'] + // specific CPU build variations + const includes = [ + { + platform: 'linux/amd64', + target_cpu: 'haswell', + tag_suffix: '-haswell' + } + ] + // Image builds + core.setOutput('build_matrix', JSON.stringify({ + platform: platforms, + target_cpu: ['base'], + include: includes + })) + // image publishes + core.setOutput('merge_matrix', JSON.stringify({ + tag_suffix: ['', ...includes.map((i) => i.tag_suffix).filter((i) => typeof i == 'string')] + })) + build-image: + runs-on: ubuntu-latest + # only build if we can push to at least one registry + if: needs.define-variables.outputs.images != '' + needs: define-variables + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + attestations: write + id-token: write + strategy: + matrix: ${{ fromJSON(needs.define-variables.outputs.build_matrix) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + # For publishing multi-platform manifests + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to Docker Hub + if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} + uses: docker/login-action@v3 + with: + registry: docker.io + username: ${{ vars.DOCKER_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitLab Container Registry + if: ${{ (vars.GITLAB_USERNAME != '') && (env.GITLAB_TOKEN != '') }} + uses: docker/login-action@v3 + with: + registry: registry.gitlab.com + username: ${{ vars.GITLAB_USERNAME }} + password: ${{ secrets.GITLAB_TOKEN }} + + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Extract metadata (labels, annotations) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{needs.define-variables.outputs.images}} + # default labels & annotations: https://github.com/docker/metadata-action/blob/master/src/meta.ts#L509 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index + + # These steps work around docker mount caches not being cached between runs in CI providers. + # It manually injects the mounts into Docker. + # We use + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: v0-rust-${{matrix.platform}}-${{matrix.target_cpu}} + id: rust-cache + + - name: Inject cache into Docker + uses: reproducible-containers/buildkit-cache-dance@v3.1.2 + with: + cache-map: | + { + "/home/runner/.cargo/registry": "/usr/local/cargo/registry", + "/home/runner/.cargo/git/db": "/usr/local/cargo/git/db", + "./target": "/app/target" + } + # skip-extraction: ${{ steps.rust-cache.outputs.cache-hit }} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + # It will not push images generated from a pull request + - name: Get short git commit SHA + id: sha + run: | + calculatedSha=$(git rev-parse --short ${{ github.sha }}) + echo "COMMIT_SHORT_SHA=$calculatedSha" >> $GITHUB_ENV + - name: Get Git commit timestamps + run: echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV + - name: Build and push Docker image by digest + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: "Containerfile" + build-args: | + ${{ matrix.target_cpu != 'base' && format('TARGET_CPU={0}', matrix.target_cpu) || '' }} + CONDUWUIT_VERSION_EXTRA=${{ env.COMMIT_SHORT_SHA }} + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} + cache-from: type=gha + cache-to: type=gha,mode=max + sbom: true + outputs: type=image,"name=${{ needs.define-variables.outputs.images_list }}",push-by-digest=true,name-canonical=true,push=true + env: + SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }} + + # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[Using artifact attestations to establish provenance for builds](https://docs.github.com/en/actions/security-for-github-actions/using-artifact-attestations/using-artifact-attestations-to-establish-provenance-for-builds)." + - name: Generate artifact attestation (${{ needs.define-variables.outputs.ghcr_image}}) + uses: actions/attest-build-provenance@v2 + if: env.GHCR_ENABLED == 'true' + with: + subject-name: ${{ needs.define-variables.outputs.ghcr_image}} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true + - name: Generate artifact attestation (index.${{ needs.define-variables.outputs.docker_image}}) + uses: actions/attest-build-provenance@v2 + if: (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') + with: + # see action readme (we need to use index.docker.io rather than docker.io) + subject-name: index.${{ needs.define-variables.outputs.docker_image}} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true + - name: Generate artifact attestation (${{ needs.define-variables.outputs.glhr_image}}) + uses: actions/attest-build-provenance@v2 + if: (vars.GITLAB_USERNAME != '') && (env.GITLAB_TOKEN != '') + with: + subject-name: ${{ needs.define-variables.outputs.glhr_image}} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true + + # For publishing multi-platform manifests + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: ${{matrix.tag_suffix || ''}}digests-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + runs-on: ubuntu-latest + needs: [define-variables, build-image] + strategy: + matrix: ${{ fromJSON(needs.define-variables.outputs.merge_matrix) }} + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: ${{matrix.tag_suffix || ''}}digests-* + merge-multiple: true + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to Docker Hub + if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} + uses: docker/login-action@v3 + with: + registry: docker.io + username: ${{ vars.DOCKER_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitLab Container Registry + if: ${{ (vars.GITLAB_USERNAME != '') && (env.GITLAB_TOKEN != '') }} + uses: docker/login-action@v3 + with: + registry: registry.gitlab.com + username: ${{ vars.GITLAB_USERNAME }} + password: ${{ secrets.GITLAB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Extract metadata (tags) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + flavor: | + suffix=-tiny${{ matrix.tag_suffix }} + tags: | + type=semver,pattern=v{{version}} + type=semver,pattern=v{{major}}.{{minor}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.0.') }} + type=semver,pattern=v{{major}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.') }} + type=ref,event=branch + type=ref,event=pr + type=sha,format=long,prefix={{branch}}- + images: ${{needs.define-variables.outputs.images}} + # default labels & annotations: https://github.com/docker/metadata-action/blob/master/src/meta.ts#L509 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: index + + - name: Create manifest list and push + working-directory: /tmp/digests + env: + IMAGES: ${{needs.define-variables.outputs.images}} + run: | + IFS=$'\n' + IMAGES_LIST=($IMAGES) + ANNOTATIONS_LIST=($DOCKER_METADATA_OUTPUT_ANNOTATIONS) + TAGS_LIST=($DOCKER_METADATA_OUTPUT_TAGS) + for REPO in "${IMAGES_LIST[@]}"; do + docker buildx imagetools create \ + $(for tag in "${TAGS_LIST[@]}"; do echo "--tag"; echo "$tag"; done) \ + $(for annotation in "${ANNOTATIONS_LIST[@]}"; do echo "--annotation"; echo "$annotation"; done) \ + $(for reference in *; do printf "$REPO@sha256:%s\n" $reference; done) + done + + - name: Inspect image + env: + IMAGES: ${{needs.define-variables.outputs.images}} + run: | + IMAGES_LIST=($IMAGES) + for REPO in "${IMAGES_LIST[@]}"; do + docker buildx imagetools inspect $REPO:${{ steps.meta.outputs.version }} + done \ No newline at end of file diff --git a/Containerfile b/Containerfile new file mode 100644 index 000000000..5d99e2502 --- /dev/null +++ b/Containerfile @@ -0,0 +1,142 @@ + +ARG BUILDPLATFORM + +FROM --platform=$BUILDPLATFORM rust:latest AS builder + +ARG TARGETPLATFORM +# Determine target platform identifiers +RUN RUST_TARGET=$(case $TARGETPLATFORM in \ + "linux/amd64") echo x86_64-unknown-linux-gnu ;; \ + "linux/arm64") echo aarch64-unknown-linux-gnu ;; \ + *) echo "Unsupported target platform $TARGETPLATFORM" 1>&2; exit 1 ;; \ + esac) && \ + echo "RUST_TARGET=$RUST_TARGET" >> /etc/environment + +RUN DEB_ARCH=$(case $TARGETPLATFORM in \ + "linux/amd64") echo amd64 ;; \ + "linux/arm64") echo arm64 ;; \ + *) exit 1 ;; \ + esac) && \ + echo "DEB_ARCH=$DEB_ARCH" >> /etc/environment + +RUN GCC_TARGET=$(case $TARGETPLATFORM in \ + "linux/amd64") echo x86-64-linux-gnu ;; \ + "linux/arm64") echo aarch64-linux-gnu ;; \ + *) exit 1 ;; \ + esac) && \ + echo "GCC_TARGET=$GCC_TARGET" >> /etc/environment + +# install build-time deps +RUN set -o allexport && \ + . /etc/environment && \ + dpkg --add-architecture $DEB_ARCH && \ + apt-get update && apt-get install -y --no-install-recommends \ + lld \ + gcc-$GCC_TARGET g++-$GCC_TARGET \ + libc6-dev-$DEB_ARCH-cross \ + libclang-dev liburing-dev \ + liburing-dev:$DEB_ARCH && \ + rm -rf /var/lib/apt/lists/* + +# set linkers, compilers and pkg-config libdir +RUN VARS=$(case $TARGETPLATFORM in \ + "linux/amd64") \ + echo "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc" && \ + echo "CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc" && \ + echo "CXX_x86_64_unknown_linux_gnu=\"x86_64-linux-gnu-g++\"" && \ + echo "PKG_CONFIG_LIBDIR=/usr/lib/x86_64-linux-gnu/pkgconfig" \ + ;; \ + "linux/arm64") \ + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" && \ + echo "CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc" && \ + echo "CXX_aarch64_unknown_linux_gnu=\"aarch64-linux-gnu-g++\"" && \ + echo "PKG_CONFIG_LIBDIR=/usr/lib/aarch64-linux-gnu/pkgconfig" \ + ;; \ + *) exit 1 ;; \ + esac) && \ + echo "$VARS" >> /etc/environment + +# enable cross-platform linking of libraries +RUN echo "PKG_CONFIG_ALLOW_CROSS=true" >> /etc/environment + + +ARG CONDUWUIT_VERSION_EXTRA= +ENV CONDUWUIT_VERSION_EXTRA=$CONDUWUIT_VERSION_EXTRA +# Set up Rust toolchain +WORKDIR /app +COPY ./rust-toolchain.toml . +RUN rustc --version +RUN set -o allexport && \ + . /etc/environment && \ + rustup target add $RUST_TARGET + +# Developer tool versions +# renovate: datasource=github-releases depName=cargo-binstall packageName=cargo-bins/cargo-binstall +ENV BINSTALL_VERSION=1.10.17 +# renovate: github-releases depName=cargo-sbom packageName=psastras/sbom-rs +ENV CARGO_SBOM_VERSION=0.9.1 + +RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash +RUN cargo binstall --no-confirm cargo-sbom --version $CARGO_SBOM_VERSION + +# Get source +COPY . . + +# Build binary +# We disable incremental compilation to save disk space, as it only produces a minimal speedup for this case. +ENV CARGO_INCREMENTAL=0 + +ARG TARGET_CPU= +RUN if [ -n "${TARGET_CPU}" ]; then \ + echo "CFLAGS='-march=${TARGET_CPU}'" >> /etc/environment && \ + echo "CXXFLAGS='-march=${TARGET_CPU}'" >> /etc/environment && \ + echo "RUSTFLAGS='-C target-cpu=${TARGET_CPU}'" >> /etc/environment; \ +fi + +RUN cat /etc/environment +RUN mkdir /out +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + --mount=type=cache,target=/app/target \ + set -o allexport && \ + . /etc/environment && \ + cargo build --locked --release --target $RUST_TARGET && \ + cp ./target/$RUST_TARGET/release/conduwuit /out/app + +RUN cargo sbom > /out/sbom.spdx.json + +# find dynamically linked dependencies +RUN mkdir /out/libs \ + && ldd /out/app | grep '=>' | awk '{print $(NF-1)}' | xargs -I {} cp {} /out/libs/ +# libraries with a hardcoded path, like ld +# (see for example https://github.com/vlang/v/issues/8682) +# Excluding linux-vdso.so, as it is a part of the kernel +RUN mkdir /out/libs-root \ + && ldd /out/app | grep -v '=>' | grep -v 'linux-vdso.so' | awk '{print $(NF-1)}' | xargs -I {} install -D {} /out/libs-root{} +# RUN ldd /out/app +# ldd /out/app | grep -v 'linux-vdso.so' | awk '{print $(NF-1)}' +# RUN ls /libs + +FROM scratch + +WORKDIR / + +# Copy root certs for tls into image +# You can also mount the certs from the host +# --volume /etc/ssl/certs:/etc/ssl/certs:ro +COPY --from=rust:latest /etc/ssl/certs /etc/ssl/certs + +# Copy our build +COPY --from=builder /out/app ./app +# Copy SBOM +COPY --from=builder /out/sbom.spdx.json ./sbom.spdx.json + +# Copy hardcoded dynamic libraries +COPY --from=builder /out/libs-root / +# Copy dynamic libraries +COPY --from=builder /out/libs /libs +# Tell Linux where to find our libraries +ENV LD_LIBRARY_PATH=/libs + + +CMD ["/app"]