diff --git a/.github/actions/dev-tools/action.yml b/.github/actions/dev-tools/action.yml new file mode 100644 index 00000000000..de260af98d6 --- /dev/null +++ b/.github/actions/dev-tools/action.yml @@ -0,0 +1,23 @@ +name: "Dev-tools install" + +description: "This action installs dev-tools, with caching." + +runs: + using: composite + steps: + - name: Environment setup + shell: bash + run: | + echo "GO_VERSION=$(go version) >> "$GITHUB_ENV" + echo "UNAME=$(uname)" >> "$GITHB_ENV" + - name: Enable cache + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + id: cache + with: + path: ~/go/bin + key: dev-tools-${{ env.UNAME }}-${{ env.GO_VERSION }}-${{ hashFiles('Makefile') }} + - if: ${{ steps.cache.outputs.cache-hit != 'true' }} + name: Install dev tools + shell: bash + continue-on-error: false + run: make install-dev-tools diff --git a/.github/actions/install-go/action.yml b/.github/actions/install-go/action.yml new file mode 100644 index 00000000000..d8af38c2cae --- /dev/null +++ b/.github/actions/install-go/action.yml @@ -0,0 +1,35 @@ +name: "Go install" + +description: "This action installs go, with caching. + By default, the latest stable release is used. + You can also pass an explicit version: + - `1.23` will install the latest stable patch release for that branch + - `1.22.2` will install that specific version + - `canary` will install the latest alpha/beta/RC" + +inputs: + version: + default: "" + description: "" + +runs: + using: composite + steps: + - name: Environment setup + shell: bash + run: | + export GO_VERSION=${{ inputs.version }} + echo "GO_VERSION=$(make install-go-resolve-version)" >> "$GITHUB_ENV" + echo "PATH=/opt/go/bin:$HOME/go/bin:$PATH" >> "$GITHUB_ENV" + echo "UNAME=$(uname)" >> "$GITHB_ENV" + - name: Enable cache + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + id: cache + with: + path: /opt/go + key: golang-${{ env.UNAME }}-${{ env.GO_VERSION }} + - if: ${{ steps.cache.outputs.cache-hit != 'true' }} + name: Install go + shell: bash + continue-on-error: false + run: GO_INSTALL_DESTINATION=/opt make install-go diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index afb24cd4516..bab3d7cee26 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,6 +11,7 @@ env: GO_VERSION: 1.24 GOTOOLCHAIN: local GO_INSTALL_DESTINATION: /opt + GOOS: linux jobs: go_current: @@ -18,32 +19,14 @@ jobs: name: "go stable" runs-on: ubuntu-24.04 steps: - # First, setup: clone, install go (with cache), install dev tools (with cache) + # Clone - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 100 - - name: Environment setup - run: | - echo "GO_VERSION=$(make install-go-resolve-version)" >> "$GITHUB_ENV" - echo "PATH=$GO_INSTALL_DESTINATION/go/bin:$HOME/go/bin:$PATH" >> "$GITHUB_ENV" - - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 - id: cache-go - with: - path: /opt/go - key: golang-${{ env.GO_VERSION }} - - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 - id: cache-go-dev - with: - path: ~/go/bin - key: golang-dev-${{ env.GO_VERSION }}-${{ hashFiles('Makefile') }} - - if: ${{ steps.cache-go.outputs.cache-hit != 'true' }} - name: Install go - continue-on-error: false - run: make install-go - - if: ${{ steps.cache-go-dev.outputs.cache-hit != 'true' }} - name: Install dev tools - continue-on-error: false - run: make install-dev-tools + # Install go + - name: "Install go" + uses: ./.github/actions/install-go + # Install dev-tools + - name: "Install dev-tools" + uses: ./.github/actions/dev-tools - name: imports run: make lint-imports @@ -63,29 +46,14 @@ jobs: env: GO_VERSION: canary steps: + # Clone - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - name: Environment setup - run: | - echo "GO_VERSION=$(make install-go-resolve-version)" >> "$GITHUB_ENV" - echo "PATH=$GO_INSTALL_DESTINATION/go/bin:$HOME/go/bin:$PATH" >> "$GITHUB_ENV" - - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 - id: cache-go - with: - path: /opt/go - key: golang-${{ env.GO_VERSION }} - - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 - id: cache-go-dev - with: - path: ~/go/bin - key: golang-dev-${{ env.GO_VERSION }}-${{ hashFiles('Makefile') }} - - if: ${{ steps.cache-go.outputs.cache-hit != 'true' }} - name: Install go - run: make install-go - - if: ${{ steps.cache-go-dev.outputs.cache-hit != 'true' }} - name: Install dev tools - run: make install-dev-tools + # Install go + - name: "Install go" + uses: ./.github/actions/install-go + # Install dev-tools + - name: "Install dev-tools" + uses: ./.github/actions/dev-tools - name: imports run: make lint-imports @@ -100,8 +68,6 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - name: make lint-yaml run: make lint-yaml - name: make lint-shell diff --git a/.github/workflows/test-canary.yml b/.github/workflows/test-canary.yml index f35fed4d904..31c542399fa 100644 --- a/.github/workflows/test-canary.yml +++ b/.github/workflows/test-canary.yml @@ -11,21 +11,25 @@ on: - '**.md' env: - UBUNTU_VERSION: "24.04" + GO_VERSION: canary + GOTOOLCHAIN: local + GO_INSTALL_DESTINATION: /opt + GOOS: linux GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: linux: - runs-on: "ubuntu-24.04" + runs-on: ubuntu-24.04 timeout-minutes: 40 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - name: "Prepare integration test environment" + - name: "Setup environment" run: | + # Get latest canary go version + export GO_VERSION=$(make install-go-resolve-version) + # Build integration image with updated dependencies . ./hack/build-integration-canary.sh - canary::build::integration + LOG_LEVEL=info canary::build::integration - name: "Remove snap loopback devices (conflicts with our loopback devices in TestRunDevice)" run: | sudo systemctl disable --now snapd.service snapd.socket @@ -41,6 +45,7 @@ jobs: docker run --privileged --rm tonistiigi/binfmt --install linux/amd64 docker run --privileged --rm tonistiigi/binfmt --install linux/arm64 docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7 + - name: "Run unit tests" run: go test -v ./pkg/... - name: "Run integration tests" @@ -50,48 +55,56 @@ jobs: windows: timeout-minutes: 40 - runs-on: windows-latest + runs-on: windows-2022 + env: + GOOS: windows defaults: run: shell: bash steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: "Clone" + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: "Setup go environment" + run: | + echo "GO_VERSION=$(make install-go-resolve-version)" >> "$GITHUB_ENV" + echo "PATH=$GO_INSTALL_DESTINATION/go/bin:$HOME/go/bin:$PATH" >> "$GITHUB_ENV" + - name: "Cache go" + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + id: cache-go + with: + path: /opt/go + key: golang-${{ env.GOOS }}-${{ env.GO_VERSION }} + - if: ${{ steps.cache-go.outputs.cache-hit != 'true' }} + name: "Setup go" + continue-on-error: false + run: make install-go + - name: "Cache dev-tools" + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + id: cache-go-dev with: - fetch-depth: 1 - - name: Set GO env + path: ~/go/bin + key: golang-dev-${{ env.GOOS }}-${{ env.GO_VERSION }}-${{ hashFiles('Makefile') }} + - if: ${{ steps.cache-go-dev.outputs.cache-hit != 'true' }} + name: "Setup dev tools" + continue-on-error: false + run: make install-dev-tools + - name: "Setup CNI & containerd" run: | - # Get latest containerd + # CNI install script has not been modified in 3+ years. + # There is little to no reason to update this to latest containerd + curl -fsSLO https://raw.githubusercontent.com/containerd/containerd/refs/tags/v1.7.25/script/setup/install-cni-windows + GOPATH=$(go env GOPATH) ./install-cni-windows + # Setup nerdctl + go install ./cmd/nerdctl + # Get latest tagged containerd args=(curl --proto '=https' --tlsv1.2 -fsSL -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28") [ "${GITHUB_TOKEN:-}" == "" ] && { - >&2 printf "GITHUB_TOKEN is not set - you might face rate limitations with the Github API\n" + echo "GITHUB_TOKEN is not set - you might face rate limitations with the Github API" } || args+=(-H "Authorization: Bearer $GITHUB_TOKEN") ctd_v="$("${args[@]}" https://api.github.com/repos/containerd/containerd/tags | jq -rc .[0].name)" - echo "CONTAINERD_VERSION=${ctd_v:1}" >> "$GITHUB_ENV" - - . ./hack/build-integration-canary.sh - canary::golang::latest - - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 - with: - go-version: ${{ env.GO_VERSION }} - check-latest: true - - run: go install ./cmd/nerdctl - - run: make install-dev-tools - # This here is solely to get the cni install script, which has not been modified in 3+ years. - # There is little to no reason to update this to latest containerd - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: containerd/containerd - ref: "v1.7.25" - path: containerd - fetch-depth: 1 - - name: "Set up CNI" - working-directory: containerd - run: GOPATH=$(go env GOPATH) script/setup/install-cni-windows - # Windows setup script can only use released versions - - name: "Set up containerd" - env: - ctrdVersion: ${{ env.CONTAINERD_VERSION }} - run: powershell hack/configure-windows-ci.ps1 + ctrdVersion="${ctd_v:1}" powershell hack/configure-windows-ci.ps1 + - name: "Run unit tests" + run: go test -v ./pkg/... - name: "Run integration tests" run: ./hack/test-integration.sh -test.only-flaky=false - name: "Run integration tests (flaky)" diff --git a/.github/workflows/test-kube.yml b/.github/workflows/test-kube.yml index 580a9a2181a..577a015ccc4 100644 --- a/.github/workflows/test-kube.yml +++ b/.github/workflows/test-kube.yml @@ -12,14 +12,12 @@ on: jobs: linux: - runs-on: "ubuntu-24.04" + runs-on: ubuntu-24.04 timeout-minutes: 40 env: ROOTFUL: true steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - name: "Run Kubernetes integration tests" # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b5313b778a3..f4a4ffa0366 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,8 +39,6 @@ jobs: ARCH: "${{ matrix.arch }}" steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - name: "Expose GitHub Runtime variables for gha" uses: crazy-max/ghaction-github-runtime@b3a9207c0e1ef41f4cf215303c976869d0c2c1c4 # v3.0.0 - name: "Build dependencies for the integration test environment image" @@ -72,8 +70,6 @@ jobs: goos: linux steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: go-version: ${{ env.GO_VERSION }} @@ -119,8 +115,6 @@ jobs: UBUNTU_VERSION: "${{ matrix.ubuntu }}" steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - name: "Expose GitHub Runtime variables for gha" uses: crazy-max/ghaction-github-runtime@b3a9207c0e1ef41f4cf215303c976869d0c2c1c4 # v3.0.0 - name: "Prepare integration test environment" @@ -168,8 +162,6 @@ jobs: UBUNTU_VERSION: "${{ matrix.ubuntu }}" steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - name: Enable ipv4 and ipv6 forwarding run: | sudo sysctl -w net.ipv6.conf.all.forwarding=1 @@ -267,8 +259,6 @@ jobs: EOT sudo systemctl restart apparmor.service - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - name: "Register QEMU (tonistiigi/binfmt)" run: | # `--install all` will only install emulation for architectures that cannot be natively executed @@ -310,8 +300,6 @@ jobs: go-version: ["1.23.x", "1.24.x"] steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: go-version: ${{ matrix.go-version }} @@ -325,8 +313,6 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: go-version: ${{ env.GO_VERSION }} @@ -361,8 +347,6 @@ jobs: shell: bash steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: go-version: ${{ env.GO_VERSION }} @@ -394,8 +378,6 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 with: path: /root/.vagrant.d diff --git a/Makefile b/Makefile index 4be567fd717..2dca0759b8e 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ ORG_PREFIXES := "github.com/containerd" DOCKER ?= docker GO ?= go -GOOS ?= $(shell $(GO) env GOOS) +GOOS ?= $(shell $(GO) env GOOS 2>/dev/null || true) ifeq ($(GOOS),windows) BIN_EXT := .exe endif @@ -55,13 +55,16 @@ ARCH := amd64 ifeq ($(shell uname -m), aarch64) ARCH = arm64 endif -OS := linux +OS := windows ifeq ($(shell uname -s), Darwin) OS = darwin endif ifeq ($(shell uname -s), FreeBSD) OS = freebsd endif +ifeq ($(shell uname -s), Linux) + OS = linux +endif ifdef VERBOSE VERBOSE_FLAG := -v @@ -270,11 +273,13 @@ GO_INSTALL_DESTINATION ?= /opt/$(BINARY)-dev-tools install-go: $(call title, $@) @mkdir -p $(GO_INSTALL_DESTINATION) - @if [ ! -e $(GO_INSTALL_DESTINATION)/go ]; then curl -fsSL --proto '=https' --tlsv1.2 https://go.dev/dl/$(shell curl -fsSL --proto "=https" --tlsv1.3 "https://go.dev/dl/?mode=json&include=all" | \ - jq -rc 'map(select($(GO_VERSION_SELECTOR)))[0].files | map(select(.os=="$(OS)" and .arch=="$(ARCH)"))[0].filename') | \ - tar xzC $(GO_INSTALL_DESTINATION) || exit; \ + @if [ ! -e $(GO_INSTALL_DESTINATION)/go ]; then cd $(GO_INSTALL_DESTINATION); \ + curl -o go.archive -fsSL --proto '=https' --tlsv1.2 https://go.dev/dl/$(shell \ + curl -fsSL --proto "=https" --tlsv1.2 "https://go.dev/dl/?mode=json&include=all" | \ + jq -rc 'map(select($(GO_VERSION_SELECTOR)))[0].files | map(select(.os=="$(OS)" and .arch=="$(ARCH)"))[0].filename'); \ + [ "$(OS)" = windows ] && unzip go.archive >/dev/null || tar xzf go.archive; \ else \ - echo "Install already detected, doing nothing."; \ + echo "Install already detected in $(GO_INSTALL_DESTINATION), doing nothing."; \ fi @echo Remember to add to your profile: export PATH="$(GO_INSTALL_DESTINATION)/go/bin:\$$HOME/go/bin:\$$PATH" $(call footer, $@) diff --git a/hack/build-integration-canary.sh b/hack/build-integration-canary.sh index 0396ada100e..0a471be9e15 100755 --- a/hack/build-integration-canary.sh +++ b/hack/build-integration-canary.sh @@ -28,7 +28,8 @@ readonly root # "Blacklisting" here means that any dependency which name is blacklisted will be left untouched, at the version # currently pinned in the Dockerfile. # This is convenient so that currently broken alpha/beta/RC can be held back temporarily to keep the build green -blacklist=() +blacklist=( +) # List all the repositories we depend on to build and run integration tests dependencies=( @@ -162,13 +163,18 @@ latest::release(){ [ ! "$ignore" ] || ! grep -q "$ignore" <<<"$line" || continue name="$(echo "$line" | jq -rc .name)" if [ "$name" == "" ] || [ "$name" == null ] ; then - log::debug " > bogus release name ($name) ignored" - continue + # Kubo latest releases are NOT setting a release name anymore :/ + # Hail mary on .tag_name... + name="$(echo "$line" | jq -rc .tag_name)" + if [ "$name" == "" ] || [ "$name" == null ] ; then + log::debug " > bogus release name: $line" + continue + fi fi log::debug " > found release: $name" - if version::compare <(echo "$line" | jq -rc .name); then + if version::compare <(echo "$name"); then higher_data="$line" - higher_readable="$(echo "$line" | jq -rc .name | sed -E 's/(.*[ ])?(v?[0-9][0-9.a-z-]+).*/\2/')" + higher_readable="$(echo "$name" | sed -E 's/(.*[ ])?(v?[0-9][0-9.a-z-]+).*/\2/')" fi done < <(github::releases "$repo") @@ -211,7 +217,6 @@ assets::get(){ ###################### # Script ###################### - canary::build::integration(){ docker_args=(docker build -t test-integration --target test-integration) @@ -291,58 +296,8 @@ canary::build::integration(){ docker_args+=(--build-arg "${shortsafename}_VERSION=$higher_readable") done - hub_available_go_version="$(canary::golang::hublatest)" - if [ "$hub_available_go_version" != "" ]; then - docker_args+=(--build-arg "GO_VERSION=$hub_available_go_version") - fi + docker_args+=(--build-arg "GO_VERSION=$GO_VERSION") log::debug "${docker_args[*]} ." "${docker_args[@]}" "." } - -# Hub usually has a delay before available golang version show-up. This method will find the latest available one. -# See -# - https://github.com/containerd/nerdctl/issues/3224 -# - https://github.com/containerd/nerdctl/issues/3306 -canary::golang::hublatest(){ - local hub_tags - local go_version - local available_version="" - local index - - hub_tags="$(http::get /dev/stdout "https://registry-1.docker.io/v2/library/golang/tags/list" -H "Authorization: Bearer $(http::get /dev/stdout "https://auth.docker.io/token?service=registry.docker.io&scope=repository%3Alibrary%2Fgolang%3Apull" | jq -rc .access_token)")" - - index=0 - while [ "$available_version" == "" ] && [ "$index" -lt 5 ]; do - go_version="$(http::get /dev/stdout "https://go.dev/dl/?mode=json&include=all" | jq -rc .[$index].version)" - go_version="${go_version##*go}" - available_version="$(printf "%s" "$hub_tags" | jq -rc ".tags[] | select(.==\"$go_version\")")" - ((index++)) - done || true - - printf "%s" "$available_version" -} - -canary::golang::latest(){ - # Enable extended globbing features to use advanced pattern matching - shopt -s extglob - - # Get latest golang version and split it in components - norm=() - while read -r line; do - line_trimmed="${line//+([[:space:]])/}" - norm+=("$line_trimmed") - done < \ - <(sed -E 's/^go([0-9]+)[.]([0-9]+)([.]([0-9]+))?(([a-z]+)([0-9]+))?/\1.\2\n\4\n\6\n\7/i' \ - <(curl -fsSL "https://go.dev/dl/?mode=json&include=all" | jq -rc .[0].version) \ - ) - - # Serialize version, making sure we have a patch version, and separate possible rcX into .rc-X - [ "${norm[1]}" != "" ] || norm[1]="0" - norm[1]=".${norm[1]}" - [ "${norm[2]}" == "" ] || norm[2]="-${norm[2]}" - [ "${norm[3]}" == "" ] || norm[3]=".${norm[3]}" - # Save it - IFS= - echo "GO_VERSION=${norm[*]}" >> "$GITHUB_ENV" -} diff --git a/mod/tigron/.golangci.yml b/mod/tigron/.golangci.yml index 66e0b8d2ade..dc5e2c92b54 100644 --- a/mod/tigron/.golangci.yml +++ b/mod/tigron/.golangci.yml @@ -52,3 +52,4 @@ linters-settings: - "golang.org/x/term" - "gotest.tools" - "github.com/creack/pty" + - "github.com/containerd/nerdctl/mod/tigron"