Revamp testing strategy

This:
- slightly simplifies the Dockerfile
- add a ./test-integration.sh script that will additionally provide information about slow tests
- revamp lint testing on the CI (enable lint for non-linux)
- removes nick retries entirely
- separate flaky and non flaky runs in different steps

Signed-off-by: apostasie <[email protected]>

Author: apostasie

.github/workflows/test-canary.yml

@@ -65,7 +65,7 @@ jobs:
       - name: "Run unit tests"
         run: go test -v ./pkg/...
       - name: "Run integration tests"
-        run: docker run -t --rm --privileged test-integration
+        run: docker run -t --rm --privileged test-integration ./hack/test-integration.sh
 
   windows:
     runs-on: windows-latest

.github/workflows/test.yml

@@ -11,13 +11,32 @@ on:
 
 env:
   GO_VERSION: 1.23.x
+  SHORT_TIMEOUT: 5
+  LONG_TIMEOUT: 60
 
 jobs:
-  lint:
-    runs-on: ubuntu-24.04
-    timeout-minutes: 20
+  lint-go:
+    # Supposed to work: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/evaluate-expressions-in-workflows-and-actions#example-returning-a-json-data-type
+    # Apparently does not
+    # timeout-minutes: ${{ fromJSON(env.SHORT_TIMEOUT) }}
+    timeout-minutes: 5
+    name: lint-go ${{ matrix.goos }}
+    runs-on: "${{ matrix.os }}"
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-24.04
+            goos: linux
+          - os: ubuntu-24.04
+            goos: freebsd
+          # FIXME: this is currently failing in a non-sensical way, so, running on linux instead...
+          # - os: windows-2022
+          - os: ubuntu-24.04
+            goos: windows
+    env:
+      GOOS: "${{ matrix.goos }}"
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - uses: actions/setup-go@v5
@@ -26,24 +45,45 @@ jobs:
           check-latest: true
           cache: true
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v6.1.1
+        uses: golangci/golangci-lint-action@v6
         with:
-          version: v1.60.1
           args: --verbose
-      - name: yamllint-lint
+
+  lint-other:
+    timeout-minutes: 5
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          check-latest: true
+          cache: true
+      - name: yaml
         run: make lint-yaml
-      - name: shellcheck
+      - name: shell
         run: make lint-shell
       - name: go imports ordering
         run: |
           go install -v github.com/incu6us/goimports-reviser/v3@latest
           make lint-imports
 
   test-unit:
-    runs-on: ubuntu-24.04
-    timeout-minutes: 20
+    timeout-minutes: 5
+    name: unit ${{ matrix.goos }}
+    runs-on: "${{ matrix.os }}"
+    strategy:
+      matrix:
+        include:
+          # FIXME: currently disabled as a lot more work is required to make these tests pass on windows
+          # - os: windows-2022
+          #  goos: windows
+          - os: ubuntu-24.04
+            goos: linux
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - uses: actions/setup-go@v5
@@ -52,11 +92,12 @@ jobs:
           check-latest: true
           cache: true
       - name: "Run unit tests"
-        run: go test -v ./pkg/...
+        run: make test-unit
 
   test-integration:
+    timeout-minutes: 60
+    name: integration ${{ matrix.containerd }} ${{ matrix.runner }}
     runs-on: "${{ matrix.runner }}"
-    timeout-minutes: 40
     strategy:
       fail-fast: false
       matrix:
@@ -78,7 +119,7 @@ jobs:
       UBUNTU_VERSION: "${{ matrix.ubuntu }}"
       CONTAINERD_VERSION: "${{ matrix.containerd }}"
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - name: "Prepare integration test environment"
@@ -99,16 +140,16 @@ jobs:
           docker run --privileged --rm tonistiigi/binfmt --install linux/arm64
           docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7
       - name: "Run integration tests"
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 30
-          max_attempts: 2
-          retry_on: error
-          command: docker run -t --rm --privileged test-integration
+        run: |
+          docker run -t --rm --privileged test-integration ./hack/test-integration.sh
+      - name: "Run integration tests (flaky)"
+        run: |
+          docker run -t --rm --privileged test-integration ./hack/test-integration.sh -test.only-flaky
 
   test-integration-ipv6:
+    timeout-minutes: 60
+    name: ipv6 ${{ matrix.containerd }} ${{ matrix.ubuntu }}
     runs-on: "ubuntu-${{ matrix.ubuntu }}"
-    timeout-minutes: 40
     strategy:
       fail-fast: false
       matrix:
@@ -120,7 +161,7 @@ jobs:
       UBUNTU_VERSION: "${{ matrix.ubuntu }}"
       CONTAINERD_VERSION: "${{ matrix.containerd }}"
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - name: Enable ipv4 and ipv6 forwarding
@@ -133,7 +174,7 @@ jobs:
           echo '{"ipv6": true, "fixed-cidr-v6": "2001:db8:1::/64", "experimental": true, "ip6tables": true}' | sudo tee /etc/docker/daemon.json
           sudo systemctl restart docker
       - name: "Prepare integration test environment"
-        run: docker build -t test-integration-ipv6 --target test-integration-ipv6 --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} .
+        run: docker build -t test-integration --target test-integration --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} .
       - name: "Remove snap loopback devices (conflicts with our loopback devices in TestRunDevice)"
         run: |
           sudo systemctl disable --now snapd.service snapd.socket
@@ -151,20 +192,16 @@ jobs:
           docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7
       - name: "Run integration tests"
         # The nested IPv6 network inside docker and qemu is complex and needs a bunch of sysctl config.
-        # Therefore it's hard to debug why the IPv6 tests fail in such an isolation layer.
+        # Therefore, it's hard to debug why the IPv6 tests fail in such an isolation layer.
         # On the other side, using the host network is easier at configuration.
         # Besides, each job is running on a different instance, which means using host network here
         # is safe and has no side effects on others.
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 30
-          max_attempts: 2
-          retry_on: error
-          command: docker run --network host -t --rm --privileged test-integration-ipv6
+        run: docker run --network host -t --rm --privileged test-integration ./hack/test-integration.sh -test.only-ipv6
 
   test-integration-rootless:
-    runs-on: "ubuntu-${{ matrix.ubuntu }}"
     timeout-minutes: 60
+    name: rootless ${{ matrix.containerd }} ${{ matrix.rootlesskit }} ${{ matrix.ubuntu }} ${{ matrix.target }}
+    runs-on: "ubuntu-${{ matrix.ubuntu }}"
     strategy:
       fail-fast: false
       matrix:
@@ -207,7 +244,7 @@ jobs:
           }
           EOT
           sudo systemctl restart apparmor.service
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - name: "Register QEMU (tonistiigi/binfmt)"
@@ -230,21 +267,18 @@ jobs:
           fi
           echo "WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622}" >> "$GITHUB_ENV"
       - name: "Test (network driver=slirp4netns, port driver=builtin)"
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 30
-          max_attempts: 2
-          retry_on: error
-          command: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} ${TEST_TARGET}
+        run: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} ${TEST_TARGET} /test-integration-rootless.sh ./hack/test-integration.sh
+      - name: "Test (network driver=slirp4netns, port driver=builtin) (flaky)"
+        run: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} ${TEST_TARGET} /test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky
 
   cross:
+    timeout-minutes: 5
     runs-on: ubuntu-24.04
-    timeout-minutes: 40
     strategy:
       matrix:
         go-version: ["1.22.x", "1.23.x"]
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - uses: actions/setup-go@v5
@@ -256,10 +290,10 @@ jobs:
         run: GO_VERSION="$(echo ${{ matrix.go-version }} | sed -e s/.x//)" make binaries
 
   test-integration-docker-compatibility:
+    timeout-minutes: 60
     runs-on: ubuntu-24.04
-    timeout-minutes: 45
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - uses: actions/setup-go@v5
@@ -284,31 +318,22 @@ jobs:
       - name: "Prepare integration test environment"
         run: |
           sudo apt-get install -y expect
+          go install -v gotest.tools/gotestsum@v1
       - name: "Ensure that the integration test suite is compatible with Docker"
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 30
-          max_attempts: 2
-          retry_on: error
-          # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
-          command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon
+        run: ./hack/test-integration.sh -test.target=docker
       - name: "Ensure that the IPv6 integration test suite is compatible with Docker"
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 30
-          max_attempts: 2
-          retry_on: error
-          # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
-          command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon -test.only-ipv6
+        run: ./hack/test-integration.sh -test.target=docker -test.only-ipv6
+      - name: "Ensure that the integration test suite is compatible with Docker (flaky only)"
+        run: ./hack/test-integration.sh -test.target=docker -test.only-flaky
 
   test-integration-windows:
+    timeout-minutes: 60
     runs-on: windows-2022
-    timeout-minutes: 30
     defaults:
       run:
         shell: bash
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - uses: actions/setup-go@v5
@@ -317,7 +342,8 @@ jobs:
           cache: true
           check-latest: true
       - run: go install ./cmd/nerdctl
-      - uses: actions/[email protected]
+      - run: go install -v gotest.tools/gotestsum@v1
+      - uses: actions/checkout@v4
         with:
           repository: containerd/containerd
           ref: v1.7.22
@@ -330,19 +356,21 @@ jobs:
         env:
           ctrdVersion: 1.7.22
         run: powershell hack/configure-windows-ci.ps1
-      # TODO: Run unit tests
       - name: "Run integration tests"
-        # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
-        run: go test -p 1 -v ./cmd/nerdctl/...
+        run: |
+          ./hack/test-integration.sh
+      - name: "Run integration tests (flaky)"
+        run: |
+          ./hack/test-integration.sh -test.only-flaky
 
   test-integration-freebsd:
+    timeout-minutes: 60
     name: FreeBSD
     # ubuntu-24.04 lacks the vagrant package
     runs-on: ubuntu-22.04
-    timeout-minutes: 20
 
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4
       - uses: actions/cache@v4
         with:
           path: /root/.vagrant.d

Dockerfile

@@ -281,7 +281,8 @@ ARG DEBIAN_FRONTEND=noninteractive
 # `expect` package contains `unbuffer(1)`, which is used for emulating TTY for testing
 RUN apt-get update -qq && apt-get install -qq --no-install-recommends \
     expect \
-    git
+    git \
+    make
 COPY --from=goversion /GOVERSION /GOVERSION
 ARG TARGETARCH
 RUN curl -fsSL --proto '=https' --tlsv1.2 https://golang.org/dl/$(cat /GOVERSION).linux-${TARGETARCH:-amd64}.tar.gz | tar xzvC /usr/local
@@ -318,8 +319,6 @@ RUN curl -o nydus-static.tgz -fsSL --proto '=https' --tlsv1.2 "https://github.co
   tar xzf nydus-static.tgz && \
   mv nydus-static/nydus-image nydus-static/nydusd nydus-static/nydusify /usr/bin/ && \
   rm nydus-static.tgz
-CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
-  "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon"]
 
 FROM test-integration AS test-integration-rootless
 # Install SSH for creating systemd user session.
@@ -342,17 +341,10 @@ RUN systemctl disable test-integration-ipfs-offline
 VOLUME /home/rootless/.local/share
 COPY ./Dockerfile.d/test-integration-rootless.sh /
 RUN chmod a+rx /test-integration-rootless.sh
-CMD ["/test-integration-rootless.sh", \
-  "gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
-  "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon"]
 
 # test for CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER=slirp4netns
 FROM test-integration-rootless AS test-integration-rootless-port-slirp4netns
 COPY ./Dockerfile.d/home_rootless_.config_systemd_user_containerd.service.d_port-slirp4netns.conf /home/rootless/.config/systemd/user/containerd.service.d/port-slirp4netns.conf
 RUN chown -R rootless:rootless /home/rootless/.config
 
-FROM test-integration AS test-integration-ipv6
-CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
-  "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon", "-test.only-ipv6"]
-
 FROM base AS demo

Makefile

@@ -86,6 +86,9 @@ lint-yaml:
 lint-shell: $(call recursive_wildcard,$(MAKEFILE_DIR)/,*.sh)
 	shellcheck -a -x $^
 
+test-unit:
+	go test -v $(MAKEFILE_DIR)/pkg/...
+
 binaries: nerdctl
 
 install:

docs/testing/README.md

@@ -58,7 +58,7 @@ explicitly allow it (with a call to `t.Parallel()`).
 
 ```bash
 docker build -t test-integration --target test-integration .
-docker run -t --rm --privileged test-integration
+docker run -t --rm --privileged test-integration ./hack/test-integration.sh
 ```
 
 ### Principles