Cleanup canary

Signed-off-by: apostasie <[email protected]>
containerd · Mar 6, 2025 · 49f3afd · 49f3afd
1 parent 3e20861
commit 49f3afd
Show file tree

Hide file tree

Showing 4 changed files with 92 additions and 94 deletions.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -11,6 +11,7 @@ env:
   GO_VERSION: 1.24
   GOTOOLCHAIN: local
   GO_INSTALL_DESTINATION: /opt
+  GOOS: linux
 
 jobs:
   go_current:
@@ -30,12 +31,12 @@ jobs:
         id: cache-go
         with:
           path: /opt/go
-          key: golang-${{ env.GO_VERSION }}
+          key: golang-${{ env.GOOS }}-${{ env.GO_VERSION }}
       - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf  # v4.2.2
         id: cache-go-dev
         with:
           path: ~/go/bin
-          key: golang-dev-${{ env.GO_VERSION }}-${{ hashFiles('Makefile') }}
+          key: golang-dev-${{ env.GOOS }}-${{ env.GO_VERSION }}-${{ hashFiles('Makefile') }}
       - if: ${{ steps.cache-go.outputs.cache-hit != 'true' }}
         name: Install go
         continue-on-error: false
@@ -74,12 +75,12 @@ jobs:
         id: cache-go
         with:
           path: /opt/go
-          key: golang-${{ env.GO_VERSION }}
+          key: golang-${{ env.GOOS }}-${{ env.GO_VERSION }}
       - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf  # v4.2.2
         id: cache-go-dev
         with:
           path: ~/go/bin
-          key: golang-dev-${{ env.GO_VERSION }}-${{ hashFiles('Makefile') }}
+          key: golang-dev-${{ env.GOOS }}-${{ env.GO_VERSION }}-${{ hashFiles('Makefile') }}
       - if: ${{ steps.cache-go.outputs.cache-hit != 'true' }}
         name: Install go
         run: make install-go

diff --git a/.github/workflows/test-canary.yml b/.github/workflows/test-canary.yml
@@ -11,21 +11,26 @@ on:
       - '**.md'
 
 env:
-  UBUNTU_VERSION: "24.04"
+  GO_VERSION: canary
+  GOTOOLCHAIN: local
+  GOOS: linux
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
   linux:
-    runs-on: "ubuntu-24.04"
+    runs-on: ubuntu-24.04
     timeout-minutes: 40
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
         with:
           fetch-depth: 1
-      - name: "Prepare integration test environment"
+      - name: "Setup environment"
         run: |
+          # Get latest canary go version
+          export GO_VERSION=$(make install-go-resolve-version)
+          # Build integration image with updated dependencies
           . ./hack/build-integration-canary.sh
-          canary::build::integration
+          LOG_LEVEL=info canary::build::integration
       - name: "Remove snap loopback devices (conflicts with our loopback devices in TestRunDevice)"
         run: |
           sudo systemctl disable --now snapd.service snapd.socket
@@ -41,24 +46,29 @@ jobs:
           docker run --privileged --rm tonistiigi/binfmt --install linux/amd64
           docker run --privileged --rm tonistiigi/binfmt --install linux/arm64
           docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7
+
       - name: "Run unit tests"
-        run: go test -v ./pkg/...
+        run: docker run -t --rm --privileged test-integration go test -v ./pkg/...
       - name: "Run integration tests"
         run: docker run -t --rm --privileged test-integration ./hack/test-integration.sh -test.only-flaky=false
       - name: "Run integration tests (flaky)"
         run: docker run -t --rm --privileged test-integration ./hack/test-integration.sh -test.only-flaky=true
 
   windows:
     timeout-minutes: 40
-    runs-on: windows-latest
+    runs-on: windows-2022
+    env:
+      GO_INSTALL_DESTINATION: /c/opt
+      GOOS: windows
     defaults:
       run:
         shell: bash
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+      - name: "Clone"
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
         with:
           fetch-depth: 1
-      - name: Set GO env
+      - name: "Setup environment"
         run: |
           # Get latest containerd
           args=(curl --proto '=https' --tlsv1.2 -fsSL -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28")
@@ -67,31 +77,58 @@ jobs:
           } || args+=(-H "Authorization: Bearer $GITHUB_TOKEN")
           ctd_v="$("${args[@]}" https://api.github.com/repos/containerd/containerd/tags | jq -rc .[0].name)"
           echo "CONTAINERD_VERSION=${ctd_v:1}" >> "$GITHUB_ENV"
-
-          . ./hack/build-integration-canary.sh
-          canary::golang::latest
-      - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34  # v5.3.0
+          # Get latest go version
+          echo "GO_VERSION=$(make install-go-resolve-version)" >> "$GITHUB_ENV"
+          echo "PATH=$GO_INSTALL_DESTINATION/go/bin:$HOME/go/bin:$PATH" >> "$GITHUB_ENV"
+      - name: "Cache go"
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf  # v4.2.2
+        id: cache-go
         with:
-          go-version: ${{ env.GO_VERSION }}
-          check-latest: true
-      - run: go install ./cmd/nerdctl
-      - run: make install-dev-tools
-      # This here is solely to get the cni install script, which has not been modified in 3+ years.
-      # There is little to no reason to update this to latest containerd
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+          path: /c/opt/go
+          key: golang-${{ env.GOOS }}-${{ env.GO_VERSION }}
+      - name: "Cache dev-tools"
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf  # v4.2.2
+        id: cache-go-dev
         with:
-          repository: containerd/containerd
-          ref: "v1.7.25"
-          path: containerd
-          fetch-depth: 1
-      - name: "Set up CNI"
-        working-directory: containerd
-        run: GOPATH=$(go env GOPATH) script/setup/install-cni-windows
+          path: ~/go/bin
+          key: golang-dev-${{ env.GOOS }}-${{ env.GO_VERSION }}-${{ hashFiles('Makefile') }}
+      - if: ${{ steps.cache-go.outputs.cache-hit != 'true' }}
+        name: "Setup go"
+        continue-on-error: false
+        run: make install-go
+      - if: ${{ steps.cache-go-dev.outputs.cache-hit != 'true' }}
+        name: "Setup dev tools"
+        continue-on-error: false
+        run: |
+          which go
+          go version
+          echo $PATH
+          uname -s
+          ls -lA /c/opt/go/bin
+          echo version:
+          /c/opt/go/bin/go version
+          chmod a+x /c/opt/go/bin/go
+          echo version above
+          /c/opt/go/bin/go version
+          echo version above 2
+          make install-dev-tools
+      - name: "Setup nerdctl"
+        run: go install ./cmd/nerdctl
+      - name: "Setup cni"
+        run: |
+          # CNI install script has not been modified in 3+ years.
+          # There is little to no reason to update this to latest containerd
+          curl -fsSLO https://raw.githubusercontent.com/containerd/containerd/refs/tags/v1.7.25/script/setup/install-cni-windows
+          GOPATH=$(go env GOPATH) ./install-cni-windows
+          go install ./cmd/nerdctl
       # Windows setup script can only use released versions
-      - name: "Set up containerd"
+      - name: "Setup containerd"
         env:
           ctrdVersion: ${{ env.CONTAINERD_VERSION }}
         run: powershell hack/configure-windows-ci.ps1
+
+      - name: "Run unit tests"
+        run: go test -v ./pkg/...
       - name: "Run integration tests"
         run: ./hack/test-integration.sh -test.only-flaky=false
       - name: "Run integration tests (flaky)"

diff --git a/Makefile b/Makefile
@@ -26,7 +26,7 @@ ORG_PREFIXES := "github.com/containerd"
 
 DOCKER ?= docker
 GO ?= go
-GOOS ?= $(shell $(GO) env GOOS)
+GOOS ?= $(shell $(GO) env GOOS 2>/dev/null || true)
 ifeq ($(GOOS),windows)
 	BIN_EXT := .exe
 endif
@@ -55,13 +55,16 @@ ARCH := amd64
 ifeq ($(shell uname -m), aarch64)
 	ARCH = arm64
 endif
-OS := linux
+OS := windows
 ifeq ($(shell uname -s), Darwin)
 	OS = darwin
 endif
 ifeq ($(shell uname -s), FreeBSD)
 	OS = freebsd
 endif
+#ifeq ($(shell uname -s), Linux)
+#	OS = linux
+#endif
 
 ifdef VERBOSE
 	VERBOSE_FLAG := -v
@@ -263,11 +266,13 @@ GO_INSTALL_DESTINATION ?= /opt/$(BINARY)-dev-tools
 install-go:
 	$(call title, $@)
 	@mkdir -p $(GO_INSTALL_DESTINATION)
-	@if [ ! -e $(GO_INSTALL_DESTINATION)/go ]; then curl -fsSL --proto '=https' --tlsv1.2 https://go.dev/dl/$(shell curl -fsSL --proto "=https" --tlsv1.3 "https://go.dev/dl/?mode=json&include=all" | \
-		jq -rc 'map(select($(GO_VERSION_SELECTOR)))[0].files | map(select(.os=="$(OS)" and .arch=="$(ARCH)"))[0].filename') | \
-		tar xzC $(GO_INSTALL_DESTINATION) || exit; \
+	@if [ ! -e $(GO_INSTALL_DESTINATION)/go ]; then cd $(GO_INSTALL_DESTINATION); \
+		curl -o go.archive -fsSL --proto '=https' --tlsv1.2 https://go.dev/dl/$(shell \
+			curl -fsSL --proto "=https" --tlsv1.2 "https://go.dev/dl/?mode=json&include=all" | \
+			jq -rc 'map(select($(GO_VERSION_SELECTOR)))[0].files | map(select(.os=="$(OS)" and .arch=="$(ARCH)"))[0].filename'); \
+		[ "$(OS)" = windows ] && unzip go.archive || tar xzf go.archive; \
 	else \
-		echo "Install already detected, doing nothing."; \
+		echo "Install already detected in $(GO_INSTALL_DESTINATION), doing nothing."; \
 	fi
 	@echo Remember to add to your profile: export PATH="$(GO_INSTALL_DESTINATION)/go/bin:\$$HOME/go/bin:\$$PATH"
 	$(call footer, $@)

diff --git a/hack/build-integration-canary.sh b/hack/build-integration-canary.sh
@@ -28,7 +28,8 @@ readonly root
 # "Blacklisting" here means that any dependency which name is blacklisted will be left untouched, at the version
 # currently pinned in the Dockerfile.
 # This is convenient so that currently broken alpha/beta/RC can be held back temporarily to keep the build green
-blacklist=()
+blacklist=(
+)
 
 # List all the repositories we depend on to build and run integration tests
 dependencies=(
@@ -162,13 +163,18 @@ latest::release(){
     [ ! "$ignore" ] || ! grep -q "$ignore" <<<"$line" || continue
     name="$(echo "$line"  | jq -rc .name)"
     if [ "$name" == "" ] || [ "$name" == null ] ; then
-      log::debug " > bogus release name ($name) ignored"
-      continue
+      # Kubo latest releases are NOT setting a release name anymore :/
+      # Hail mary on .tag_name...
+      name="$(echo "$line"  | jq -rc .tag_name)"
+      if [ "$name" == "" ] || [ "$name" == null ] ; then
+        log::debug " > bogus release name: $line"
+        continue
+      fi
     fi
     log::debug " > found release: $name"
-    if version::compare <(echo "$line" | jq -rc .name); then
+    if version::compare <(echo "$name"); then
       higher_data="$line"
-      higher_readable="$(echo "$line" | jq -rc .name | sed -E 's/(.*[ ])?(v?[0-9][0-9.a-z-]+).*/\2/')"
+      higher_readable="$(echo "$name" | sed -E 's/(.*[ ])?(v?[0-9][0-9.a-z-]+).*/\2/')"
     fi
   done < <(github::releases "$repo")
 
@@ -211,7 +217,6 @@ assets::get(){
 ######################
 # Script
 ######################
-
 canary::build::integration(){
   docker_args=(docker build -t test-integration --target test-integration)
 
@@ -291,58 +296,8 @@ canary::build::integration(){
     docker_args+=(--build-arg "${shortsafename}_VERSION=$higher_readable")
   done
 
-  hub_available_go_version="$(canary::golang::hublatest)"
-  if [ "$hub_available_go_version" != "" ]; then
-    docker_args+=(--build-arg "GO_VERSION=$hub_available_go_version")
-  fi
+  docker_args+=(--build-arg "GO_VERSION=$GO_VERSION")
 
   log::debug "${docker_args[*]} ."
   "${docker_args[@]}" "."
 }
-
-# Hub usually has a delay before available golang version show-up. This method will find the latest available one.
-# See
-# - https://github.com/containerd/nerdctl/issues/3224
-# - https://github.com/containerd/nerdctl/issues/3306
-canary::golang::hublatest(){
-  local hub_tags
-  local go_version
-  local available_version=""
-  local index
-
-  hub_tags="$(http::get /dev/stdout "https://registry-1.docker.io/v2/library/golang/tags/list" -H "Authorization: Bearer $(http::get /dev/stdout "https://auth.docker.io/token?service=registry.docker.io&scope=repository%3Alibrary%2Fgolang%3Apull" | jq  -rc .access_token)")"
-
-  index=0
-  while [ "$available_version" == "" ] && [ "$index" -lt 5 ]; do
-    go_version="$(http::get /dev/stdout "https://go.dev/dl/?mode=json&include=all" | jq -rc .[$index].version)"
-    go_version="${go_version##*go}"
-    available_version="$(printf "%s" "$hub_tags" | jq -rc ".tags[] | select(.==\"$go_version\")")"
-    ((index++))
-  done || true
-
-  printf "%s" "$available_version"
-}
-
-canary::golang::latest(){
-  # Enable extended globbing features to use advanced pattern matching
-  shopt -s extglob
-
-  # Get latest golang version and split it in components
-  norm=()
-  while read -r line; do
-    line_trimmed="${line//+([[:space:]])/}"
-    norm+=("$line_trimmed")
-  done < \
-    <(sed -E 's/^go([0-9]+)[.]([0-9]+)([.]([0-9]+))?(([a-z]+)([0-9]+))?/\1.\2\n\4\n\6\n\7/i' \
-      <(curl -fsSL "https://go.dev/dl/?mode=json&include=all" | jq -rc .[0].version) \
-    )
-
-  # Serialize version, making sure we have a patch version, and separate possible rcX into .rc-X
-  [ "${norm[1]}" != "" ] || norm[1]="0"
-  norm[1]=".${norm[1]}"
-  [ "${norm[2]}" == "" ] || norm[2]="-${norm[2]}"
-  [ "${norm[3]}" == "" ] || norm[3]=".${norm[3]}"
-  # Save it
-  IFS=
-  echo "GO_VERSION=${norm[*]}" >> "$GITHUB_ENV"
-}