Improve test matrix comprehensiveness

.github/workflows/test.yml

@@ -7,127 +7,163 @@ on:
   push: # on push to the main branch
     branches:
       - main
-env:
-  DD_APPSEC_WAF_TIMEOUT: 5s
+
 jobs:
-  native:
+  bare-metal:
     strategy:
       fail-fast: false
       matrix:
-        runs-on: [ macos-13, macos-12, macos-11, ubuntu-22.04, ubuntu-20.04 ]
+        runs-on: [ macos-13, macos-12, macos-11, ubuntu-22.04, ubuntu-20.04, windows-latest ]
         go-version: [ "1.21", "1.20", "1.19" ]
-        cgo_enabled: [ "0", "1" ] # test it compiles with and without cgo
+        cgo-enabled: [ "0", "1" ] # test it compiles with and without cgo
+        go-tags:
+          - ''                      # Default behavior
+          - 'datadog.no_waf'        # Explicitly disabled WAF
+          - 'go1.22'                # Too recent go version (purego compatibility uncertain)
+          - 'datadog.no_waf,go1.22' # Explicitly disabled & too recent go version (purego compatibility uncertain)
         include:
-          - env:
-              GODEBUG=cgocheck=2
-          - go-version: "1.21"
-            env:
-              GOEXPERIMENT=cgocheck2
+          # gocheck2 is configured differently in go1.21 than in previous versions
+          - go-version: '1.21'
+            go-experiment: cgocheck2
+          - go-version: '1.20'
+            go-debug: cgocheck=2
+          - go-version: '1.19'
+            go-debug: cgocheck=2
+        exclude:
+          # Prune redundant checks (the go-next test needs only run once per platform)
+          - go-version: '1.20'
+            go-tags: go1.22
+          - go-version: '1.20'
+            go-tags: datadog.no_waf,go1.22
+          - go-version: '1.19'
+            go-tags: go1.22
+          - go-version: '1.19'
+            go-tags: datadog.no_waf,go1.22
+    name: ${{ matrix.runs-on }} go${{ matrix.go-version }} cgo=${{ matrix.cgo-enabled }} tags=${{ matrix.go-tags }}
     runs-on: ${{ matrix.runs-on }}
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go-version }}
           cache: true
+      - name: Install gotestsum
+        run: go install gotest.tools/gotestsum@latest
       - name: go test
         shell: bash
-        run: |
-          # Install gotestsum
-          env GOBIN=$PWD go install gotest.tools/gotestsum@latest
-          # Run the tests with gotestsum
-          env ${{ matrix.env }} CGO_ENABLED=${{ matrix.cgo_enabled }} ./gotestsum -- -v -count=10 -shuffle=on ./...
-
-  disabled:
-    strategy:
-      fail-fast: false
-      matrix:
-        runs-on: [ windows-latest, ubuntu-latest, macos-13 ]
-        go-args: [ "-tags datadog.no_waf", "-tags go1.22" ]
-        include:
-          - runs-on: windows-latest
-            go-args: ""
-    runs-on: ${{ matrix.runs-on }}
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v4
-        with:
-          go-version: 'stable' # get latest stable version from https://github.com/actions/go-versions/blob/main/versions-manifest.json
-          cache: true
-      - name: go test
-        shell: bash
-        run: |
-          # Install gotestsum
-          env GOBIN=$PWD go install gotest.tools/gotestsum@latest
-          # Run the tests with gotestsum
-          ./gotestsum -- -v ${{ matrix.go-tags }} -shuffle=on ./...
+        run: |-
+          gotestsum -- -v -count=10 -shuffle=on -tags='${{ matrix.go-tags }}' ./...
+        env:
+          CGO_ENABLED: ${{ matrix.cgo-enabled }}
+          DD_APPSEC_WAF_TIMEOUT: 5s
+          GODEBUG: ${{ matrix.go-debug }}
+          GOEXPERIMENT: ${{ matrix.go-experiment }}
 
-  # Same tests but on the official golang container for linux
-  golang-linux-container:
-    runs-on: ubuntu-latest
-    container:
-      image: golang:${{ matrix.go-version }}-${{ matrix.distribution }}
+  containerized:
     strategy:
       fail-fast: false
       matrix:
+        image:
+          # Standard golang image
+          - golang:{0}-alpine
+          - golang:{0}-bookworm
+          - golang:{0}-bullseye
+          - golang:{0}-buster
+          # RPM-based image
+          - amazonlinux:2 # pretty popular on AWS workloads
+        arch: [ amd64, arm64 ]
         go-version: [ "1.21", "1.20", "1.19" ]
-        distribution: [ bookworm, bullseye, buster, alpine ]
-        cgo_enabled: [ "0", "1" ] # test it compiles with and without cgo
+        cgo-enabled: [ "0", "1" ] # test it compiles with and without cgo
+        go-tags:
+          - ''                      # Default behavior
+          - 'datadog.no_waf'        # Explicitly disabled WAF
+          - 'go1.22'                # Too recent go version (purego compatibility uncertain)
+          - 'datadog.no_waf,go1.22' # Explicitly disabled & too recent go version (purego compatibility uncertain)
+        include:
+          # gocheck2 is configured differently in go1.21 than in previous versions
+          - go-version: '1.21'
+            go-experiment: cgocheck2
+          - go-version: '1.20'
+            go-debug: cgocheck=2
+          - go-version: '1.19'
+            go-debug: cgocheck=2
         exclude:
-          - go-version: 1.18
-            distribution: bookworm
-          - go-version: 1.21
-            distribution: buster
+          # Prune redundant checks (the go-next test needs only run once per platform)
+          - go-version: '1.20'
+            go-tags: go1.22
+          - go-version: '1.20'
+            go-tags: datadog.no_waf,go1.22
+          - go-version: '1.19'
+            go-tags: go1.22
+          - go-version: '1.19'
+            go-tags: datadog.no_waf,go1.22
+          # Prune inexistant build images (debian buster is on LTS but won't get new go version images)
+          - go-version: '1.21'
+            image: golang:{0}-buster
+          # The amazonlinux:2 variant is only relevant for the default go version yum ships (currently 1.20)
+          - go-version: '1.19'
+            image: amazonlinux:2
+          - go-version: '1.21'
+            image: amazonlinux:2
+    name: linux/${{ matrix.arch }} ${{ format(matrix.image, matrix.go-version) }} cgo=${{ matrix.cgo-enabled }} tags=${{ matrix.go-tags }}
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-
-      # Install gcc and the libc headers on alpine images
-      - if: ${{ matrix.distribution == 'alpine' }}
-        run: apk add gcc musl-dev libc6-compat
-
-      - name: Go modules cache
-        uses: actions/cache@v3
+      - uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: go-pkg-mod-${{ hashFiles('**/go.sum') }}
           restore-keys: go-pkg-mod-
-
-      - name: go test
-        run: |
-          # Install gotestsum
-          env GOBIN=$PWD go install gotest.tools/gotestsum@latest
-          # Run the tests with gotestsum
-          env CGO_ENABLED=${{ matrix.cgo_enabled }} ./gotestsum -- -v -count=10 -shuffle=on ./...
-
-  linux-other:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        arch: ["arm64"]
-        cgo_enabled: [ "0", "1" ] # test it compiles with and without the cgo
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v3
-      - name: Go modules cache
-        uses: actions/cache@v3
-        with:
-          path: ~/go/pkg/mod
-          key: go-pkg-mod-${{ matrix.arch }}-${{ hashFiles('**/go.sum') }}
-          restore-keys: go-pkg-mod-${{ matrix.arch }} go-pkg-mod-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
         with:
           platforms: ${{ matrix.arch }}
-      - run: docker run --platform=linux/${{ matrix.arch }} -v${HOME}/go/pkg/mod:/root/go/pkg/mod -v $PWD:$PWD -w $PWD -eCGO_ENABLED=${{ matrix.cgo_enabled }} -eDD_APPSEC_WAF_TIMEOUT=$DD_APPSEC_WAF_TIMEOUT golang go test -v -count=10 -shuffle=on ./...
+      - name: Create container
+        id: container
+        run: |-
+          docker run --name gha-${{ github.run_id }} --rm -di                   \
+            --platform="linux/${{ matrix.arch }}"                               \
+            -v "${HOME}/go/pkg/mod:/go/pkg/mod"                                 \
+            -v "$PWD:$PWD"                                                      \
+            -w "$PWD"                                                           \
+            -eCGO_ENABLED="${{ matrix.cgo-enabled }}"                           \
+            -eDD_APPSEC_WAF_TIMEOUT="${DD_APPSEC_WAF_TIMEOUT}"                  \
+            -eGODEBUG="${{ matrix.go-debug }}"                                  \
+            -eGOEXPERIMENT="${{ matrix.go-experiment }}"                        \
+            -eGOMODCACHE="/go/pkg/mod"                                          \
+            "${{ format(matrix.image, matrix.go-version) }}"
+      - name: Install alpine requirements
+        if: endsWith(matrix.image, '-alpine') && matrix.cgo-enabled == '1'
+        run: |-
+          docker exec -i gha-${{ github.run_id }}                               \
+            apk add gcc musl-dev libc6-compat
+      - name: Install AmazonLinux 2 requirements
+        if: matrix.image == 'amazonlinux:2'
+        run: |-
+          docker exec -i gha-${{ github.run_id }}                               \
+            yum install -y golang
+      - name: Install gotestsum
+        run: |-
+          docker exec -i gha-${{ github.run_id }}                               \
+            go install gotest.tools/gotestsum@latest
+      - name: go test
+        run: |-
+          docker exec -i gha-${{ github.run_id }}                               \
+            go run gotest.tools/gotestsum@latest --                             \
+              -v -count=10 -shuffle=on -tags='${{ matrix.go-tags }}'             \
+              ./...
+      - name: Stop container
+        if: always() && steps.container.outcome == 'success'
+        run: |-
+          docker stop gha-${{ github.run_id }}
 
   # A simple join target to simplify setting up branch protection settings in GH.
   done:
     name: Done
     runs-on: ubuntu-latest
     needs:
-      - native
-      - golang-linux-container
-      - linux-other
+      - bare-metal
+      - containerized
     steps:
       - name: Done
         run: echo "Done!"

go.mod

@@ -1,6 +1,6 @@
 module github.com/DataDog/go-libddwaf/v2
 
-go 1.18
+go 1.19
 
 require (
 	github.com/ebitengine/purego v0.5.0

waf_manually_disabled_test.go

@@ -14,15 +14,17 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestLoad(t *testing.T) {
-	ok, err := waf.Load()
-	require.False(t, ok)
-	require.Error(t, err)
-}
+func TestManuallyDisabled(t *testing.T) {
+	t.Run("TestLoad", func(t *testing.T) {
+		ok, err := waf.Load()
+		require.False(t, ok)
+		require.Error(t, err)
+	})
 
-func TestHealth(t *testing.T) {
-	ok, err := waf.Health()
-	require.False(t, ok)
-	require.Error(t, err)
-	require.ErrorIs(t, err, waf.ManuallyDisabledError{})
+	t.Run("TestHealth", func(t *testing.T) {
+		ok, err := waf.Health()
+		require.False(t, ok)
+		require.Error(t, err)
+		require.ErrorIs(t, err, waf.ManuallyDisabledError{})
+	})
 }

waf_support.go

@@ -7,6 +7,7 @@ package waf
 
 import (
 	"fmt"
+	"runtime"
 
 	"github.com/hashicorp/go-multierror"
 )
@@ -31,12 +32,10 @@ func (e UnsupportedOSArchError) Error() string {
 
 // UnsupportedGoVersionError is a wrapper error type helping to handle the error
 // case of trying to execute this package when the Go version is not supported.
-type UnsupportedGoVersionError struct {
-	Version string
-}
+type UnsupportedGoVersionError struct{}
 
 func (e UnsupportedGoVersionError) Error() string {
-	return fmt.Sprintf("unsupported Go version: %s", e.Version)
+	return fmt.Sprintf("unsupported Go version: %s", runtime.Version())
 }
 
 // ManuallyDisabledError is a wrapper error type helping to handle the error

waf_unsupported_go.go

@@ -8,10 +8,6 @@
 
 package waf
 
-import (
-	"runtime"
-)
-
 func init() {
-	wafSupportErrors = append(wafSupportErrors, UnsupportedGoVersionError{runtime.Version()})
+	wafSupportErrors = append(wafSupportErrors, UnsupportedGoVersionError{})
 }

waf_unsupported_go_test.go

@@ -8,26 +8,29 @@
 package waf_test
 
 import (
+	"testing"
+
 	waf "github.com/DataDog/go-libddwaf/v2"
 	"github.com/stretchr/testify/require"
-	"testing"
 )
 
-func TestSupportsTarget(t *testing.T) {
-	supported, err := waf.SupportsTarget()
-	require.False(t, supported)
-	require.Error(t, err)
-	require.ErrorIs(t, err, waf.UnsupportedGoVersionError{})
-}
+func TestUnsupportedGoRuntime(t *testing.T) {
+	t.Run("TestSupportsTarget", func(t *testing.T) {
+		supported, err := waf.SupportsTarget()
+		require.False(t, supported)
+		require.Error(t, err)
+		require.ErrorIs(t, err, waf.UnsupportedGoVersionError{})
+	})
 
-func TestLoad(t *testing.T) {
-	ok, err := waf.Load()
-	require.False(t, ok)
-	require.Error(t, err)
-}
+	t.Run("TestLoad", func(t *testing.T) {
+		ok, err := waf.Load()
+		require.False(t, ok)
+		require.Error(t, err)
+	})
 
-func TestHealth(t *testing.T) {
-	ok, err := waf.Health()
-	require.False(t, ok)
-	require.Error(t, err)
+	t.Run("TestHealth", func(t *testing.T) {
+		ok, err := waf.Health()
+		require.False(t, ok)
+		require.Error(t, err)
+	})
 }