Merge pull request #271 from rojkov/e2e-ginkgo

e2e test: reimplement with K8s testing framework

Author: bart0sh

Makefile

@@ -1,12 +1,18 @@
 GO := go
 GOFMT := gofmt
 GOCYCLO := gocyclo
+KUBECTL ?= kubectl
+KIND ?= kind
+PODMAN ?= podman
 
 BUILDTAGS ?= ""
 BUILDER ?= "docker"
 
+WEBHOOK_IMAGE_FILE = intel-fpga-admissionwebhook-devel.tgz
+
 pkgs  = $(shell $(GO) list ./... | grep -v vendor | grep -v e2e)
 cmds = $(shell ls cmd)
+e2e_tmp_dir := $(shell mktemp -u -t e2e-tests.XXXXXXXXXX)
 
 all: build
 
@@ -38,10 +44,17 @@ else
             exit $$rc
 endif
 
-test-e2e:
+test-with-kind:
 	@build/docker/build-image.sh intel/intel-fpga-admissionwebhook buildah
-	@podman tag localhost/intel/intel-fpga-admissionwebhook:devel docker.io/intel/intel-fpga-admissionwebhook:devel
-	@$(GO) test -v ./test/e2e/...
+	@$(PODMAN) tag localhost/intel/intel-fpga-admissionwebhook:devel docker.io/intel/intel-fpga-admissionwebhook:devel
+	@mkdir -p $(e2e_tmp_dir)
+	@$(PODMAN) save "docker.io/intel/intel-fpga-admissionwebhook:devel" -o $(e2e_tmp_dir)/$(WEBHOOK_IMAGE_FILE)
+	@$(KIND) create cluster --name "intel-device-plugins" --kubeconfig $(e2e_tmp_dir)/kubeconfig --image "kindest/node:v1.17.0"
+	@$(KIND) load image-archive --name "intel-device-plugins" $(e2e_tmp_dir)/$(WEBHOOK_IMAGE_FILE)
+	@$(GO) test -v ./test/e2e -args -kubeconfig $(e2e_tmp_dir)/kubeconfig -kubectl-path $(KUBECTL) -ginkgo.focus "Webhook" || rc=1; \
+	$(KIND) delete cluster --name "intel-device-plugins"; \
+	rm -rf $(e2e_tmp_dir); \
+	exit $$rc
 
 lint:
 	@rc=0 ; for f in $$(find -name \*.go | grep -v \.\/vendor) ; do golint -set_exit_status $$f || rc=1 ; done ; exit $$rc

README.md

@@ -126,6 +126,44 @@ For information on how to develop a new plugin using the framework, see the
 [Developers Guide](DEVEL.md) and the code in the
 [device plugins pkg directory](pkg/deviceplugin).
 
+## Running E2E tests
+
+Currently the E2E tests require having a Kubernetes cluster already configured
+on the nodes with the hardware required by the device plugins. Also all the
+container images with the executables under test must be available in the
+cluster. Given these two conditions are satisfied one can run the tests with
+
+```bash
+$ go test -v ./test/e2e/...
+```
+
+In case you want to run only certain tests, e.g. QAT ones, then run
+
+```bash
+$ go test -v ./test/e2e/... -args -ginkgo.focus "QAT"
+```
+
+If you need to specify paths to your custom `kubeconfig` containing
+embedded authentication info then add the `-kubeconfig` argument:
+
+```bash
+$ go test -v ./test/e2e/... -args -kubeconfig /path/to/kubeconfig
+```
+
+The full list of available options can be obtained with
+
+```bash
+$ go test ./test/e2e/... -args -help
+```
+
+Also it is possible to run the tests which don't depend on hardware
+without a pre-configured Kubernetes cluster. Just make sure you have
+[Kind](https://kind.sigs.k8s.io/) installed on your host and run
+
+```
+$ make test-with-kind
+```
+
 ## Supported Kubernetes versions
 
 Releases are made under the github [releases area](releases). Supported releases and

go.mod

@@ -7,6 +7,8 @@ require (
 	github.com/go-ini/ini v1.46.0
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/google/gousb v0.0.0-20190812193832-18f4c1d8a750
+	github.com/onsi/ginkgo v1.10.1
+	github.com/onsi/gomega v1.7.0
 	github.com/pkg/errors v0.8.1
 	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect
 	golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456
@@ -15,6 +17,8 @@ require (
 	k8s.io/api v0.17.0
 	k8s.io/apimachinery v0.17.0
 	k8s.io/client-go v0.17.0
+	k8s.io/component-base v0.17.0
+	k8s.io/klog v1.0.0
 	k8s.io/kubelet v0.0.0
 	k8s.io/kubernetes v1.17.0
 	k8s.io/utils v0.0.0-20191114184206-e782cd3c129f

go.sum

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-which cfssl > /dev/null 2>&1 || (echo "Please install 'cfssl' (e.g. with 'go get -u github.com/cloudflare/cfssl/cmd/cfssl')"; exit 1)
-which jq > /dev/null 2>&1 || (echo "Please install 'jq'"; exit 1)
+which cfssl > /dev/null 2>&1 || { echo "Please install 'cfssl' (e.g. with 'go get -u github.com/cloudflare/cfssl/cmd/cfssl')"; exit 1; }
+which jq > /dev/null 2>&1 || { echo "Please install 'jq'"; exit 1; }
 
 while [[ $# -gt 0 ]]; do
     case ${1} in
@@ -30,6 +30,8 @@ done
 [ -z ${namespace} ] && namespace="default"
 [ -z ${kubectl} ] && kubectl="kubectl"
 
+which ${kubectl} > /dev/null 2>&1 || { echo "ERROR: ${kubectl} not found"; exit 1; }
+
 csrname="${service}.${namespace}"
 tmpdir=$(mktemp -d)
 

scripts/webhook-create-signed-cert.sh

@@ -57,6 +57,8 @@ done
 [ -z ${mode} ] && mode="preprogrammed"
 [ -z ${namespace} ] && namespace="default"
 
+which ${kubectl} > /dev/null 2>&1 || { echo "ERROR: ${kubectl} not found"; exit 1; }
+
 # clean up any previously created deployment
 ${kubectl} delete MutatingWebhookConfiguration "fpga-mutator-webhook-cfg" 2>/dev/null || true
 ${kubectl} --namespace ${namespace} delete service ${service} 2>/dev/null || true

scripts/webhook-deploy.sh

@@ -0,0 +1,111 @@
+// Copyright 2020 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e_test
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+
+	_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/fpgaadmissionwebhook"
+	_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/gpu"
+	_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/qat"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/component-base/logs"
+	"k8s.io/component-base/version"
+	"k8s.io/klog"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/config"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+)
+
+func init() {
+	ginkgo.SynchronizedBeforeSuite(setupFirstNode, func(data []byte) {})
+}
+
+func setupFirstNode() []byte {
+	c, err := framework.LoadClientset()
+	if err != nil {
+		framework.Failf("Error loading client: %v", err)
+	}
+
+	// Delete any namespaces except those created by the system. This ensures no
+	// lingering resources are left over from a previous test run.
+	deleted, err := framework.DeleteNamespaces(c, nil, /* deleteFilter */
+		[]string{
+			metav1.NamespaceSystem,
+			metav1.NamespaceDefault,
+			metav1.NamespacePublic,
+			v1.NamespaceNodeLease,
+		})
+	if err != nil {
+		framework.Failf("Error deleting orphaned namespaces: %v", err)
+	}
+	framework.Logf("Waiting for deletion of the following namespaces: %v", deleted)
+	if err := framework.WaitForNamespacesDeleted(c, deleted, framework.NamespaceCleanupTimeout); err != nil {
+		framework.Failf("Failed to delete orphaned namespaces %v: %v", deleted, err)
+	}
+
+	framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))
+
+	// Ensure all pods are running and ready before starting tests (otherwise,
+	// cluster infrastructure pods that are being pulled or started can block
+	// test pods from running, and tests that ensure all pods are running and
+	// ready will fail).
+	if err := e2epod.WaitForPodsRunningReady(c, metav1.NamespaceSystem, int32(framework.TestContext.MinStartupPods),
+		int32(framework.TestContext.AllowedNotReadyNodes), framework.TestContext.SystemPodsStartupTimeout,
+		map[string]string{}); err != nil {
+		framework.DumpAllNamespaceInfo(c, metav1.NamespaceSystem)
+		framework.LogFailedContainers(c, metav1.NamespaceSystem, framework.Logf)
+		framework.Failf("Error waiting for all pods to be running and ready: %v", err)
+	}
+
+	// Log the version of the server and this client.
+	framework.Logf("e2e test version: %s", version.Get().GitVersion)
+	serverVersion, err := c.DiscoveryClient.ServerVersion()
+	if err != nil {
+		framework.Logf("Unexpected server error retrieving version: %v", err)
+	}
+	if serverVersion != nil {
+		framework.Logf("kube-apiserver version: %s", serverVersion.GitVersion)
+	}
+
+	return []byte{}
+}
+
+func TestDevicePlugins(t *testing.T) {
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecs(t, "E2E Device Plugins Suite")
+}
+
+func TestMain(m *testing.M) {
+	klog.SetOutput(ginkgo.GinkgoWriter)
+
+	logs.InitLogs()
+	config.CopyFlags(config.Flags, flag.CommandLine)
+	framework.RegisterCommonFlags(flag.CommandLine)
+	framework.RegisterClusterFlags(flag.CommandLine)
+	flag.Parse()
+
+	// Register framework flags, then handle flags.
+	framework.AfterReadingAllFlags(&framework.TestContext)
+
+	// Now run the test suite.
+	os.Exit(m.Run())
+}