Merge pull request kubernetes#26470 from timstclair/node-e2e

Automatic merge from submit-queue

Refactor common pod patterns to e2e framework.

Priveledged pods could benifit from this as well, but I'll wait for kubernetes#26228 to go in.

Prerequisite for kubernetes#26215

Author: k8s-merge-robot

test/e2e/e2e_test.go

@@ -23,7 +23,8 @@ import (
 )
 
 func init() {
-	framework.RegisterFlags()
+	framework.RegisterCommonFlags()
+	framework.RegisterClusterFlags()
 }
 
 func TestE2E(t *testing.T) {

test/e2e/framework/pods.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"sync"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/client/unversioned"
+
+	. "github.com/onsi/gomega"
+)
+
+// TODO: Consolidate pod-specific framework functions here.
+
+// Convenience method for getting a pod client interface in the framework's namespace.
+func (f *Framework) PodClient() unversioned.PodInterface {
+	return f.Client.Pods(f.Namespace.Name)
+}
+
+// Create a new pod according to the framework specifications, and wait for it to start.
+func (f *Framework) CreatePod(pod *api.Pod) {
+	f.CreatePodAsync(pod)
+	ExpectNoError(f.WaitForPodRunning(pod.Name))
+}
+
+// Create a new pod according to the framework specifications (don't wait for it to start).
+func (f *Framework) CreatePodAsync(pod *api.Pod) {
+	f.MungePodSpec(pod)
+	_, err := f.PodClient().Create(pod)
+	ExpectNoError(err, "Error creating Pod")
+}
+
+// Batch version of CreatePod. All pods are created before waiting.
+func (f *Framework) CreatePods(pods []*api.Pod) {
+	for _, pod := range pods {
+		f.CreatePodAsync(pod)
+	}
+	var wg sync.WaitGroup
+	for _, pod := range pods {
+		wg.Add(1)
+		podName := pod.Name
+		go func() {
+			ExpectNoError(f.WaitForPodRunning(podName))
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+}
+
+// Apply test-suite specific transformations to the pod spec.
+// TODO: figure out a nicer, more generic way to tie this to framework instances.
+func (f *Framework) MungePodSpec(pod *api.Pod) {
+	if TestContext.NodeName != "" {
+		Expect(pod.Spec.NodeName).To(Or(BeZero(), Equal(TestContext.NodeName)), "Test misconfigured")
+		pod.Spec.NodeName = TestContext.NodeName
+	}
+}

test/e2e/framework/test_context.go

@@ -65,6 +65,8 @@ type TestContextType struct {
 	CreateTestingNS CreateTestingNSFn
 	// If set to true test will dump data about the namespace in which test was running.
 	DumpLogsOnFailure bool
+	// Name of the node to run tests on (node e2e suite only).
+	NodeName string
 }
 
 type CloudConfig struct {
@@ -83,7 +85,8 @@ type CloudConfig struct {
 var TestContext TestContextType
 var federatedKubeContext string
 
-func RegisterFlags() {
+// Register flags common to all e2e test suites.
+func RegisterCommonFlags() {
 	// Turn on verbose by default to get spec names
 	config.DefaultReporterConfig.Verbose = true
 
@@ -93,6 +96,19 @@ func RegisterFlags() {
 	// Randomize specs as well as suites
 	config.GinkgoConfig.RandomizeAllSpecs = true
 
+	flag.StringVar(&TestContext.GatherKubeSystemResourceUsageData, "gather-resource-usage", "false", "If set to 'true' or 'all' framework will be monitoring resource usage of system all add-ons in (some) e2e tests, if set to 'master' framework will be monitoring master node only, if set to 'none' of 'false' monitoring will be turned off.")
+	flag.BoolVar(&TestContext.GatherLogsSizes, "gather-logs-sizes", false, "If set to true framework will be monitoring logs sizes on all machines running e2e tests.")
+	flag.BoolVar(&TestContext.GatherMetricsAfterTest, "gather-metrics-at-teardown", false, "If set to true framwork will gather metrics from all components after each test.")
+	flag.StringVar(&TestContext.OutputPrintType, "output-print-type", "hr", "Comma separated list: 'hr' for human readable summaries 'json' for JSON ones.")
+	flag.BoolVar(&TestContext.DumpLogsOnFailure, "dump-logs-on-failure", true, "If set to true test will dump data about the namespace in which test was running.")
+}
+
+// Register flags specific to the cluster e2e test suite.
+func RegisterClusterFlags() {
+	// TODO: Move to common flags once namespace deletion is fixed for node e2e.
+	flag.BoolVar(&TestContext.DeleteNamespace, "delete-namespace", true, "If true tests will delete namespace after completion. It is only designed to make debugging easier, DO NOT turn it off by default.")
+
+	flag.BoolVar(&TestContext.VerifyServiceAccount, "e2e-verify-service-account", true, "If true tests will verify the service account before running.")
 	flag.StringVar(&TestContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.")
 	flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'")
 	flag.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "", "ContentType used to communicate with apiserver")
@@ -126,12 +142,10 @@ func RegisterFlags() {
 	flag.DurationVar(&TestContext.SystemPodsStartupTimeout, "system-pods-startup-timeout", 10*time.Minute, "Timeout for waiting for all system pods to be running before starting tests.")
 	flag.StringVar(&TestContext.UpgradeTarget, "upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an upgrade test.")
 	flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.")
-	flag.BoolVar(&TestContext.VerifyServiceAccount, "e2e-verify-service-account", true, "If true tests will verify the service account before running.")
-	flag.BoolVar(&TestContext.DeleteNamespace, "delete-namespace", true, "If true tests will delete namespace after completion. It is only designed to make debugging easier, DO NOT turn it off by default.")
 	flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.")
-	flag.StringVar(&TestContext.GatherKubeSystemResourceUsageData, "gather-resource-usage", "false", "If set to 'true' or 'all' framework will be monitoring resource usage of system all add-ons in (some) e2e tests, if set to 'master' framework will be monitoring master node only, if set to 'none' of 'false' monitoring will be turned off.")
-	flag.BoolVar(&TestContext.GatherLogsSizes, "gather-logs-sizes", false, "If set to true framework will be monitoring logs sizes on all machines running e2e tests.")
-	flag.BoolVar(&TestContext.GatherMetricsAfterTest, "gather-metrics-at-teardown", false, "If set to true framwork will gather metrics from all components after each test.")
-	flag.StringVar(&TestContext.OutputPrintType, "output-print-type", "hr", "Comma separated list: 'hr' for human readable summaries 'json' for JSON ones.")
-	flag.BoolVar(&TestContext.DumpLogsOnFailure, "dump-logs-on-failure", true, "If set to true test will dump data about the namespace in which test was running.")
+}
+
+// Register flags specific to the node e2e test suite.
+func RegisterNodeFlags() {
+	flag.StringVar(&TestContext.NodeName, "node-name", "", "Name of the node to run tests on (node e2e suite only).")
 }

test/e2e_node/configmap.go

@@ -121,13 +121,8 @@ var _ = framework.KubeDescribe("ConfigMap", func() {
 			},
 		}
 
-		assignPodToNode(pod)
-
 		By("Creating the pod")
-		_, err = f.Client.Pods(f.Namespace.Name).Create(pod)
-		Expect(err).NotTo(HaveOccurred())
-
-		framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.Client, pod.Name, f.Namespace.Name))
+		f.CreatePod(pod)
 
 		pollLogs := func() (string, error) {
 			return framework.GetPodLogs(f.Client, f.Namespace.Name, pod.Name, containerName)
@@ -184,7 +179,7 @@ var _ = framework.KubeDescribe("ConfigMap", func() {
 			},
 		}
 
-		assignPodToNode(pod)
+		f.MungePodSpec(pod)
 
 		framework.TestContainerOutput("consume configMaps", f.Client, pod, 0, []string{
 			"CONFIG_DATA_1=value-1",
@@ -265,7 +260,7 @@ func doConfigMapE2EWithoutMappings(f *framework.Framework, uid, fsGroup int64) {
 		pod.Spec.SecurityContext.FSGroup = &fsGroup
 	}
 
-	assignPodToNode(pod)
+	f.MungePodSpec(pod)
 
 	framework.TestContainerOutput("consume configMaps", f.Client, pod, 0, []string{
 		"content of file \"/etc/configmap-volume/data-1\": value-1",
@@ -338,7 +333,7 @@ func doConfigMapE2EWithMappings(f *framework.Framework, uid, fsGroup int64) {
 		pod.Spec.SecurityContext.FSGroup = &fsGroup
 	}
 
-	assignPodToNode(pod)
+	f.MungePodSpec(pod)
 
 	framework.TestContainerOutput("consume configMaps", f.Client, pod, 0, []string{
 		"content of file \"/etc/configmap-volume/path/to/data-2\": value-2",

test/e2e_node/container.go

@@ -21,36 +21,33 @@ import (
 
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/errors"
-	client "k8s.io/kubernetes/pkg/client/unversioned"
 	"k8s.io/kubernetes/pkg/util"
+	"k8s.io/kubernetes/test/e2e/framework"
 )
 
 // One pod one container
+// TODO: This should be migrated to the e2e framework.
 type ConformanceContainer struct {
+	Framework        *framework.Framework
 	Container        api.Container
-	Client           *client.Client
 	RestartPolicy    api.RestartPolicy
 	Volumes          []api.Volume
 	ImagePullSecrets []string
-	NodeName         string
-	Namespace        string
 
 	podName string
 }
 
-func (cc *ConformanceContainer) Create() error {
+func (cc *ConformanceContainer) Create() {
 	cc.podName = cc.Container.Name + string(util.NewUUID())
 	imagePullSecrets := []api.LocalObjectReference{}
 	for _, s := range cc.ImagePullSecrets {
 		imagePullSecrets = append(imagePullSecrets, api.LocalObjectReference{Name: s})
 	}
 	pod := &api.Pod{
 		ObjectMeta: api.ObjectMeta{
-			Name:      cc.podName,
-			Namespace: cc.Namespace,
+			Name: cc.podName,
 		},
 		Spec: api.PodSpec{
-			NodeName:      cc.NodeName,
 			RestartPolicy: cc.RestartPolicy,
 			Containers: []api.Container{
 				cc.Container,
@@ -59,33 +56,31 @@ func (cc *ConformanceContainer) Create() error {
 			ImagePullSecrets: imagePullSecrets,
 		},
 	}
-
-	_, err := cc.Client.Pods(cc.Namespace).Create(pod)
-	return err
+	cc.Framework.CreatePodAsync(pod)
 }
 
 func (cc *ConformanceContainer) Delete() error {
-	return cc.Client.Pods(cc.Namespace).Delete(cc.podName, api.NewDeleteOptions(0))
+	return cc.Framework.PodClient().Delete(cc.podName, api.NewDeleteOptions(0))
 }
 
 func (cc *ConformanceContainer) IsReady() (bool, error) {
-	pod, err := cc.Client.Pods(cc.Namespace).Get(cc.podName)
+	pod, err := cc.Framework.PodClient().Get(cc.podName)
 	if err != nil {
 		return false, err
 	}
 	return api.IsPodReady(pod), nil
 }
 
 func (cc *ConformanceContainer) GetPhase() (api.PodPhase, error) {
-	pod, err := cc.Client.Pods(cc.Namespace).Get(cc.podName)
+	pod, err := cc.Framework.PodClient().Get(cc.podName)
 	if err != nil {
 		return api.PodUnknown, err
 	}
 	return pod.Status.Phase, nil
 }
 
 func (cc *ConformanceContainer) GetStatus() (api.ContainerStatus, error) {
-	pod, err := cc.Client.Pods(cc.Namespace).Get(cc.podName)
+	pod, err := cc.Framework.PodClient().Get(cc.podName)
 	if err != nil {
 		return api.ContainerStatus{}, err
 	}
@@ -97,7 +92,7 @@ func (cc *ConformanceContainer) GetStatus() (api.ContainerStatus, error) {
 }
 
 func (cc *ConformanceContainer) Present() (bool, error) {
-	_, err := cc.Client.Pods(cc.Namespace).Get(cc.podName)
+	_, err := cc.Framework.PodClient().Get(cc.podName)
 	if err == nil {
 		return true, nil
 	}

test/e2e_node/container_manager_test.go

@@ -36,14 +36,11 @@ var _ = framework.KubeDescribe("Kubelet Container Manager", func() {
 
 			BeforeEach(func() {
 				podName = "bin-false" + string(util.NewUUID())
-				pod := &api.Pod{
+				f.CreatePodAsync(&api.Pod{
 					ObjectMeta: api.ObjectMeta{
-						Name:      podName,
-						Namespace: f.Namespace.Name,
+						Name: podName,
 					},
 					Spec: api.PodSpec{
-						// Force the Pod to schedule to the node without a scheduler running
-						NodeName: *nodeName,
 						// Don't restart the Pod since it is expected to exit
 						RestartPolicy: api.RestartPolicyNever,
 						Containers: []api.Container{
@@ -54,15 +51,12 @@ var _ = framework.KubeDescribe("Kubelet Container Manager", func() {
 							},
 						},
 					},
-				}
-
-				_, err := f.Client.Pods(f.Namespace.Name).Create(pod)
-				Expect(err).To(BeNil(), fmt.Sprintf("Error creating Pod %v", err))
+				})
 			})
 
 			It("should have an error terminated reason", func() {
 				Eventually(func() error {
-					podData, err := f.Client.Pods(f.Namespace.Name).Get(podName)
+					podData, err := f.PodClient().Get(podName)
 					if err != nil {
 						return err
 					}
@@ -81,7 +75,7 @@ var _ = framework.KubeDescribe("Kubelet Container Manager", func() {
 			})
 
 			It("should be possible to delete", func() {
-				err := f.Client.Pods(f.Namespace.Name).Delete(podName, &api.DeleteOptions{})
+				err := f.PodClient().Delete(podName, &api.DeleteOptions{})
 				Expect(err).To(BeNil(), fmt.Sprintf("Error deleting Pod %v", err))
 			})
 		})

test/e2e_node/downward_api_test.go

@@ -158,7 +158,7 @@ func testDownwardAPI(f *framework.Framework, podName string, env []api.EnvVar, e
 			RestartPolicy: api.RestartPolicyNever,
 		},
 	}
-	assignPodToNode(pod)
+	f.MungePodSpec(pod)
 
 	f.TestContainerOutputRegexp("downward api env vars", pod, 0, expectations)
 }

test/e2e_node/e2e_node_suite_test.go

@@ -31,6 +31,8 @@ import (
 	"testing"
 	"time"
 
+	"k8s.io/kubernetes/test/e2e/framework"
+
 	"github.com/golang/glog"
 	. "github.com/onsi/ginkgo"
 	more_reporters "github.com/onsi/ginkgo/reporters"
@@ -42,6 +44,11 @@ var e2es *e2eService
 var prePullImages = flag.Bool("prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.")
 var junitFileNumber = flag.Int("junit-file-number", 1, "Used to create junit filename - e.g. junit_01.xml.")
 
+func init() {
+	framework.RegisterCommonFlags()
+	framework.RegisterNodeFlags()
+}
+
 func TestE2eNode(t *testing.T) {
 	flag.Parse()
 
@@ -67,12 +74,12 @@ var _ = BeforeSuite(func() {
 	if *buildServices {
 		buildGo()
 	}
-	if *nodeName == "" {
+	if framework.TestContext.NodeName == "" {
 		output, err := exec.Command("hostname").CombinedOutput()
 		if err != nil {
 			glog.Fatalf("Could not get node name from hostname %v.  Output:\n%s", err, output)
 		}
-		*nodeName = strings.TrimSpace(fmt.Sprintf("%s", output))
+		framework.TestContext.NodeName = strings.TrimSpace(fmt.Sprintf("%s", output))
 	}
 
 	// Pre-pull the images tests depend on so we can fail immediately if there is an image pull issue
@@ -89,7 +96,7 @@ var _ = BeforeSuite(func() {
 	maskLocksmithdOnCoreos()
 
 	if *startServices {
-		e2es = newE2eService(*nodeName)
+		e2es = newE2eService(framework.TestContext.NodeName)
 		if err := e2es.start(); err != nil {
 			Fail(fmt.Sprintf("Unable to start node services.\n%v", err))
 		}