Merge pull request #429 from cloudfoundry/skip-tests-on-k8s

After applying this change we couldn't actually get any cats tests to pass when targeting a cf-for-k8s foundation.  We ran into incompatibilities like specifying buildpacks, which cf-for-k8s doesn't currently allow.

Author: ericpromislow

README.md

@@ -192,6 +192,8 @@ include_v3
 * `volume_service_plan_name`: The name of the plan of the service provided by the volume service broker.
 * `volume_service_create_config`: The JSON configuration that is used when volume service is created.
 
+* `infrastructure`: The name of the infrastructure for the environment that the tests will run against. Must be either "vms" or "kubernetes". Defaults to "vms".
+
 #### Buildpack Names
 Many tests specify a buildpack when pushing an app, so that on diego the app staging process completes in less time. The default names for the buildpacks are as follows; if you have buildpacks with different names, you can override them by setting different names:
 

apps/admin_buildpack_lifecycle.go

@@ -32,6 +32,8 @@ var _ = AppsDescribe("Admin Buildpacks", func() {
 		buildpackArchivePath string
 	)
 
+	SkipOnK8s()
+
 	noAppDetectedErrorRegexp := "NoAppDetectedError|An app was not successfully detected by any available buildpack"
 	buildpackCompileFailedRegexp := "BuildpackCompileFailed|App staging failed in the buildpack compile phase"
 	buildpackReleaseFailedRegexp := "BuildpackReleaseFailed|App staging failed in the buildpack release phase"

apps/app_stack.go

@@ -34,6 +34,8 @@ var _ = AppsDescribe("Specifying a specific stack", func() {
 		tmpdir string
 	)
 
+	SkipOnK8s()
+
 	matchingFilename := func(appName string) string {
 		return fmt.Sprintf("stack-match-%s", appName)
 	}

apps/app_staging_environment.go

@@ -32,6 +32,8 @@ var _ = AppsDescribe("Buildpack Environment", func() {
 		tmpdir string
 	)
 
+	SkipOnK8s()
+
 	matchingFilename := func(appName string) string {
 		return fmt.Sprintf("buildpack-environment-match-%s", appName)
 	}

apps/buildpack_cache.go

@@ -32,6 +32,8 @@ var _ = AppsDescribe("Buildpack cache", func() {
 		buildpackArchivePath string
 	)
 
+	SkipOnK8s()
+
 	matchingFilename := func(appName string) string {
 		return fmt.Sprintf("buildpack-for-buildpack-cache-test-%s", appName)
 	}

apps/default_environment_variables.go

@@ -62,6 +62,8 @@ exit 1
 		var appName string
 		var buildpackName string
 
+		SkipOnK8s()
+
 		BeforeEach(func() {
 			appName = random_name.CATSRandomName("APP")
 		})

apps/environment_variables_group.go

@@ -102,6 +102,8 @@ exit 1
 		var buildpackName string
 		var envVarName string
 
+		SkipOnK8s()
+
 		BeforeEach(func() {
 			appName = random_name.CATSRandomName("APP")
 			envVarName = fmt.Sprintf("CATS_STAGING_TEST_VAR_%s", strconv.Itoa(int(time.Now().UnixNano())))

cats_suite_helpers/cats_suite_helpers.go

@@ -28,6 +28,14 @@ var (
 	SftpPath  string
 )
 
+func SkipOnK8s() {
+	BeforeEach(func() {
+		if Config.RunningOnK8s() {
+			Skip(skip_messages.SkipK8sMessage)
+		}
+	})
+}
+
 func AppsDescribe(description string, callback func()) bool {
 	return Describe("[apps]", func() {
 		BeforeEach(func() {
@@ -246,6 +254,7 @@ func CredhubDescribe(description string, callback func()) bool {
 				Skip(skip_messages.SkipCredhubMessage)
 			}
 		})
+		SkipOnK8s()
 		Describe(description, callback)
 	})
 }
@@ -282,6 +291,7 @@ func WindowsCredhubDescribe(description string, callback func()) bool {
 				Skip(skip_messages.SkipCredhubMessage)
 			}
 		})
+		SkipOnK8s()
 		Describe(description, callback)
 	})
 }
@@ -315,6 +325,7 @@ func WindowsDescribe(description string, callback func()) bool {
 				Skip(skip_messages.SkipWindowsMessage)
 			}
 		})
+		SkipOnK8s()
 		Describe(description, callback)
 	})
 }

credhub/service_bindings.go

@@ -134,6 +134,9 @@ var _ = CredhubDescribe("service bindings", func() {
 
 			tmpdir string
 		)
+
+		SkipOnK8s()
+
 		BeforeEach(func() {
 			workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() {
 				buildpackName = random_name.CATSRandomName("BPK")

helpers/config/config.go

@@ -94,6 +94,8 @@ type CatsConfig interface {
 	SleepTimeoutDuration() time.Duration
 
 	GetPublicDockerAppImage() string
+
+	RunningOnK8s() bool
 }
 
 func NewCatsConfig(path string) (CatsConfig, error) {

helpers/config/config_struct.go

@@ -112,6 +112,8 @@ type config struct {
 	NamePrefix *string `json:"name_prefix"`
 
 	ReporterConfig *reporterConfig `json:"reporter_config"`
+
+	Infrastructure *string `json:"infrastructure"`
 }
 
 type reporterConfig struct {
@@ -223,6 +225,8 @@ func getDefaults() config {
 	defaults.NamePrefix = ptrToString("CATS")
 
 	defaults.Stacks = &[]string{"cflinuxfs3"}
+
+	defaults.Infrastructure = ptrToString("vms")
 	return defaults
 }
 
@@ -447,6 +451,9 @@ func validateConfig(config *config) Errors {
 	if config.NamePrefix == nil {
 		errs.Add(fmt.Errorf("* 'name_prefix' must not be null"))
 	}
+	if config.Infrastructure == nil {
+		errs.Add(fmt.Errorf("* 'infrastructure' must not be null"))
+	}
 
 	return errs
 }
@@ -1032,3 +1039,7 @@ func (c *config) GetReporterConfig() reporterConfig {
 
 	return reporterConfig{}
 }
+
+func (c *config) RunningOnK8s() bool {
+	return *c.Infrastructure == "kubernetes"
+}

helpers/config/config_test.go

@@ -71,6 +71,8 @@ type testConfig struct {
 	ReporterConfig *testReporterConfig `json:"reporter_config"`
 
 	Stacks *[]string `json:"stacks,omitempty"`
+
+	Infrastructure *string `json:"infrastructure"`
 }
 
 type allConfig struct {
@@ -156,6 +158,8 @@ type allConfig struct {
 	NamePrefix *string `json:"name_prefix"`
 
 	Stacks *[]string `json:"stacks"`
+
+	Infrastructure *string `json:"infrastructure"`
 }
 
 type testReporterConfig struct {
@@ -210,6 +214,7 @@ var _ = Describe("Config", func() {
 		testCfg.AdminPassword = ptrToString("admin")
 		testCfg.SkipSSLValidation = ptrToBool(true)
 		testCfg.AppsDomain = ptrToString("cf-app.bosh-lite.com")
+		testCfg.Infrastructure = ptrToString("vms")
 	})
 
 	JustBeforeEach(func() {
@@ -311,6 +316,8 @@ var _ = Describe("Config", func() {
 		Expect(config.GetRequireProxiedAppTraffic()).To(BeFalse())
 
 		Expect(config.GetStacks()).To(ConsistOf("cflinuxfs3"))
+
+		Expect(config.RunningOnK8s()).To(BeFalse(), "RunningOnK8s should be false")
 	})
 
 	Context("when all values are null", func() {
@@ -388,6 +395,8 @@ var _ = Describe("Config", func() {
 			Expect(err.Error()).To(ContainSubstring("'name_prefix' must not be null"))
 
 			Expect(err.Error()).To(ContainSubstring("'stacks' must not be null"))
+
+			Expect(err.Error()).To(ContainSubstring("'infrastructure' must not be null"))
 		})
 	})
 
@@ -403,6 +412,7 @@ var _ = Describe("Config", func() {
 			testCfg.TimeoutScale = ptrToFloat(1.0)
 			testCfg.UnallocatedIPForSecurityGroup = ptrToString("192.168.0.1")
 			testCfg.RequireProxiedAppTraffic = ptrToBool(true)
+			testCfg.Infrastructure = ptrToString("kubernetes")
 		})
 
 		It("respects the overriden values", func() {
@@ -419,6 +429,7 @@ var _ = Describe("Config", func() {
 			Expect(config.SleepTimeoutDuration()).To(Equal(101 * time.Second))
 			Expect(config.GetUnallocatedIPForSecurityGroup()).To(Equal("192.168.0.1"))
 			Expect(config.GetRequireProxiedAppTraffic()).To(BeTrue())
+			Expect(config.RunningOnK8s()).To(BeTrue(), "RunningOnK8s should be true")
 		})
 	})
 

helpers/skip_messages/skip_messages.go

@@ -50,3 +50,4 @@ const SkipNoAlternateStacksMessage = `Skipping this test because config.Stacks i
 const SkipVolumeServicesMessage = `Skipping this test because config.IncludeVolumeServices is set to 'false'.
 NOTE: Ensure that volume services are enabled on your platform and volume service broker is registered before running this test.`
 const SkipVolumeServicesDockerEnabledMessage = `Skipping this test because config.IncludeDocker is set to 'true'`
+const SkipK8sMessage = `Skipping this test because config.Infrastructure is set to 'kubernetes'`

security_groups/running_security_groups.go

@@ -285,6 +285,8 @@ var _ = SecurityGroupsDescribe("App Instance Networking", func() {
 	Describe("Using staging security groups", func() {
 		var testAppName, buildpack string
 
+		SkipOnK8s()
+
 		BeforeEach(func() {
 			serverAppName, privateHost, privatePort = pushServerApp()
 

v3/buildpacks.go

@@ -41,6 +41,8 @@ var _ = V3Describe("buildpack", func() {
 		Version       string `json:"version"`
 	}
 
+	SkipOnK8s()
+
 	Context("With a single buildpack app", func() {
 		BeforeEach(func() {
 			appName = random_name.CATSRandomName("APP")