From 9e58265f61b43293a50eb30105780659696f400d Mon Sep 17 00:00:00 2001 From: timflannagan Date: Fri, 10 Jan 2025 21:28:56 +0000 Subject: [PATCH] Remove the test/consulvaulte2e e2e suite The test/consulvaulte2e suite was one of the test suites that cloudbuilder was configured to run on PRs. See https://github.com/k8sgateway/k8sgateway/issues/10432 for the issue that tracks whether we should migrate the other suite being run (e.g. run-e2e-tests) and whether it's still relevant for this project. Signed-off-by: timflannagan --- test/consulvaulte2e/README.md | 14 - test/consulvaulte2e/consul_vault_test.go | 381 ----------------------- test/consulvaulte2e/e2e_suite_test.go | 65 ---- 3 files changed, 460 deletions(-) delete mode 100644 test/consulvaulte2e/README.md delete mode 100644 test/consulvaulte2e/consul_vault_test.go delete mode 100644 test/consulvaulte2e/e2e_suite_test.go diff --git a/test/consulvaulte2e/README.md b/test/consulvaulte2e/README.md deleted file mode 100644 index 9f8496cea0ae..000000000000 --- a/test/consulvaulte2e/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Consul/Vault Tests - -## Setup -The consul vault test downloads and runs vault and is disabled by default. To enable, set `RUN_VAULT_TESTS=1` and `RUN_CONSUL_TESTS=1` in your local environment. - -## Note to developers: -These tests set up and run Gloo with a different than normal path for generating the runtime options. - -If you have made changes to the setup loop and these tests are suddenly failing, you may need to make corresponding -changes here: -https://github.com/solo-io/gloo/blob/61d35b0d4ce3b2b28ed47c7be06d9acaadf37074/test/services/gateway.go#L249 - -## TODO: Instructions for running locally - diff --git a/test/consulvaulte2e/consul_vault_test.go b/test/consulvaulte2e/consul_vault_test.go deleted file mode 100644 index 05a64fa93220..000000000000 --- a/test/consulvaulte2e/consul_vault_test.go +++ /dev/null @@ -1,381 +0,0 @@ -package consulvaulte2e_test - -import ( - "context" - "flag" - "fmt" - "os" - "path/filepath" - "time" - - "github.com/solo-io/gloo/test/ginkgo/decorators" - - "github.com/solo-io/gloo/test/services/envoy" - - "google.golang.org/protobuf/types/known/wrapperspb" - - gatewaydefaults "github.com/solo-io/gloo/projects/gateway/pkg/defaults" - "github.com/solo-io/solo-kit/pkg/api/v1/resources" - "github.com/solo-io/solo-kit/pkg/utils/prototime" - - fdssetup "github.com/solo-io/gloo/projects/discovery/pkg/fds/setup" - udssetup "github.com/solo-io/gloo/projects/discovery/pkg/uds/setup" - "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/options/rest" - - consulapi "github.com/hashicorp/consul/api" - vaultapi "github.com/hashicorp/vault/api" - v1 "github.com/solo-io/gloo/projects/gateway/pkg/api/v1" - "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/ssl" - bootstrap "github.com/solo-io/gloo/projects/gloo/pkg/bootstrap/clients" - "github.com/solo-io/gloo/projects/gloo/pkg/bootstrap/clients/vault" - "github.com/solo-io/gloo/projects/gloo/pkg/setup" - "github.com/solo-io/gloo/test/helpers" - "github.com/solo-io/gloo/test/v1helpers" - "github.com/solo-io/solo-kit/pkg/api/v1/clients/factory" - "github.com/solo-io/solo-kit/pkg/utils/protoutils" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1" - "github.com/solo-io/gloo/projects/gloo/pkg/defaults" - "github.com/solo-io/gloo/test/services" - "github.com/solo-io/solo-kit/pkg/api/v1/clients" - "github.com/solo-io/solo-kit/pkg/api/v1/resources/core" -) - -var _ = Describe("Consul + Vault Configuration Happy Path e2e", decorators.Vault, decorators.Consul, func() { - - var ( - ctx context.Context - cancel context.CancelFunc - consulInstance *services.ConsulInstance - vaultInstance *services.VaultInstance - envoyInstance *envoy.Instance - svc1 *v1helpers.TestUpstream - err error - settingsDir string - - consulClient *consulapi.Client - vaultClient *vaultapi.Client - consulResources factory.ResourceClientFactory - vaultResources factory.ResourceClientFactory - - petstorePort int - glooPort int - validationPort int - restXdsPort int - proxyDebugPort int - ) - - const writeNamespace = defaults.GlooSystem - const customSecretEngine = "custom-secret-engine" - - BeforeEach(func() { - ctx, cancel = context.WithCancel(context.Background()) - - glooPort = int(services.AllocateGlooPort()) - validationPort = int(services.AllocateGlooPort()) - restXdsPort = int(services.AllocateGlooPort()) - proxyDebugPort = int(services.AllocateGlooPort()) - - // Start Consul - consulInstance, err = consulFactory.NewConsulInstance() - Expect(err).NotTo(HaveOccurred()) - err = consulInstance.Run(ctx) - Expect(err).NotTo(HaveOccurred()) - - // Start Vault - vaultInstance, err = vaultFactory.NewVaultInstance() - Expect(err).NotTo(HaveOccurred()) - err = vaultInstance.Run(ctx) - Expect(err).NotTo(HaveOccurred()) - err = vaultInstance.EnableSecretEngine(customSecretEngine) - Expect(err).NotTo(HaveOccurred()) - - envoyInstance = envoyFactory.NewInstance() - - vaultSecretSource := getVaultSecretSource(vaultInstance, customSecretEngine) - - // write settings telling Gloo to use consul/vault - settingsDir, err = os.MkdirTemp("", "") - Expect(err).NotTo(HaveOccurred()) - - settings, err := writeSettings(settingsDir, glooPort, validationPort, restXdsPort, proxyDebugPort, writeNamespace, vaultSecretSource) - Expect(err).NotTo(HaveOccurred()) - - consulClient, err = bootstrap.ConsulClientForSettings(ctx, settings) - Expect(err).NotTo(HaveOccurred()) - - vaultClient, err = vault.VaultClientForSettings(ctx, vaultSecretSource) - Expect(err).NotTo(HaveOccurred()) - - consulResources = &factory.ConsulResourceClientFactory{ - RootKey: bootstrap.DefaultRootKey, - Consul: consulClient, - QueryOptions: &consulapi.QueryOptions{RequireConsistent: true}, - } - - gatewayClient, err := v1.NewGatewayClient(ctx, consulResources) - Expect(err).NotTo(HaveOccurred(), "Should be able to build the gateway client") - err = helpers.WriteDefaultGateways(writeNamespace, gatewayClient) - Expect(err).NotTo(HaveOccurred(), "Should be able to write the default gateways") - - vaultResources = vault.NewVaultSecretClientFactory(ctx, vault.NoopVaultClientInitFunc(vaultClient), customSecretEngine, bootstrap.DefaultRootKey) - - // set flag for gloo to use settings dir - err = flag.Set("dir", settingsDir) - err = flag.Set("namespace", writeNamespace) - Expect(err).NotTo(HaveOccurred()) - go func() { - defer GinkgoRecover() - - // Start Gloo - err = setup.Main(ctx) - Expect(err).NotTo(HaveOccurred()) - }() - go func() { - defer GinkgoRecover() - // Start FDS - err = fdssetup.Main(ctx) - Expect(err).NotTo(HaveOccurred()) - }() - go func() { - defer GinkgoRecover() - // Start UDS - err = udssetup.Main(ctx) - Expect(err).NotTo(HaveOccurred()) - }() - - // Start Envoy - err = envoyInstance.RunWithRoleAndRestXds(writeNamespace+"~"+gatewaydefaults.GatewayProxyName, glooPort, restXdsPort) - Expect(err).NotTo(HaveOccurred()) - - // Run a simple web application locally - svc1 = v1helpers.NewTestHttpUpstream(ctx, envoyInstance.LocalAddr()) - - // Run the petstore locally - petstorePort = 1234 - go func() { - defer GinkgoRecover() - // Start petstore - err = services.RunPetstore(ctx, petstorePort) - if err != nil { - Expect(err.Error()).To(ContainSubstring("http: Server closed")) - } - }() - - // Register services with consul - err = consulInstance.RegisterService("my-svc", "my-svc-1", envoyInstance.GlooAddr, []string{"svc", "1"}, svc1.Port) - Expect(err).NotTo(HaveOccurred()) - - err = consulInstance.RegisterService("petstore", "petstore-1", envoyInstance.GlooAddr, []string{"svc", "petstore"}, uint32(petstorePort)) - Expect(err).NotTo(HaveOccurred()) - }) - - AfterEach(func() { - envoyInstance.Clean() - - os.RemoveAll(settingsDir) - - cancel() - }) - - It("can be configured using consul k-v and read secrets using vault", func() { - cert := helpers.Certificate() - - secret := &gloov1.Secret{ - Metadata: &core.Metadata{ - Name: "secret", - Namespace: "default", - }, - Kind: &gloov1.Secret_Tls{ - Tls: &gloov1.TlsSecret{ - CertChain: cert, - PrivateKey: helpers.PrivateKey(), - }, - }, - } - - secretClient, err := gloov1.NewSecretClient(ctx, vaultResources) - Expect(err).NotTo(HaveOccurred()) - - _, err = secretClient.Write(secret, clients.WriteOpts{Ctx: ctx}) - Expect(err).NotTo(HaveOccurred()) - - vsClient, err := v1.NewVirtualServiceClient(ctx, consulResources) - Expect(err).NotTo(HaveOccurred()) - - proxyClient, err := gloov1.NewProxyClient(ctx, consulResources) - Expect(err).NotTo(HaveOccurred()) - - vs := makeSslVirtualService(writeNamespace, secret.Metadata.Ref()) - - vs, err = vsClient.Write(vs, clients.WriteOpts{Ctx: ctx}) - Expect(err).NotTo(HaveOccurred()) - - // Wait for vs and gw to be accepted - helpers.EventuallyResourceAccepted(func() (resources.InputResource, error) { - return vsClient.Read(vs.Metadata.Namespace, vs.Metadata.Name, clients.ReadOpts{Ctx: ctx}) - }, "60s", ".2s") - - // Wait for the proxy to be accepted. this can take up to 40 seconds, as the vault snapshot - // updates every 30 seconds. - helpers.EventuallyResourceAccepted(func() (resources.InputResource, error) { - return proxyClient.Read(writeNamespace, gatewaydefaults.GatewayProxyName, clients.ReadOpts{Ctx: ctx}) - }, "60s", ".2s") - - v1helpers.TestUpstreamReachable(envoyInstance.HttpsPort, svc1, &cert) - }) - It("can do function routing with consul services", func() { - - vsClient, err := v1.NewVirtualServiceClient(ctx, consulResources) - Expect(err).NotTo(HaveOccurred()) - - proxyClient, err := gloov1.NewProxyClient(ctx, consulResources) - Expect(err).NotTo(HaveOccurred()) - - us := &core.ResourceRef{Namespace: "gloo-system", Name: "petstore"} - - vs := makeFunctionRoutingVirtualService(writeNamespace, us, "findPetById") - - vs, err = vsClient.Write(vs, clients.WriteOpts{Ctx: ctx}) - Expect(err).NotTo(HaveOccurred()) - - // Wait for the proxy to be accepted. - helpers.EventuallyResourceAccepted(func() (resources.InputResource, error) { - return proxyClient.Read(writeNamespace, gatewaydefaults.GatewayProxyName, clients.ReadOpts{Ctx: ctx}) - }, "60s", ".2s") - - v1helpers.ExpectHttpOK(nil, nil, envoyInstance.HttpPort, - `[{"id":1,"name":"Dog","status":"available"},{"id":2,"name":"Cat","status":"pending"}] -`) - }) -}) - -func makeSslVirtualService(vsNamespace string, secret *core.ResourceRef) *v1.VirtualService { - return &v1.VirtualService{ - Metadata: &core.Metadata{ - Name: "vs-ssl", - Namespace: vsNamespace, - }, - VirtualHost: &v1.VirtualHost{ - Domains: []string{"*"}, - Routes: []*v1.Route{{ - Action: &v1.Route_RouteAction{ - RouteAction: &gloov1.RouteAction{ - Destination: &gloov1.RouteAction_Single{ - Single: &gloov1.Destination{ - DestinationType: &gloov1.Destination_Consul{ - Consul: &gloov1.ConsulServiceDestination{ - ServiceName: "my-svc", - Tags: []string{"svc", "1"}, - }, - }, - }, - }, - }, - }, - }}, - }, - SslConfig: &ssl.SslConfig{ - SslSecrets: &ssl.SslConfig_SecretRef{ - SecretRef: &core.ResourceRef{ - Name: secret.Name, - Namespace: secret.Namespace, - }, - }, - }, - } -} - -func makeFunctionRoutingVirtualService(vsNamespace string, upstream *core.ResourceRef, funcName string) *v1.VirtualService { - return &v1.VirtualService{ - Metadata: &core.Metadata{ - Name: "vs-functions", - Namespace: vsNamespace, - }, - VirtualHost: &v1.VirtualHost{ - Domains: []string{"*"}, - Routes: []*v1.Route{{ - Action: &v1.Route_RouteAction{ - RouteAction: &gloov1.RouteAction{ - Destination: &gloov1.RouteAction_Single{ - Single: &gloov1.Destination{ - DestinationType: &gloov1.Destination_Upstream{ - Upstream: upstream, - }, - DestinationSpec: &gloov1.DestinationSpec{ - DestinationType: &gloov1.DestinationSpec_Rest{ - Rest: &rest.DestinationSpec{ - FunctionName: funcName, - }, - }, - }, - }, - }, - }, - }, - }}, - }, - } -} - -func getVaultSecretSource(vaultInstance *services.VaultInstance, secretEngine string) *gloov1.Settings_VaultSecrets { - return &gloov1.Settings_VaultSecrets{ - Address: vaultInstance.Address(), - Token: vaultInstance.Token(), - PathPrefix: secretEngine, - RootKey: bootstrap.DefaultRootKey, - } -} - -func writeSettings( - settingsDir string, - glooPort, validationPort, restXdsPort, proxyDebugPort int, - writeNamespace string, - vaultSecretSource *gloov1.Settings_VaultSecrets, -) (*gloov1.Settings, error) { - settings := &gloov1.Settings{ - ConfigSource: &gloov1.Settings_ConsulKvSource{ - ConsulKvSource: &gloov1.Settings_ConsulKv{}, - }, - SecretSource: &gloov1.Settings_VaultSecretSource{ - VaultSecretSource: vaultSecretSource, - }, - ArtifactSource: &gloov1.Settings_DirectoryArtifactSource{ - DirectoryArtifactSource: &gloov1.Settings_Directory{ - Directory: settingsDir, - }, - }, - Discovery: &gloov1.Settings_DiscoveryOptions{ - FdsMode: gloov1.Settings_DiscoveryOptions_BLACKLIST, - }, - Consul: &gloov1.Settings_ConsulConfiguration{ - ServiceDiscovery: &gloov1.Settings_ConsulConfiguration_ServiceDiscoveryOptions{}, - }, - Gloo: &gloov1.GlooOptions{ - XdsBindAddr: fmt.Sprintf("0.0.0.0:%v", glooPort), - ValidationBindAddr: fmt.Sprintf("0.0.0.0:%v", validationPort), - RestXdsBindAddr: fmt.Sprintf("0.0.0.0:%v", restXdsPort), - ProxyDebugBindAddr: fmt.Sprintf("0.0.0.0:%v", proxyDebugPort), - }, - Gateway: &gloov1.GatewayOptions{ - PersistProxySpec: &wrapperspb.BoolValue{Value: true}, - }, - RefreshRate: prototime.DurationToProto(time.Second * 1), - DiscoveryNamespace: writeNamespace, - Metadata: &core.Metadata{Namespace: writeNamespace, Name: "default"}, - } - yam, err := protoutils.MarshalYAML(settings) - if err != nil { - return nil, err - } - if err := os.MkdirAll(filepath.Join(settingsDir, writeNamespace), 0755); err != nil { - return nil, err - } - // must create a directory for artifacts so gloo doesn't error - if err := os.MkdirAll(filepath.Join(settingsDir, "artifacts", "default"), 0755); err != nil { - return nil, err - } - return settings, os.WriteFile(filepath.Join(settingsDir, writeNamespace, "default.yaml"), yam, 0644) -} diff --git a/test/consulvaulte2e/e2e_suite_test.go b/test/consulvaulte2e/e2e_suite_test.go deleted file mode 100644 index 7a170ebb6c5a..000000000000 --- a/test/consulvaulte2e/e2e_suite_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package consulvaulte2e_test - -import ( - "os" - "testing" - - "github.com/solo-io/gloo/projects/gloo/constants" - - "github.com/solo-io/gloo/test/services/envoy" - - "github.com/solo-io/gloo/test/ginkgo/labels" - - testhelpers "github.com/solo-io/gloo/test/testutils" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/solo-io/gloo/test/services" - "github.com/solo-io/solo-kit/test/helpers" -) - -func TestE2e(t *testing.T) { - // set KUBECONFIG to a nonexistent cfg. - // this way we are also testing that Gloo can run without a kubeconfig present - os.Setenv("KUBECONFIG", ".") - - helpers.RegisterCommonFailHandlers() - helpers.SetupLog() - - RunSpecs(t, "Consul+Vault E2e Suite", Label(labels.E2E)) -} - -var ( - envoyFactory envoy.Factory - consulFactory *services.ConsulFactory - vaultFactory *services.VaultFactory -) - -var _ = BeforeSuite(func() { - testhelpers.ValidateRequirementsAndNotifyGinkgo( - testhelpers.Consul(), - testhelpers.Vault(), - ) - - var err error - envoyFactory = envoy.NewFactory() - - consulFactory, err = services.NewConsulFactory() - Expect(err).NotTo(HaveOccurred()) - vaultFactory, err = services.NewVaultFactory() - Expect(err).NotTo(HaveOccurred()) - - // The consulvaulte2e test suite is not run against a k8s cluster, so we must disable the features that require a k8s cluster - err = os.Setenv(constants.GlooGatewayEnableK8sGwControllerEnv, "false") - Expect(err).NotTo(HaveOccurred()) -}) - -var _ = AfterSuite(func() { - envoyFactory.Clean() - _ = consulFactory.Clean() - _ = vaultFactory.Clean() - - err := os.Unsetenv(constants.GlooGatewayEnableK8sGwControllerEnv) - Expect(err).NotTo(HaveOccurred()) -})