diff --git a/changelog/v1.18.4/7505-gw-api-cipher-suites.yaml b/changelog/v1.18.4/7505-gw-api-cipher-suites.yaml new file mode 100644 index 00000000000..74f6654d9bc --- /dev/null +++ b/changelog/v1.18.4/7505-gw-api-cipher-suites.yaml @@ -0,0 +1,9 @@ + +changelog: +- type: FIX + issueLink: https://github.com/solo-io/solo-projects/issues/7505 + resolvesIssue: true + description: > + Add new SSL options to GatewayTLSConfig to enable configuring additional SSL options + which were previously available using the edge API. This includes cipher suites, minimum TLS version, + maximum TLS version, client certificate validation, and one way TLS. diff --git a/projects/gateway2/translator/listener/gateway_listener_translator.go b/projects/gateway2/translator/listener/gateway_listener_translator.go index 2997f48dfc6..8a028921b2e 100644 --- a/projects/gateway2/translator/listener/gateway_listener_translator.go +++ b/projects/gateway2/translator/listener/gateway_listener_translator.go @@ -750,7 +750,7 @@ func translateSslConfig( if sniDomain != nil { sniDomains = []string{string(*sniDomain)} } - return &ssl.SslConfig{ + cfg := &ssl.SslConfig{ SslSecrets: &ssl.SslConfig_SecretRef{SecretRef: secretRef}, SniDomains: sniDomains, VerifySubjectAltName: nil, @@ -760,7 +760,12 @@ func translateSslConfig( DisableTlsSessionResumption: nil, TransportSocketConnectTimeout: nil, OcspStaplePolicy: 0, - }, nil + } + + // Apply known SSL Extension options + sslutils.ApplySslExtensionOptions(ctx, tls, cfg) + + return cfg, nil } // makeVhostName computes the name of a virtual host based on the parent name and domain. diff --git a/projects/gateway2/translator/sslutils/ssl_utils.go b/projects/gateway2/translator/sslutils/ssl_utils.go index 193e942b8c5..d34444392b0 100644 --- a/projects/gateway2/translator/sslutils/ssl_utils.go +++ b/projects/gateway2/translator/sslutils/ssl_utils.go @@ -1,12 +1,31 @@ package sslutils import ( + "context" "crypto/tls" "fmt" + "strings" + "github.com/hashicorp/go-multierror" "github.com/rotisserie/eris" + "github.com/solo-io/gloo/projects/gateway2/wellknown" + "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/ssl" + "github.com/solo-io/go-utils/contextutils" + "google.golang.org/protobuf/types/known/wrapperspb" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/util/cert" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// Gateway API has an extension point for implementation specific tls settings, they can be found [here](https://gateway-api.sigs.k8s.io/guides/tls/#extensions) +const ( + GatewaySslOptionsPrefix = wellknown.GatewayAnnotationPrefix + "/ssl" + + GatewaySslCipherSuites = GatewaySslOptionsPrefix + "/cipher-suites" + GatewaySslMinimumTlsVersion = GatewaySslOptionsPrefix + "/minimum-tls-version" + GatewaySslMaximumTlsVersion = GatewaySslOptionsPrefix + "/maximum-tls-version" + GatewaySslOneWayTls = GatewaySslOptionsPrefix + "/one-way-tls" + GatewaySslVerifySubjectAltName = GatewaySslOptionsPrefix + "/verify-subject-alt-name" ) var ( @@ -59,3 +78,95 @@ func cleanedSslKeyPair(certChain, privateKey, rootCa string) (cleanedChain strin return cleanedChain, err } + +type SslExtensionOptionFunc = func(ctx context.Context, in string, out *ssl.SslConfig) error + +func ApplyCipherSuites(ctx context.Context, in string, out *ssl.SslConfig) error { + if out.GetParameters() == nil { + out.Parameters = &ssl.SslParameters{} + } + cipherSuites := strings.Split(in, ",") + out.GetParameters().CipherSuites = cipherSuites + return nil +} + +func ApplyMinimumTlsVersion(ctx context.Context, in string, out *ssl.SslConfig) error { + if out.GetParameters() == nil { + out.Parameters = &ssl.SslParameters{} + } + if parsed, ok := ssl.SslParameters_ProtocolVersion_value[in]; ok { + out.GetParameters().MinimumProtocolVersion = ssl.SslParameters_ProtocolVersion(parsed) + if out.GetParameters().GetMaximumProtocolVersion() != ssl.SslParameters_TLS_AUTO && out.GetParameters().GetMaximumProtocolVersion() < out.GetParameters().GetMinimumProtocolVersion() { + err := eris.Errorf("maximum tls version %s is less than minimum tls version %s", out.GetParameters().GetMaximumProtocolVersion().String(), in) + out.GetParameters().MaximumProtocolVersion = ssl.SslParameters_TLS_AUTO + out.GetParameters().MinimumProtocolVersion = ssl.SslParameters_TLS_AUTO + return err + } + } else { + return eris.Errorf("invalid minimum tls version: %s", in) + } + return nil +} + +func ApplyMaximumTlsVersion(ctx context.Context, in string, out *ssl.SslConfig) error { + if out.GetParameters() == nil { + out.Parameters = &ssl.SslParameters{} + } + if parsed, ok := ssl.SslParameters_ProtocolVersion_value[in]; ok { + out.GetParameters().MaximumProtocolVersion = ssl.SslParameters_ProtocolVersion(parsed) + if out.GetParameters().GetMaximumProtocolVersion() != ssl.SslParameters_TLS_AUTO && out.GetParameters().GetMaximumProtocolVersion() < out.GetParameters().GetMinimumProtocolVersion() { + err := eris.Errorf("maximum tls version %s is less than minimum tls version %s", in, out.GetParameters().GetMinimumProtocolVersion().String()) + out.GetParameters().MaximumProtocolVersion = ssl.SslParameters_TLS_AUTO + out.GetParameters().MinimumProtocolVersion = ssl.SslParameters_TLS_AUTO + return err + } + } else { + return eris.Errorf("invalid maximum tls version: %s", in) + } + return nil +} + +func ApplyOneWayTls(ctx context.Context, in string, out *ssl.SslConfig) error { + if strings.ToLower(in) == "true" { + out.OneWayTls = wrapperspb.Bool(true) + } else if strings.ToLower(in) == "false" { + out.OneWayTls = wrapperspb.Bool(false) + } else { + return eris.Errorf("invalid value for one-way-tls: %s", in) + } + return nil +} + +func ApplyVerifySubjectAltName(ctx context.Context, in string, out *ssl.SslConfig) error { + altNames := strings.Split(in, ",") + out.VerifySubjectAltName = altNames + return nil +} + +var SslExtensionOptionFuncs = map[string]SslExtensionOptionFunc{ + GatewaySslCipherSuites: ApplyCipherSuites, + GatewaySslMinimumTlsVersion: ApplyMinimumTlsVersion, + GatewaySslMaximumTlsVersion: ApplyMaximumTlsVersion, + GatewaySslOneWayTls: ApplyOneWayTls, + GatewaySslVerifySubjectAltName: ApplyVerifySubjectAltName, +} + +// ApplySslExtensionOptions applies the GatewayTLSConfig options to the SslConfig +// This function will never exit early, even if an error is encountered. +// It will apply all options and log all errors encountered. +func ApplySslExtensionOptions(ctx context.Context, in *gwv1.GatewayTLSConfig, out *ssl.SslConfig) { + var wrapped error + for key, option := range in.Options { + if extensionFunc, ok := SslExtensionOptionFuncs[string(key)]; ok { + if err := extensionFunc(ctx, string(option), out); err != nil { + wrapped = multierror.Append(wrapped, err) + } + } else { + wrapped = multierror.Append(wrapped, eris.Errorf("unknown ssl option: %s", key)) + } + } + + if wrapped != nil { + contextutils.LoggerFrom(ctx).Warnf("error applying ssl extension options: %v", wrapped) + } +} diff --git a/projects/gateway2/translator/sslutils/ssl_utils_test.go b/projects/gateway2/translator/sslutils/ssl_utils_test.go new file mode 100644 index 00000000000..b66794bc1ef --- /dev/null +++ b/projects/gateway2/translator/sslutils/ssl_utils_test.go @@ -0,0 +1,221 @@ +package sslutils + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/ssl" + "github.com/solo-io/go-utils/contextutils" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/wrapperspb" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +func TestApplySslExtensionOptions(t *testing.T) { + testCases := []struct { + name string + out *ssl.SslConfig + in *gwv1.GatewayTLSConfig + errors []string + }{ + { + name: "one_way_tls_true", + out: &ssl.SslConfig{ + OneWayTls: wrapperspb.Bool(true), + }, + in: &gwv1.GatewayTLSConfig{ + Options: map[gwv1.AnnotationKey]gwv1.AnnotationValue{ + GatewaySslOneWayTls: "true", + }, + }, + }, + { + name: "one_way_tls_true_incorrect_casing", + out: &ssl.SslConfig{ + OneWayTls: wrapperspb.Bool(true), + }, + in: &gwv1.GatewayTLSConfig{ + Options: map[gwv1.AnnotationKey]gwv1.AnnotationValue{ + GatewaySslOneWayTls: "True", + }, + }, + }, + { + name: "one_way_tls_false", + out: &ssl.SslConfig{ + OneWayTls: wrapperspb.Bool(false), + }, + in: &gwv1.GatewayTLSConfig{ + Options: map[gwv1.AnnotationKey]gwv1.AnnotationValue{ + GatewaySslOneWayTls: "false", + }, + }, + }, + { + name: "one_way_tls_false_incorrect_casing", + out: &ssl.SslConfig{ + OneWayTls: wrapperspb.Bool(false), + }, + in: &gwv1.GatewayTLSConfig{ + Options: map[gwv1.AnnotationKey]gwv1.AnnotationValue{ + GatewaySslOneWayTls: "False", + }, + }, + }, + { + name: "invalid_one_way_tls", + out: &ssl.SslConfig{}, + in: &gwv1.GatewayTLSConfig{ + Options: map[gwv1.AnnotationKey]gwv1.AnnotationValue{ + GatewaySslOneWayTls: "Foo", + }, + }, + errors: []string{"invalid value for one-way-tls: Foo"}, + }, + { + name: "cipher_suites", + out: &ssl.SslConfig{ + Parameters: &ssl.SslParameters{ + CipherSuites: []string{"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"}, + }, + }, + in: &gwv1.GatewayTLSConfig{ + Options: map[gwv1.AnnotationKey]gwv1.AnnotationValue{ + GatewaySslCipherSuites: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + }, + }, + }, + { + name: "subject_alt_names", + out: &ssl.SslConfig{ + VerifySubjectAltName: []string{"foo", "bar"}, + }, + in: &gwv1.GatewayTLSConfig{ + Options: map[gwv1.AnnotationKey]gwv1.AnnotationValue{ + GatewaySslVerifySubjectAltName: "foo,bar", + }, + }, + }, + { + name: "tls_max_version", + out: &ssl.SslConfig{ + Parameters: &ssl.SslParameters{ + MaximumProtocolVersion: ssl.SslParameters_TLSv1_2, + }, + }, + in: &gwv1.GatewayTLSConfig{ + Options: map[gwv1.AnnotationKey]gwv1.AnnotationValue{ + GatewaySslMaximumTlsVersion: "TLSv1_2", + }, + }, + }, + { + name: "tls_min_version", + out: &ssl.SslConfig{ + Parameters: &ssl.SslParameters{ + MinimumProtocolVersion: ssl.SslParameters_TLSv1_3, + }, + }, + in: &gwv1.GatewayTLSConfig{ + Options: map[gwv1.AnnotationKey]gwv1.AnnotationValue{ + GatewaySslMinimumTlsVersion: "TLSv1_3", + }, + }, + }, + { + name: "invalid_tls_versions", + out: &ssl.SslConfig{ + Parameters: &ssl.SslParameters{}, + }, + in: &gwv1.GatewayTLSConfig{ + Options: map[gwv1.AnnotationKey]gwv1.AnnotationValue{ + GatewaySslMinimumTlsVersion: "TLSv1.3", + GatewaySslMaximumTlsVersion: "TLSv1.2", + }, + }, + errors: []string{ + "invalid maximum tls version: TLSv1.2", + "invalid minimum tls version: TLSv1.3", + }, + }, + { + name: "maximium_tls_version_less_than_minimum", + out: &ssl.SslConfig{ + VerifySubjectAltName: []string{"foo", "bar"}, + Parameters: &ssl.SslParameters{}, + }, + in: &gwv1.GatewayTLSConfig{ + Options: map[gwv1.AnnotationKey]gwv1.AnnotationValue{ + GatewaySslMinimumTlsVersion: "TLSv1_3", + GatewaySslMaximumTlsVersion: "TLSv1_2", + GatewaySslVerifySubjectAltName: "foo,bar", + }, + }, + errors: []string{ + "maximum tls version TLSv1_2 is less than minimum tls version TLSv1_3", + }, + }, + { + name: "multiple_options", + out: &ssl.SslConfig{ + VerifySubjectAltName: []string{"foo", "bar"}, + OneWayTls: wrapperspb.Bool(true), + Parameters: &ssl.SslParameters{ + MaximumProtocolVersion: ssl.SslParameters_TLSv1_3, + MinimumProtocolVersion: ssl.SslParameters_TLSv1_2, + CipherSuites: []string{"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"}, + }, + }, + in: &gwv1.GatewayTLSConfig{ + Options: map[gwv1.AnnotationKey]gwv1.AnnotationValue{ + GatewaySslMaximumTlsVersion: "TLSv1_3", + GatewaySslMinimumTlsVersion: "TLSv1_2", + GatewaySslVerifySubjectAltName: "foo,bar", + GatewaySslOneWayTls: "true", + GatewaySslCipherSuites: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + }, + }, + }, + { + name: "misspelled_option", + out: &ssl.SslConfig{}, + in: &gwv1.GatewayTLSConfig{ + Options: map[gwv1.AnnotationKey]gwv1.AnnotationValue{ + GatewaySslMinimumTlsVersion + "s": "TLSv1_3", + }, + }, + errors: []string{ + "unknown ssl option: gateway.gloo.solo.io/ssl/minimum-tls-versions", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + b := &zaptest.Buffer{} + logger := zap.New(zapcore.NewCore( + zapcore.NewJSONEncoder(zap.NewDevelopmentEncoderConfig()), + b, + zapcore.DebugLevel, + )) + ctx := contextutils.WithExistingLogger(context.Background(), logger.Sugar()) + out := &ssl.SslConfig{} + ApplySslExtensionOptions(ctx, tc.in, out) + assert.Empty(t, cmp.Diff(tc.out, out, protocmp.Transform())) + if len(tc.errors) > 0 { + assert.Contains(t, b.String(), "error applying ssl extension options") + for _, err := range tc.errors { + assert.Contains(t, b.String(), err) + } + } else { + assert.Empty(t, b.String()) + } + }) + + } +} diff --git a/projects/gateway2/wellknown/controller.go b/projects/gateway2/wellknown/controller.go index dbbd3153854..6f2444d2c98 100644 --- a/projects/gateway2/wellknown/controller.go +++ b/projects/gateway2/wellknown/controller.go @@ -8,11 +8,14 @@ const ( // It is configured to manage GatewayClasses with the name GatewayClassName GatewayControllerName = "solo.io/gloo-gateway" + // GatewayAnnotationPrefix is the prefix for all Gateway annotations/options + GatewayAnnotationPrefix = "gateway.gloo.solo.io" + // GatewayParametersAnnotationName is the name of the Gateway annotation that specifies // the name of a GatewayParameters CR, which is used to dynamically provision the data plane // resources for the Gateway. The GatewayParameters is assumed to be in the same namespace // as the Gateway. - GatewayParametersAnnotationName = "gateway.gloo.solo.io/gateway-parameters-name" + GatewayParametersAnnotationName = GatewayAnnotationPrefix + "/gateway-parameters-name" // DefaultGatewayParametersName is the name of the GatewayParameters which is attached by // parametersRef to the GatewayClass. diff --git a/test/kubernetes/e2e/features/server_tls/suite.go b/test/kubernetes/e2e/features/server_tls/edge_suite.go similarity index 87% rename from test/kubernetes/e2e/features/server_tls/suite.go rename to test/kubernetes/e2e/features/server_tls/edge_suite.go index 9d28d0049f9..627745fabd0 100644 --- a/test/kubernetes/e2e/features/server_tls/suite.go +++ b/test/kubernetes/e2e/features/server_tls/edge_suite.go @@ -14,15 +14,14 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "github.com/solo-io/gloo/pkg/utils/kubeutils" - "github.com/solo-io/gloo/pkg/utils/requestutils/curl" "github.com/solo-io/gloo/test/kubernetes/e2e" ) -var _ e2e.NewSuiteFunc = NewTestingSuite +var _ e2e.NewSuiteFunc = NewEdgeTestingSuite -// serverTlsTestingSuite is the entire Suite of tests for gloo gateway proxy serving terminated TLS. +// edgeServerTlsTestingSuite is the entire Suite of tests for gloo gateway proxy serving terminated TLS. // The assertions in these tests assume that the warnMissingTlsSecret setting is "false" -type serverTlsTestingSuite struct { +type edgeServerTlsTestingSuite struct { suite.Suite ctx context.Context @@ -38,15 +37,15 @@ type serverTlsTestingSuite struct { vs1, vs2, vsWithOneWay, vsWithoutOneWay []byte } -func NewTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite { - return &serverTlsTestingSuite{ +func NewEdgeTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite { + return &edgeServerTlsTestingSuite{ ctx: ctx, testInstallation: testInst, ns: testInst.Metadata.InstallNamespace, } } -func (s *serverTlsTestingSuite) SetupSuite() { +func (s *edgeServerTlsTestingSuite) SetupSuite() { var err error // These functions each substitute our namespace for placeholders in the given manifest // file via os.ExpandEnv in order to place our referenced resources in our NS. @@ -70,14 +69,14 @@ func (s *serverTlsTestingSuite) SetupSuite() { } -func (s *serverTlsTestingSuite) TearDownSuite() { +func (s *edgeServerTlsTestingSuite) TearDownSuite() { err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, testdefaults.CurlPodManifest) s.NoError(err, "can delete Curl setup manifest") } // TestOneVirtualService validates the happy path that a VirtualService referencing an existent TLS secret // terminates TLS and responds appropriately. -func (s *serverTlsTestingSuite) TestOneVirtualService() { +func (s *edgeServerTlsTestingSuite) TestOneVirtualService() { vs1 := vs1(s.ns) s.T().Cleanup(func() { // ordering here matters if strict validation enabled @@ -105,7 +104,7 @@ func (s *serverTlsTestingSuite) TestOneVirtualService() { // TestTwoVirtualServices validates the happy path that two VirtualServices referencing existent TLS secrets // terminate TLS and respond appropriately. -func (s *serverTlsTestingSuite) TestTwoVirtualServices() { +func (s *edgeServerTlsTestingSuite) TestTwoVirtualServices() { vs1 := vs1(s.ns) vs2 := vs2(s.ns) s.T().Cleanup(func() { @@ -147,7 +146,7 @@ func (s *serverTlsTestingSuite) TestTwoVirtualServices() { // to test this properly, we require persistProxySpec to be off, validating that both VS are working correctly, // then we delete the secret for one of the VS and restart the Gloo pod. This ensures that we are still // serving on the other VS. -func (s *serverTlsTestingSuite) TestTwoVirtualServicesOneMissingTlsSecret() { +func (s *edgeServerTlsTestingSuite) TestTwoVirtualServicesOneMissingTlsSecret() { vs1 := vs1(s.ns) vs2 := vs2(s.ns) s.T().Cleanup(func() { @@ -207,7 +206,7 @@ func (s *serverTlsTestingSuite) TestTwoVirtualServicesOneMissingTlsSecret() { // TestOneWayServerTlsFailsWithoutOneWayTls validates that one-way server TLS traffic fails when CA data // is provided in the TLS secret. This is because the Gloo translation loop assumes that mTLS is desired // if the secret contains a CA cert. -func (s *serverTlsTestingSuite) TestOneWayServerTlsFailsWithoutOneWayTls() { +func (s *edgeServerTlsTestingSuite) TestOneWayServerTlsFailsWithoutOneWayTls() { vs := vsWithoutOneWay(s.ns) s.T().Cleanup(func() { // ordering here matters if strict validation enabled @@ -235,7 +234,7 @@ func (s *serverTlsTestingSuite) TestOneWayServerTlsFailsWithoutOneWayTls() { // TestOneWayServerTlsWorksWithOneWayTls validates that one-way server TLS traffic succeeds when CA data // is provided in the TLS secret IF oneWayTls is set on the sslConfig. This is because the Gloo translation // loop assumes that mTLS is desired if the secret contains a CA cert unless oneWayTls is set. -func (s *serverTlsTestingSuite) TestOneWayServerTlsWorksWithOneWayTls() { +func (s *edgeServerTlsTestingSuite) TestOneWayServerTlsWorksWithOneWayTls() { vs := vsWithOneWay(s.ns) s.T().Cleanup(func() { // ordering here matters if strict validation enabled @@ -259,19 +258,7 @@ func (s *serverTlsTestingSuite) TestOneWayServerTlsWorksWithOneWayTls() { s.assertEventualResponse(vs.GetName(), expectedHealthyResponseWithOneWay) } -func curlOptions(ns, hostHeaderValue string) []curl.Option { - return []curl.Option{ - curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: ns})), - // The host header must match the domain in the VirtualService - curl.WithHostHeader(hostHeaderValue), - curl.WithPort(443), - curl.IgnoreServerCert(), - curl.WithScheme("https"), - curl.WithSni(hostHeaderValue), - } -} - -func (s *serverTlsTestingSuite) assertEventualResponse(hostHeaderValue string, matcher *matchers.HttpResponse) { +func (s *edgeServerTlsTestingSuite) assertEventualResponse(hostHeaderValue string, matcher *matchers.HttpResponse) { // Make sure our proxy pod is running listOpts := metav1.ListOptions{ @@ -283,11 +270,11 @@ func (s *serverTlsTestingSuite) assertEventualResponse(hostHeaderValue string, m s.testInstallation.Assertions.AssertEventualCurlResponse( s.ctx, testdefaults.CurlPodExecOpt, - curlOptions(s.ns, hostHeaderValue), + curlOptions(defaults.GatewayProxyName, s.ns, hostHeaderValue), matcher) } -func (s *serverTlsTestingSuite) assertEventuallyConsistentResponse(hostHeaderValue string, matcher *matchers.HttpResponse, timeouts ...time.Duration) { +func (s *edgeServerTlsTestingSuite) assertEventuallyConsistentResponse(hostHeaderValue string, matcher *matchers.HttpResponse, timeouts ...time.Duration) { // Make sure our proxy pod is running listOpts := metav1.ListOptions{ @@ -299,12 +286,12 @@ func (s *serverTlsTestingSuite) assertEventuallyConsistentResponse(hostHeaderVal s.testInstallation.Assertions.AssertEventuallyConsistentCurlResponse( s.ctx, testdefaults.CurlPodExecOpt, - curlOptions(s.ns, hostHeaderValue), + curlOptions(defaults.GatewayProxyName, s.ns, hostHeaderValue), matcher, timeouts...) } -func (s *serverTlsTestingSuite) assertEventualError(hostHeaderValue string, code int) { +func (s *edgeServerTlsTestingSuite) assertEventualError(hostHeaderValue string, code int) { // Make sure our proxy pod is running listOpts := metav1.ListOptions{ @@ -316,11 +303,11 @@ func (s *serverTlsTestingSuite) assertEventualError(hostHeaderValue string, code s.testInstallation.Assertions.AssertEventualCurlError( s.ctx, testdefaults.CurlPodExecOpt, - curlOptions(s.ns, hostHeaderValue), + curlOptions(defaults.GatewayProxyName, s.ns, hostHeaderValue), code) } -func (s *serverTlsTestingSuite) eventuallyInSnapshot(gvk schema.GroupVersionKind, meta metav1.ObjectMeta) { +func (s *edgeServerTlsTestingSuite) eventuallyInSnapshot(gvk schema.GroupVersionKind, meta metav1.ObjectMeta) { s.testInstallation.Assertions.AssertGlooAdminApi( s.ctx, metav1.ObjectMeta{ @@ -330,7 +317,7 @@ func (s *serverTlsTestingSuite) eventuallyInSnapshot(gvk schema.GroupVersionKind s.testInstallation.Assertions.InputSnapshotContainsElement(gvk, meta), ) } -func (s *serverTlsTestingSuite) eventuallyNotInSnapshot(gvk schema.GroupVersionKind, meta metav1.ObjectMeta) { +func (s *edgeServerTlsTestingSuite) eventuallyNotInSnapshot(gvk schema.GroupVersionKind, meta metav1.ObjectMeta) { s.testInstallation.Assertions.AssertGlooAdminApi( s.ctx, metav1.ObjectMeta{ diff --git a/test/kubernetes/e2e/features/server_tls/k8s_suite.go b/test/kubernetes/e2e/features/server_tls/k8s_suite.go new file mode 100644 index 00000000000..3cad9bd3b73 --- /dev/null +++ b/test/kubernetes/e2e/features/server_tls/k8s_suite.go @@ -0,0 +1,135 @@ +package server_tls + +import ( + "context" + "net/http" + "time" + + "github.com/solo-io/gloo/test/gomega/matchers" + "github.com/stretchr/testify/suite" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" + "github.com/solo-io/gloo/pkg/utils/requestutils/curl" + "github.com/solo-io/gloo/test/kubernetes/e2e" +) + +var _ e2e.NewSuiteFunc = NewK8sTestingSuite + +// k8sServerTlsTestingSuite is the entire Suite of tests for gloo gateway proxy serving terminated TLS. +// The assertions in these tests assume that the warnMissingTlsSecret setting is "false" +type k8sServerTlsTestingSuite struct { + suite.Suite + + ctx context.Context + + // testInstallation contains all the metadata/utilities necessary to execute a series of tests + // against an installation of Gloo Gateway + testInstallation *e2e.TestInstallation + + // ns is the namespace in which the feature suite is being executed. + ns string +} + +func NewK8sTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite { + return &k8sServerTlsTestingSuite{ + ctx: ctx, + testInstallation: testInst, + ns: testInst.Metadata.InstallNamespace, + } +} + +var manifests = map[string]func() ([]byte, error){ + "tls_secret": tlsSecret1Manifest, + "tls_secret_with_ca": tlsSecretWithCaManifest, + "gateway": gatewayManifest, + "http_route": httpRouteManifest, + "setup": setupManifest, +} + +func (s *k8sServerTlsTestingSuite) SetupSuite() { + for key, manifest := range manifests { + manifestByt, err := manifest() + s.NoError(err, "can substitute NS in %s", key) + err = s.testInstallation.Actions.Kubectl().Apply(s.ctx, manifestByt, "-n", s.ns) + s.NoError(err, "can apply %s", key) + } + + // Make sure our proxy pod is running + s.testInstallation.Assertions.EventuallyPodsRunning( + s.ctx, + s.testInstallation.Metadata.InstallNamespace, + metav1.ListOptions{ + LabelSelector: "gloo=kube-gateway", + }, + time.Minute*2, + ) + s.testInstallation.Assertions.EventuallyPodsRunning( + s.ctx, + s.testInstallation.Metadata.InstallNamespace, + metav1.ListOptions{ + LabelSelector: "app=httpbin", + }, + time.Minute*2, + ) + s.testInstallation.Assertions.EventuallyPodsRunning( + s.ctx, + s.testInstallation.Metadata.InstallNamespace, + metav1.ListOptions{ + LabelSelector: "app=curl", + }, + time.Minute*2, + ) +} + +func (s *k8sServerTlsTestingSuite) TearDownSuite() { + for key, manifest := range manifests { + manifestByt, err := manifest() + s.NoError(err, "can substitute NS in %s", key) + err = s.testInstallation.Actions.Kubectl().Delete(s.ctx, manifestByt, "-n", s.ns) + s.NoError(err, "can delete %s", key) + } +} + +// TestOneWayServerTlsFailsWithoutOneWayTls validates that one-way server TLS traffic fails when CA data +// is provided in the TLS secret. This is because the Gloo translation loop assumes that mTLS is desired +// if the secret contains a CA cert. +func (s *k8sServerTlsTestingSuite) TestOneWayServerTlsFailsWithoutOneWayTls() { + s.assertEventualError("nooneway.example.com", expectedFailedResponseCodeInvalidVs) +} + +// TestOneWayServerTlsWorksWithOneWayTls validates that one-way server TLS traffic succeeds when CA data +// is provided in the TLS secret IF oneWayTls is set on the sslConfig. This is because the Gloo translation +// loop assumes that mTLS is desired if the secret contains a CA cert unless oneWayTls is set. +func (s *k8sServerTlsTestingSuite) TestOneWayServerTlsWorksWithOneWayTls() { + s.assertEventualResponse("oneway.example.com", &matchers.HttpResponse{ + StatusCode: http.StatusOK, + }) +} + +func (s *k8sServerTlsTestingSuite) assertEventualResponse(hostHeaderValue string, matcher *matchers.HttpResponse) { + + // Check curl works against expected response + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + kubectl.PodExecOptions{ + Name: "curl", + Namespace: s.ns, + Container: "curl", + }, + append(curlOptions("gloo-proxy-gw", s.ns, hostHeaderValue), curl.WithPath("/status/200")), + matcher) +} + +func (s *k8sServerTlsTestingSuite) assertEventualError(hostHeaderValue string, code int) { + // Check curl works against expected response + s.testInstallation.Assertions.AssertEventualCurlError( + s.ctx, + kubectl.PodExecOptions{ + Name: "curl", + Namespace: s.ns, + Container: "curl", + }, + append(curlOptions("gloo-proxy-gw", s.ns, hostHeaderValue), curl.WithPath("/status/200")), + code) +} diff --git a/test/kubernetes/e2e/features/server_tls/testdata/vs-1.yaml b/test/kubernetes/e2e/features/server_tls/testdata/edge/vs-1.yaml similarity index 100% rename from test/kubernetes/e2e/features/server_tls/testdata/vs-1.yaml rename to test/kubernetes/e2e/features/server_tls/testdata/edge/vs-1.yaml diff --git a/test/kubernetes/e2e/features/server_tls/testdata/vs-2.yaml b/test/kubernetes/e2e/features/server_tls/testdata/edge/vs-2.yaml similarity index 100% rename from test/kubernetes/e2e/features/server_tls/testdata/vs-2.yaml rename to test/kubernetes/e2e/features/server_tls/testdata/edge/vs-2.yaml diff --git a/test/kubernetes/e2e/features/server_tls/testdata/vs-with-oneway.yaml b/test/kubernetes/e2e/features/server_tls/testdata/edge/vs-with-oneway.yaml similarity index 100% rename from test/kubernetes/e2e/features/server_tls/testdata/vs-with-oneway.yaml rename to test/kubernetes/e2e/features/server_tls/testdata/edge/vs-with-oneway.yaml diff --git a/test/kubernetes/e2e/features/server_tls/testdata/vs-without-oneway.yaml b/test/kubernetes/e2e/features/server_tls/testdata/edge/vs-without-oneway.yaml similarity index 100% rename from test/kubernetes/e2e/features/server_tls/testdata/vs-without-oneway.yaml rename to test/kubernetes/e2e/features/server_tls/testdata/edge/vs-without-oneway.yaml diff --git a/test/kubernetes/e2e/features/server_tls/testdata/k8s/gateway.yaml b/test/kubernetes/e2e/features/server_tls/testdata/k8s/gateway.yaml new file mode 100644 index 00000000000..9c46463aef2 --- /dev/null +++ b/test/kubernetes/e2e/features/server_tls/testdata/k8s/gateway.yaml @@ -0,0 +1,45 @@ +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: gw +spec: + gatewayClassName: gloo-gateway + listeners: + - protocol: HTTPS + port: 443 + name: standard + hostname: "standard.example.com" + tls: + mode: Terminate + certificateRefs: + - name: tls-secret-1 + kind: Secret + allowedRoutes: + namespaces: + from: All + - protocol: HTTPS + port: 443 + name: oneway + hostname: "oneway.example.com" + tls: + mode: Terminate + certificateRefs: + - name: tls-secret-with-ca + kind: Secret + options: + "gateway.gloo.solo.io/ssl/one-way-tls": "true" + allowedRoutes: + namespaces: + from: All + - protocol: HTTPS + port: 443 + name: nooneway + hostname: "nooneway.example.com" + tls: + mode: Terminate + certificateRefs: + - name: tls-secret-with-ca + kind: Secret + allowedRoutes: + namespaces: + from: All diff --git a/test/kubernetes/e2e/features/server_tls/testdata/k8s/httproute.yaml b/test/kubernetes/e2e/features/server_tls/testdata/k8s/httproute.yaml new file mode 100644 index 00000000000..ef07d2db054 --- /dev/null +++ b/test/kubernetes/e2e/features/server_tls/testdata/k8s/httproute.yaml @@ -0,0 +1,14 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: httproute1 +spec: + parentRefs: + - name: gw + hostnames: + - "*.example.com" + rules: + - backendRefs: + - name: httpbin + port: 8000 + diff --git a/test/kubernetes/e2e/features/server_tls/testdata/k8s/setup.yaml b/test/kubernetes/e2e/features/server_tls/testdata/k8s/setup.yaml new file mode 100644 index 00000000000..86b79d94a1d --- /dev/null +++ b/test/kubernetes/e2e/features/server_tls/testdata/k8s/setup.yaml @@ -0,0 +1,78 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: httpbin +--- +apiVersion: v1 +kind: Service +metadata: + name: httpbin + labels: + app: httpbin + service: httpbin +spec: + ports: + - name: http + port: 8000 + targetPort: 8080 + - name: tcp + port: 9000 + selector: + app: httpbin +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: httpbin +spec: + replicas: 1 + selector: + matchLabels: + app: httpbin + version: v1 + template: + metadata: + labels: + app: httpbin + version: v1 + spec: + serviceAccountName: httpbin + containers: + - image: docker.io/mccutchen/go-httpbin:v2.6.0 + imagePullPolicy: IfNotPresent + name: httpbin + command: [ go-httpbin ] + args: + - "-port" + - "8080" + - "-max-duration" + - "600s" # override default 10s + ports: + - containerPort: 8080 + resources: + requests: + cpu: "100m" + limits: + cpu: "200m" +--- +apiVersion: v1 +kind: Pod +metadata: + name: curl + labels: + app: curl + version: v1 +spec: + containers: + - name: curl + image: curlimages/curl:7.83.1 + imagePullPolicy: IfNotPresent + command: + - "tail" + - "-f" + - "/dev/null" + resources: + requests: + cpu: "100m" + limits: + cpu: "200m" diff --git a/test/kubernetes/e2e/features/server_tls/types.go b/test/kubernetes/e2e/features/server_tls/types.go index 9096d2e25e2..32b855d4070 100644 --- a/test/kubernetes/e2e/features/server_tls/types.go +++ b/test/kubernetes/e2e/features/server_tls/types.go @@ -6,6 +6,8 @@ import ( "path/filepath" "github.com/onsi/gomega" + "github.com/solo-io/gloo/pkg/utils/kubeutils" + "github.com/solo-io/gloo/pkg/utils/requestutils/curl" kubev1 "github.com/solo-io/gloo/projects/gateway/pkg/api/v1/kube/apis/gateway.solo.io/v1" "github.com/solo-io/gloo/test/gomega/matchers" "github.com/solo-io/skv2/codegen/util" @@ -19,10 +21,15 @@ var ( tlsSecret1Manifest = func() ([]byte, error) { return manifestFromFile("tls-secret-1.yaml") } tlsSecret2Manifest = func() ([]byte, error) { return manifestFromFile("tls-secret-2.yaml") } tlsSecretWithCaManifest = func() ([]byte, error) { return manifestFromFile("tls-secret-with-ca.yaml") } - vs1Manifest = func() ([]byte, error) { return manifestFromFile("vs-1.yaml") } - vs2Manifest = func() ([]byte, error) { return manifestFromFile("vs-2.yaml") } - vsWithOneWayManifest = func() ([]byte, error) { return manifestFromFile("vs-with-oneway.yaml") } - vsWithoutOneWayManifest = func() ([]byte, error) { return manifestFromFile("vs-without-oneway.yaml") } + + vs1Manifest = func() ([]byte, error) { return manifestFromFile("edge/vs-1.yaml") } + vs2Manifest = func() ([]byte, error) { return manifestFromFile("edge/vs-2.yaml") } + vsWithOneWayManifest = func() ([]byte, error) { return manifestFromFile("edge/vs-with-oneway.yaml") } + vsWithoutOneWayManifest = func() ([]byte, error) { return manifestFromFile("edge/vs-without-oneway.yaml") } + + gatewayManifest = func() ([]byte, error) { return manifestFromFile("k8s/gateway.yaml") } + httpRouteManifest = func() ([]byte, error) { return manifestFromFile("k8s/httproute.yaml") } + setupManifest = func() ([]byte, error) { return manifestFromFile("k8s/setup.yaml") } // When we apply the deployer-provision.yaml file, we expect resources to be created with this metadata glooProxyObjectMeta = func(ns string) metav1.ObjectMeta { @@ -135,3 +142,15 @@ func withSubstitutions(fname string) ([]byte, error) { // Replace environment variables placeholders with their values return []byte(os.ExpandEnv(string(raw))), nil } + +func curlOptions(name, ns, hostHeaderValue string) []curl.Option { + return []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: name, Namespace: ns})), + // The host header must match the domain in the VirtualService + curl.WithHostHeader(hostHeaderValue), + curl.WithPort(443), + curl.IgnoreServerCert(), + curl.WithScheme("https"), + curl.WithSni(hostHeaderValue), + } +} diff --git a/test/kubernetes/e2e/tests/k8s_gw_tests.go b/test/kubernetes/e2e/tests/k8s_gw_tests.go index 0c6f336bf25..ff0a9dc046a 100644 --- a/test/kubernetes/e2e/tests/k8s_gw_tests.go +++ b/test/kubernetes/e2e/tests/k8s_gw_tests.go @@ -13,6 +13,7 @@ import ( "github.com/solo-io/gloo/test/kubernetes/e2e/features/port_routing" "github.com/solo-io/gloo/test/kubernetes/e2e/features/route_delegation" "github.com/solo-io/gloo/test/kubernetes/e2e/features/route_options" + "github.com/solo-io/gloo/test/kubernetes/e2e/features/server_tls" "github.com/solo-io/gloo/test/kubernetes/e2e/features/services/httproute" "github.com/solo-io/gloo/test/kubernetes/e2e/features/services/tcproute" "github.com/solo-io/gloo/test/kubernetes/e2e/features/upstreams" @@ -37,6 +38,7 @@ func KubeGatewaySuiteRunner() e2e.SuiteRunner { kubeGatewaySuiteRunner.Register("DirectResponse", directresponse.NewTestingSuite) kubeGatewaySuiteRunner.Register("CRDCategories", crd_categories.NewTestingSuite) kubeGatewaySuiteRunner.Register("Metrics", metrics.NewTestingSuite) + kubeGatewaySuiteRunner.Register("ServerTls", server_tls.NewK8sTestingSuite) return kubeGatewaySuiteRunner } diff --git a/test/kubernetes/e2e/tests/validation_always_accept_tests.go b/test/kubernetes/e2e/tests/validation_always_accept_tests.go index f73e7e8f4ee..5821eae82e5 100644 --- a/test/kubernetes/e2e/tests/validation_always_accept_tests.go +++ b/test/kubernetes/e2e/tests/validation_always_accept_tests.go @@ -14,7 +14,7 @@ func ValidationAlwaysAcceptSuiteRunner() e2e.SuiteRunner { validationSuiteRunner.Register("ValidationAllowWarnings", validation_allow_warnings.NewTestingSuite) // Server TLS tests are run here because they rely on VirtualService resources being applied // with missing TLS references. This is an error in validation unless warnMissingTlsSecret=true - validationSuiteRunner.Register("ServerTls", server_tls.NewTestingSuite) + validationSuiteRunner.Register("ServerTls", server_tls.NewEdgeTestingSuite) return validationSuiteRunner }